[WIP] feat: migrate containers to vmd102066

Justfile

@@ -31,7 +31,7 @@ _render_templates:
 		echo No route to WAN. Skipping template rendering...
 	else
 		source $(just -v _get_nix_path {{invocation_directory()}}/nix/variables/versions.nix)
-		nix/scripts/pre-eval-fixed.sh nix/home-manager/profiles/dotfiles/vcsh{.tmpl,}.nix
+		# nix/scripts/pre-eval-fixed.sh nix/home-manager/profiles/dotfiles/vcsh{.tmpl,}.nix
 	fi
 
 _rebuild-device dir rebuildarg="dry-activate" +moreargs="": _render_templates

README.md

@@ -30,12 +30,16 @@ In the unlikely case that you actually read this and have any questions please d
   - [x] declare shell.nix with runtime deps
   - [x] partition/encrypt/format disks
 - [ ] Document bootstrap process
+  - [ ] `apt install sudo cryptsetup` as a requirements on a deb admin machine
   - [ ] a new machine
   - [ ] an install media
 - [ ] Design disaster recovery
 - [ ] Automatic synchronization of other state files - see https://gitlab.com/steveeJ/nix-expressions/issues/2
 - [ ] Recycle *\_archived*
 - [x] Maybe make this a nix-overlay
+- [ ] container migrations
+  - [ ] ensure DDNS is updated _before_ the containers are started
+
 
 ## Bugs
 - [ ] home-manager leaves ~/.gnupg at 0755
@@ -66,3 +70,11 @@ just --list
 2. disconnect remove the previous drive
 3. replace the driveId in the device's hw.nix
 4. run the `just disk-relabel nix/os/devices/<deviceName> <prevDiskId>` command to rename the filesystem and volume group
+
+## Backup
+
+### Copy existing subvolumes to new backup target
+
+```
+`systemctl cat bkp-run | grep ExecStart | awk -F '=' '{print $2}'` --verbose --progress archive /var/lib/container-volumes ssh://[IP]:[PORT]/mnt/backup/container-volumes/
+```

nix/os/containers/backup-target.nix

@@ -1,7 +1,7 @@
 { hostAddress, localAddress, containerBackupCfg
-, sshPort ? containerBackupCfg.portInt }: {
+, sshPort ? containerBackupCfg.portInt, autoStart ? false }: {
   config = { config, pkgs, lib, ... }: {
-    system.stateVersion = "21.11"; # Did you read the comment?
+    system.stateVersion = "22.05"; # Did you read the comment?
 
     imports = [ ../profiles/containers/configuration.nix ];
 
@@ -50,7 +50,7 @@
     };
   };
 
-  autoStart = true;
+  inherit autoStart;
 
   bindMounts = {
     "/${containerBackupCfg.targetPath}" = {
@@ -59,6 +59,8 @@
     };
   };
 
+  extraFlags = [ "--resolv-conf=bind-host" ];
+
   privateNetwork = true;
   forwardPorts = [{
     # ssh

nix/os/containers/backup.nix

@@ -1,9 +1,5 @@
-<<<<<<< HEAD
-{ config, hostAddress, localAddress, subvolumes, targetPathSuffix ? "" }:
-=======
 { config, hostAddress, localAddress, subvolumes, targetPathSuffix ? ""
 , autoStart ? false }:
->>>>>>> 82ff04b (chore: nixfmt *)
 
 let
   passwords = import ../../variables/passwords.crypt.nix;
@@ -89,7 +85,7 @@ in {
     };
   };
 
-  autoStart = true;
+  inherit autoStart;
 
   bindMounts = {
     "${subvolumeParentDir}" = {
@@ -113,11 +109,8 @@ in {
     modifier = "rw";
   }];
 
-<<<<<<< HEAD
-=======
   extraFlags = [ "--resolv-conf=bind-host" ];
 
->>>>>>> 82ff04b (chore: nixfmt *)
   privateNetwork = true;
   forwardPorts = [ ];
 

nix/os/containers/mailserver.nix

@@ -1,4 +1,5 @@
-{ hostAddress, localAddress, imapsPort ? 993, sievePort ? 4190 }:
+{ hostAddress, localAddress, imapsPort ? 993, sievePort ? 4190
+, autoStart ? false }:
 
 let passwords = import ../../variables/passwords.crypt.nix;
 
@@ -124,7 +125,7 @@ in {
     };
   };
 
-  autoStart = true;
+  inherit autoStart;
 
   bindMounts = {
     "/etc/secrets/" = {
@@ -138,6 +139,8 @@ in {
     };
   };
 
+  extraFlags = [ "--resolv-conf=bind-host" ];
+
   privateNetwork = true;
   forwardPorts = [
     {

nix/os/containers/syncthing.nix

@@ -1,5 +1,5 @@
 { hostAddress, localAddress, syncthingPort ? 22000
-, syncthingLocalAnnouncePort ? 21027 }:
+, syncthingLocalAnnouncePort ? 21027, autoStart ? false }:
 
 {
 
@@ -21,7 +21,7 @@
     };
   };
 
-  autoStart = true;
+  inherit autoStart;
 
   bindMounts = {
     "/var/lib/syncthing/" = {
@@ -30,6 +30,8 @@
     };
   };
 
+  extraFlags = [ "--resolv-conf=bind-host" ];
+
   privateNetwork = true;
   forwardPorts = [
     {

nix/os/containers/webserver.nix

@@ -1,4 +1,5 @@
-{ hostAddress, localAddress, httpPort ? 80, httpsPort ? 443 }:
+{ hostAddress, localAddress, httpPort ? 80, httpsPort ? 443, autoStart ? false
+}:
 
 let passwords = import ../../variables/passwords.crypt.nix;
 in {
@@ -121,7 +122,7 @@ in {
     };
   };
 
-  autoStart = true;
+  inherit autoStart;
 
   bindMounts = {
     "/etc/secrets/" = {
@@ -145,6 +146,8 @@ in {
     };
   };
 
+  extraFlags = [ "--resolv-conf=bind-host" ];
+
   privateNetwork = true;
   forwardPorts = [
     {

nix/os/devices/disk.nix

@@ -50,7 +50,7 @@ in rec {
     [[ -e "''${NIXOS_CONFIG}" ]]
     [[ -e "${mntRootVol}/nixos" ]]
 
-    sudo -E $SHELL <<EOF
+    sudo --preserve-env=PATH -E $SHELL <<EOF
     # 'having $system set breaks nixos-install'
     unset system
     echo $NIX_PATH
@@ -134,7 +134,7 @@ in rec {
     sudo mount ${ownLib.disk.rootFsDevice diskId} ${mntRootVol}
     sudo btrfs subvolume create ${mntRootVol}/nixos
     sudo btrfs subvolume create ${mntRootVol}/home
-    sudo mkdir ${mntRootVol}/nixos/{boot,home}
+    sudo mkdir ${mntRootVol}/nixos/{boot,home,tmp}
 
     ${diskUmount}
   '';

nix/os/devices/srv0.home-ch.stefanjunker.de/system.nix

@@ -2,6 +2,7 @@
 
 let
   keys = import ../../../variables/keys.nix;
+  passwords = import ../../../variables/passwords.crypt.nix;
 
 in {
   # TASK: new device
@@ -50,37 +51,7 @@ in {
   networking.useHostResolvConf = false;
   services.resolved = { enable = true; };
 
-  containers = {
+  containers = { };
-    mailserver = import ../../containers/mailserver.nix {
-      hostAddress = "192.168.100.10";
-      localAddress = "192.168.100.11";
-
-      imapsPort = 993;
-      sievePort = 4190;
-    };
-
-    webserver = import ../../containers/webserver.nix {
-      hostAddress = "192.168.100.12";
-      localAddress = "192.168.100.13";
-
-      httpPort = 80;
-      httpsPort = 443;
-    };
-
-    syncthing = import ../../containers/syncthing.nix {
-      hostAddress = "192.168.100.14";
-      localAddress = "192.168.100.15";
-
-      syncthingPort = 22000;
-    };
-
-    backup = import ../../containers/backup.nix {
-      inherit config;
-      hostAddress = "192.168.100.16";
-      localAddress = "192.168.100.17";
-      subvolumes = [ "mailserver" "webserver" "backup" "syncthing" ];
-    };
-  };
 
   # This value determines the NixOS release from which the default
   # settings for stateful data, like file locations and database versions

nix/os/devices/vmd102066.contaboserver.net/boot.nix

@@ -0,0 +1,6 @@
+{ lib, ... }:
+
+{
+  boot.loader.grub.efiSupport = lib.mkForce false;
+  boot.extraModulePackages = [ ];
+}

nix/os/devices/vmd102066.contaboserver.net/configuration.nix

@@ -0,0 +1,14 @@
+{ ... }:
+
+{
+  disabledModules = [ ];
+  imports = [
+    ../../profiles/common/configuration.nix
+    ../../modules/opinionatedDisk.nix
+
+    ./system.nix
+    ./hw.nix
+    ./pkg.nix
+    ./boot.nix
+  ];
+}

nix/os/devices/vmd102066.contaboserver.net/hw.nix

@@ -0,0 +1,28 @@
+{ ... }:
+
+let
+  stage1Modules = [
+    "aesni_intel"
+    "kvm-intel"
+
+    "virtio_balloon"
+    "virtio_scsi"
+    "virtio_net"
+    "virtio_pci"
+    "virtio_ring"
+    "virtio"
+    "scsi_mod"
+  ];
+
+in {
+  # TASK: new device
+  hardware.opinionatedDisk = {
+    enable = true;
+    encrypted = true;
+    diskId = "scsi-0QEMU_QEMU_HARDDISK_drive-scsi0";
+  };
+
+  boot.initrd.availableKernelModules = stage1Modules;
+  boot.initrd.kernelModules = stage1Modules;
+  boot.extraModprobeConfig = "";
+}

nix/os/devices/vmd102066.contaboserver.net/pkg.nix

@@ -0,0 +1,51 @@
+{ config, pkgs, lib, ... }:
+
+{
+  nixpkgs.config.packageOverrides = pkgs:
+    with pkgs; {
+      nixPath = (import ../../../default.nix {
+        versionsPath = ./versions.nix;
+      }).nixPath;
+    };
+  home-manager.users.steveej =
+    import ../../../home-manager/configuration/text-minimal.nix {
+      inherit pkgs;
+    };
+
+  nix.buildMachines = [{
+    hostName = "localhost";
+    system = "x86_64-linux";
+    supportedFeatures = [ "kvm" "nixos-test" "big-parallel" "benchmark" ];
+    maxJobs = 4;
+  }];
+
+  services.hydra = {
+    enable = false;
+    hydraURL = "http://localhost:3000"; # externally visible URL
+    notificationSender =
+      "hydra@${config.networking.hostName}.stefanjunker.de"; # e-mail of hydra service
+    # a standalone hydra will require you to unset the buildMachinesFiles list to avoid using a nonexistant /etc/nix/machines
+    buildMachinesFiles = [ ];
+    # you will probably also want, otherwise *everything* will be built from scratch
+    useSubstitutes = true;
+  };
+
+  services.gitlab-runner = {
+    enable = false;
+
+    extraPackages = with pkgs; [ bash gitlab-runner nix gitFull git-crypt ];
+
+    concurrent = 2;
+    checkInterval = 0;
+    services = {
+      nixRunner = {
+        executor = "shell";
+        runUntagged = true;
+        registrationConfigFile =
+          "/etc/secrets/gitlab-runner/nix-runner.registration";
+        tagList = [ "nix" ];
+      };
+    };
+
+  };
+}

nix/os/devices/vmd102066.contaboserver.net/system.nix

@@ -0,0 +1,155 @@
+{ pkgs, lib, config, ... }:
+
+let
+  keys = import ../../../variables/keys.nix;
+  passwords = import ../../../variables/passwords.crypt.nix;
+
+in {
+  # TASK: new device
+  networking.hostName = "vmd102066"; # Define your hostname.
+  networking.domain = "contaboserver.net";
+
+  networking.firewall.enable = true;
+  networking.firewall.allowedTCPPorts = [
+    # iperf3
+    5201
+  ];
+  networking.firewall.logRefusedConnections = false;
+
+  networking.usePredictableInterfaceNames = false;
+  networking.dhcpcd = {
+    enable = true;
+    persistent = true;
+  };
+
+  networking.interfaces.eth0 = {
+    useDHCP = true;
+    ipv6.addresses = [{
+      address = "2a02:c206:3010:2066::1";
+      prefixLength = 64;
+    }];
+  };
+  networking.defaultGateway6 = {
+    address = "fe80::1";
+    interface = "eth0";
+  };
+
+  networking.nat = {
+    enable = true;
+    internalInterfaces = [ "ve-+" ];
+    externalInterface = "eth0";
+  };
+
+  # Kubernetes
+  # services.kubernetes.roles = ["master" "node"];
+
+  # virtualization
+  virtualisation = { docker.enable = true; };
+
+  services.spice-vdagentd.enable = true;
+  services.qemuGuest.enable = true;
+
+  systemd.services."sshd-status" = {
+    enable = true;
+    description = "sshd-status service";
+    path = [ pkgs.systemd ];
+    script = ''
+      systemctl status sshd | grep -i tasks
+    '';
+  };
+
+  systemd.services.sshd.serviceConfig = { TasksMax = 32; };
+
+  systemd.timers."sshd-status" = {
+    description = "Timer to trigger sshd-status periodically";
+    enable = true;
+    wantedBy = [ "timer.target" "multi-user.target" ];
+    timerConfig = {
+      OnActiveSec = "5s";
+      OnUnitActiveSec = "5s";
+      AccuracySec = "1s";
+      Unit = "sshd-status.service";
+    };
+  };
+
+  nix.gc = { automatic = true; };
+
+  boot.initrd.network = {
+    enable = true;
+    udhcpc.extraArgs = [ "-x hostname:${config.networking.hostName}" ];
+
+    ssh = {
+      enable = true;
+      authorizedKeys = keys.users.steveej.openssh;
+      hostKeys = [
+        "/etc/secrets/initrd/ssh_host_rsa_key"
+        "/etc/secrets/initrd/ssh_host_ed25519_key"
+      ];
+    };
+  };
+
+  boot.initrd.postMountCommands = ''
+    for iface in $(cd /sys/class/net && ls); do
+      echo "Bringing down $iface..."
+      ip address flush dev $iface
+      ip link set $iface down
+    done
+  '';
+
+  # networking.useHostResolvConf = true;
+
+  containers = {
+    mailserver = import ../../containers/mailserver.nix {
+      autoStart = true;
+
+      hostAddress = "192.168.100.10";
+      localAddress = "192.168.100.11";
+
+      imapsPort = 993;
+      sievePort = 4190;
+    };
+
+    webserver = import ../../containers/webserver.nix {
+      autoStart = true;
+
+      hostAddress = "192.168.100.12";
+      localAddress = "192.168.100.13";
+
+      httpPort = 80;
+      httpsPort = 443;
+    };
+
+    syncthing = import ../../containers/syncthing.nix {
+      autoStart = true;
+
+      hostAddress = "192.168.100.14";
+      localAddress = "192.168.100.15";
+
+      syncthingPort = 22000;
+    };
+
+    backup = import ../../containers/backup.nix {
+      autoStart = false;
+
+      inherit config;
+      hostAddress = "192.168.100.16";
+      localAddress = "192.168.100.17";
+      subvolumes = [ "mailserver" "webserver" "backup" "syncthing" ];
+    };
+
+    bkpTarget = import ../../containers/backup-target.nix {
+      autoStart = false;
+      hostAddress = "192.168.100.18";
+      localAddress = "192.168.100.19";
+      containerBackupCfg = passwords.storage.backupTarget;
+    };
+  };
+
+  # This value determines the NixOS release from which the default
+  # settings for stateful data, like file locations and database versions
+  # on your system were taken. It‘s perfectly fine and recommended to leave
+  # this value at the release version of the first install of this system.
+  # Before changing this value read the documentation for this option
+  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
+  system.stateVersion = "22.05"; # Did you read the comment?
+}

nix/os/devices/vmd102066.contaboserver.net/versions.nix

@@ -0,0 +1,21 @@
+let
+  nixpkgs = {
+    url = "https://github.com/NixOS/nixpkgs/";
+    ref = "nixos-22.05";
+    rev = "b3a8f7ed267e0a7ed100eb7d716c9137ff120fe3";
+  };
+
+in {
+  inherit nixpkgs;
+  "channels-nixos-stable" = nixpkgs;
+  "nixpkgs-master" = {
+    url = "https://github.com/NixOS/nixpkgs/";
+    ref = "master";
+    rev = "6b10854c8194f1ebaa5bce623e71c6da1c008861";
+  };
+  "home-manager-module" = {
+    url = "https://github.com/nix-community/home-manager";
+    ref = "release-22.05";
+    rev = "f0ecd4b1db5e15103e955b18cb94bea4296e5c45";
+  };
+}

nix/os/devices/vmd102066.contaboserver.net/versions.tmpl.nix

@@ -0,0 +1,27 @@
+let
+  nixpkgs = {
+    url = "https://github.com/NixOS/nixpkgs/";
+    ref = "nixos-22.05";
+    rev = ''
+      <% git ls-remote https://github.com/nixos/nixpkgs nixos-22.05 | awk '{ print $1 }' | tr -d '
+      ' -%>'';
+  };
+
+in {
+  inherit nixpkgs;
+  "channels-nixos-stable" = nixpkgs;
+  "nixpkgs-master" = {
+    url = "https://github.com/NixOS/nixpkgs/";
+    ref = "master";
+    rev = ''
+      <% git ls-remote https://github.com/NixOS/nixpkgs.git master | head -n1 | awk '{ print $1 }' | tr -d '
+      ' -%>'';
+  };
+  "home-manager-module" = {
+    url = "https://github.com/nix-community/home-manager";
+    ref = "release-22.05";
+    rev = ''
+      <% git ls-remote https://github.com/nix-community/home-manager.git release-22.05 | awk '{ print $1 }' | tr -d '
+      ' -%>'';
+  };
+}

nix/os/devices/vmd32387.contaboserver.net/pkg.nix

@@ -31,7 +31,7 @@
   };
 
   services.gitlab-runner = {
-    enable = true;
+    enable = false;
 
     extraPackages = with pkgs; [ bash gitlab-runner nix gitFull git-crypt ];
 

nix/os/devices/vmd32387.contaboserver.net/system.nix

@@ -98,13 +98,7 @@ in {
 
   networking.useHostResolvConf = true;
 
-  containers = {
+  containers = { };
-    bkpTarget = import ../../containers/backup-target.nix {
-      hostAddress = "192.168.100.18";
-      localAddress = "192.168.100.19";
-      containerBackupCfg = passwords.storage.backupTarget;
-    };
-  };
 
   # This value determines the NixOS release from which the default
   # settings for stateful data, like file locations and database versions

nix/os/devices/vmd32387.contaboserver.net/versions.tmpl.nix

@@ -1,51 +1,15 @@
 let
   nixpkgs = {
     url = "https://github.com/NixOS/nixpkgs/";
-    ref = "nixos-21.11";
+    ref = "nixos-22.05";
     rev = ''
-      <% git ls-remote https://github.com/nixos/nixpkgs nixos-21.11 | awk '{ print $1 }' | tr -d '
+      <% git ls-remote https://github.com/nixos/nixpkgs nixos-22.05 | awk '{ print $1 }' | tr -d '
       ' -%>'';
   };
 
 in {
   inherit nixpkgs;
-  nixos = nixpkgs // { suffix = "/nixos"; };
   "channels-nixos-stable" = nixpkgs;
-  "channels-nixos-21.05" = {
-    url = "https://github.com/NixOS/nixpkgs/";
-    ref = "nixos-21.05";
-    rev = ''
-      <% git ls-remote https://github.com/nixos/nixpkgs nixos-20.05 | awk '{ print $1 }' | tr -d '
-      ' -%>'';
-  };
-  "channels-nixos-20.09" = {
-    url = "https://github.com/NixOS/nixpkgs/";
-    ref = "nixos-20.09";
-    rev = ''
-      <% git ls-remote https://github.com/nixos/nixpkgs nixos-20.09 | awk '{ print $1 }' | tr -d '
-      ' -%>'';
-  };
-  "channels-nixos-20.03" = {
-    url = "https://github.com/NixOS/nixpkgs/";
-    ref = "nixos-20.03";
-    rev = ''
-      <% git ls-remote https://github.com/nixos/nixpkgs nixos-20.03 | awk '{ print $1 }' | tr -d '
-      ' -%>'';
-  };
-  "channels-nixos-19.09" = {
-    url = "https://github.com/NixOS/nixpkgs/";
-    ref = "nixos-19.09";
-    rev = ''
-      <% git ls-remote https://github.com/nixos/nixpkgs nixos-19.09 | awk '{ print $1 }' | tr -d '
-      ' -%>'';
-  };
-  "channels-nixos-unstable" = {
-    url = "https://github.com/NixOS/nixpkgs/";
-    ref = "nixos-unstable";
-    rev = ''
-      <% git ls-remote https://github.com/nixos/nixpkgs nixos-unstable | awk '{ print $1 }' | tr -d '
-      ' -%>'';
-  };
   "nixpkgs-master" = {
     url = "https://github.com/NixOS/nixpkgs/";
     ref = "master";
@@ -55,9 +19,9 @@ in {
   };
   "home-manager-module" = {
     url = "https://github.com/nix-community/home-manager";
-    ref = "release-21.05";
+    ref = "release-22.05";
     rev = ''
-      <% git ls-remote https://github.com/nix-community/home-manager.git release-21.05 | awk '{ print $1 }' | tr -d '
+      <% git ls-remote https://github.com/nix-community/home-manager.git release-22.05 | awk '{ print $1 }' | tr -d '
       ' -%>'';
   };
 }

shell.nix

@@ -23,6 +23,7 @@ in pkgs.stdenv.mkDerivation {
     git
     nixUnstable
     niv
+    nixos-install-tools
 
     vncdo
     tesseract