WIP k3s

flake.lock

@@ -585,6 +585,22 @@
         "type": "github"
       }
     },
+    "nixpkgs-kanidm": {
+      "locked": {
+        "lastModified": 1729071019,
+        "narHash": "sha256-c4J/ZiMbjMf98FawO5XJaTWqvrvIXpxnIpxu4OV3CGA=",
+        "owner": "steveej-forks",
+        "repo": "nixpkgs",
+        "rev": "984b1d5a286d3a072b840b30ec49d96878d01e64",
+        "type": "github"
+      },
+      "original": {
+        "owner": "steveej-forks",
+        "ref": "kanidm",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "nixpkgs-lib": {
       "locked": {
         "dir": "lib",
@@ -794,6 +810,7 @@
         "nixpkgs-2305": "nixpkgs-2305",
         "nixpkgs-2311": "nixpkgs-2311",
         "nixpkgs-2405": "nixpkgs-2405",
+        "nixpkgs-kanidm": "nixpkgs-kanidm",
         "nixpkgs-unstable": "nixpkgs-unstable",
         "nixpkgs-vscodium": "nixpkgs-vscodium",
         "nixpkgs-wayland": "nixpkgs-wayland",

flake.nix

@@ -125,6 +125,8 @@
       url = "git+https://git.codelinaro.org/clo/ath-firmware/ath11k-firmware.git";
       flake = false;
     };
+
+    nixpkgs-kanidm.url = "github:steveej-forks/nixpkgs/kanidm";
   };
 
   outputs = inputs @ {
@@ -360,6 +362,7 @@
         devShells = let
           all = import ./nix/devShells.nix {
             inherit
+              self
               self'
               inputs'
               pkgs

nix/devShells.nix

@@ -1,4 +1,5 @@
 {
+  self,
   self',
   inputs',
   pkgs,
@@ -82,9 +83,13 @@ in {
       wireguard-tools
 
       screen
+
+      inputs'.nixpkgs-kanidm.legacyPackages.kanidm
     ];
 
     # Set Environment Variables
     RUST_BACKTRACE = 1;
+
+    KANIDM_URL = self.nixosConfigurations.sj-srv1.config.containers.webserver.config.services.kanidm.serverSettings.origin;
   };
 }

nix/os/containers/mycelium/flake.lock

@@ -46,11 +46,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1715438114,
+        "lastModified": 1723875769,
-        "narHash": "sha256-btb702TXuhDg0D6tW0dCOy4+II9Wl6BJ0LvpT+O9wrs=",
+        "narHash": "sha256-66GofByLJ+S4ZZphIC+vJKeL9VJ2bzH2VbcJ3OqteMM=",
         "owner": "pdtpartners",
         "repo": "nix-snapshotter",
-        "rev": "7b251c9356bc7bb383ebeedcd0045b3ae431bff7",
+        "rev": "6eaadfd8f89e5e7d79b2013626bbd36e388159da",
         "type": "github"
       },
       "original": {
@@ -61,11 +61,11 @@
     },
     "nixlib": {
       "locked": {
-        "lastModified": 1712450863,
+        "lastModified": 1728781282,
-        "narHash": "sha256-K6IkdtMtq9xktmYPj0uaYc8NsIqHuaAoRBaMgu9Fvrw=",
+        "narHash": "sha256-hUP9oxmnOmNnKcDOf5Y55HQ+NnoT0+bLWHLQWLLw9Ks=",
         "owner": "nix-community",
         "repo": "nixpkgs.lib",
-        "rev": "3c62b6a12571c9a7f65ab037173ee153d539905f",
+        "rev": "16340f605f4e8e5cf07fd74dcbe692eee2d4f51b",
         "type": "github"
       },
       "original": {
@@ -82,11 +82,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1718025593,
+        "lastModified": 1728867876,
-        "narHash": "sha256-WZ1gdKq/9u1Ns/oXuNsDm+W0salonVA0VY1amw8urJ4=",
+        "narHash": "sha256-NCyOA8WZNoojmXH+kBDrQj3LwvakYNzSc0h+LTXkmPE=",
         "owner": "nix-community",
         "repo": "nixos-generators",
-        "rev": "35c20ba421dfa5059e20e0ef2343c875372bdcf3",
+        "rev": "fdf142111597f6c6283cf5ffe092b6293a3911d0",
         "type": "github"
       },
       "original": {
@@ -97,42 +97,25 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1718086528,
+        "lastModified": 1728897630,
-        "narHash": "sha256-hoB7B7oPgypePz16cKWawPfhVvMSXj4G/qLsfFuhFjw=",
+        "narHash": "sha256-0utJPs4o2Mody8GDwo4hnGuxc8dJqju4u9lLJY4d/Lw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "47b604b07d1e8146d5398b42d3306fdebd343986",
+        "rev": "c9f0b4a395289ce18727e2a8e43cae6796693ccc",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-24.05",
+        "ref": "nixos-unstable-small",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
-    "nixpkgs-systemd256": {
-      "locked": {
-        "lastModified": 1718397913,
-        "narHash": "sha256-omV+dq3GdXQQTaewxhkBgxM4Bbwqa4D9FVS4dTITxOQ=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "962cf03fb8c782c5e00f465397e03dc84284acc9",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "962cf03fb8c782c5e00f465397e03dc84284acc9",
-        "type": "github"
-      }
-    },
     "root": {
       "inputs": {
         "nix-snapshotter": "nix-snapshotter",
         "nixos-generators": "nixos-generators",
-        "nixpkgs": "nixpkgs",
+        "nixpkgs": "nixpkgs"
-        "nixpkgs-systemd256": "nixpkgs-systemd256"
       }
     }
   },

nix/os/containers/mycelium/flake.nix

@@ -1,7 +1,7 @@
 {
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
-    nixpkgs-systemd256.url = "github:NixOS/nixpkgs/962cf03fb8c782c5e00f465397e03dc84284acc9";
+    # nixpkgs-systemd256.url = "github:NixOS/nixpkgs/962cf03fb8c782c5e00f465397e03dc84284acc9";
     nixos-generators = {
       url = "github:nix-community/nixos-generators";
       inputs.nixpkgs.follows = "nixpkgs";
@@ -102,7 +102,7 @@
             imports = [
               (modulesPath + "/profiles/minimal.nix")
             ];
-            system.stateVersion = "24.05";
+            system.stateVersion = "24.11";
 
             # https://github.com/hercules-ci/arion/blob/c24c185e67f093298a081900b49ca18716077dec/src/nix/modules/nixos/container-systemd.nix
             boot.isContainer = true;
@@ -116,22 +116,37 @@
             services.nscd.enable = false;
             system.nssModules = lib.mkForce [];
             systemd.services.systemd-logind.enable = false;
-            systemd.services.console-getty.enable = true;
+            systemd.services.console-getty.enable = false;
 
             systemd.sockets.nix-daemon.enable = false;
             systemd.services.nix-daemon.enable = false;
             systemd.oomd.enable = false;
             networking.useDHCP = false;
             networking.firewall.enable = false;
+
+            # system.build.earlyMountScript =
+            #   lib.mkForce ''
+            #   '';
+            # system.activationScripts.specialfs =
+            #   lib.mkForce ''
+            #   '';
             boot.postBootCommands = ''
               ls -lha /run
               mkdir -p /run/wrappers
             '';
+
+            boot.kernelParams = [
+              "systemd.log_level=debug"
+            ];
+
             # services.udev.enable = false;
 
             # TODO: this is only needed because `/run/current-system` is missing
             # environment.variables.PATH = "${lib.makeBinPath config.environment.systemPackages}:$PATH";
 
+            systemd.mounts = lib.mkForce [];
+            fileSystems = lib.mkForce {};
+
             services.mycelium.enable = false;
             services.mycelium.keyFile = "/var/lib/secrets/mycelium-keyfile";
             systemd.services.mycelium.serviceConfig.DynamicUser = lib.mkForce false;
@@ -151,17 +166,18 @@
               serviceConfig = {
                 SyslogIdentifier = "testing-credential";
                 StateDirectory = "testing-credentials";
-                # DynamicUser = true;
+                DynamicUser = true;
                 # User = "tc";
                 # ProtectHome = true;
-                # ProtectSystem = true;
+                ProtectSystem = true;
-                LoadCredential = [
+                # LoadCredential = [
-                  "mycelium-keyfile:${self.nixosConfigurations.default.config.services.mycelium.keyFile}"
+                #   "mycelium-keyfile:${self.nixosConfigurations.default.config.services.mycelium.keyFile}"
-                  "hosts:/etc/hosts"
+                #   "hosts:/etc/hosts"
-                ];
+                # ];
-                SetCredential = "nosecret:not secret string";
+                SetCredential = "mycelium-keyfile:not secret string";
                 ExecStart = lib.mkForce (pkgs.writeShellScript "mycelium" ''
                   cd $STATE_DIRECTORY
+                  pwd
                   env
                   while true; do
                     ls -lha $CREDENTIALS_DIRECTORY
@@ -228,6 +244,9 @@
           ];
           volumes = {
             # "/var/lib/private/mycelium/key.bin" = {};
+            # "/run" = {};
+            # "/tmp" = {};
+            # "/etc" = {};
           };
           copyToRoot = [
             # self.nixosConfigurations.default.config.system.build.toplevel
@@ -312,6 +331,7 @@
           nix build --impure .#image
           sudo nix2container load ./result
           sudo -E nerdctl run  --name ${name} --privileged -dt \
+            --cgroup-manager cgroupfs \
             --volume "$PWD/key.bin.crypt:${self.nixosConfigurations.default.config.services.mycelium.keyFile}:ro" \
             "nix:0$(readlink result):latest"
         '';

nix/os/containers/webserver.nix

@@ -17,16 +17,19 @@ in {
     lib,
     repoFlake,
     nodeFlake,
+    system,
     ...
   }: {
     system.stateVersion = "22.05"; # Did you read the comment?
 
     disabledModules = [
       "services/misc/forgejo.nix"
+      "services/security/kanidm.nix"
     ];
 
     imports = [
       "${nodeFlake.inputs.nixpkgs-unstable}/nixos/modules/services/misc/forgejo.nix"
+      "${repoFlake.inputs.nixpkgs-kanidm}/nixos/modules/services/security/kanidm.nix"
 
       ../profiles/containers/configuration.nix
 
@@ -90,6 +93,16 @@ in {
           reverse_proxy http://127.0.0.1:${builtins.toString config.services.forgejo.settings.server.HTTP_PORT}
         '';
       };
+
+      virtualHosts."kanidm.${domain}" = {
+        extraConfig = ''
+          reverse_proxy https://${builtins.toString config.services.kanidm.serverSettings.bindaddress} {
+            transport http {
+                tls_server_name ${config.services.kanidm.serverSettings.domain}
+            }
+          }
+        '';
+      };
     };
 
     services.hedgedoc = {
@@ -116,12 +129,34 @@ in {
           url = "ldap://127.0.0.1:${builtins.toString config.services.lldap.settings.ldap_port}";
           bindDn = "uid=admin,ou=people,dc=stefanjunker,dc=de";
           # these are set via the `environmentFile`
-          bindCredentials = "$LDAP_ADMIN_PASSWORD";
+          # bindCredentials = "$LDAP_ADMIN_PASSWORD";
           searchBase = "ou=people,dc=stefanjunker,dc=de";
           searchFilter = "(&(memberOf=cn=hedgedoc,ou=groups,dc=stefanjunker,dc=de)(uid={{username}}))";
           useridField = "uid";
         };
 
+        oauth2 = let
+          originURL = config.services.kanidm.serverSettings.origin;
+        in {
+          providerName = "kanidm (${originURL})";
+
+          authorizationURL = "${originURL}/ui/oauth2";
+          tokenURL = "${originURL}/oauth2/token";
+          userProfileURL = "${originURL}/oauth2/openid/hedgedoc/userinfo";
+
+          scope = "openid email profile";
+          # rolesClaim = "roles";
+          # accessRole = "role/hedgedoc";
+
+          userProfileUsernameAttr = "name";
+          userProfileDisplayNameAttr = "displayname";
+          userProfileEmailAttr = "email";
+
+          clientID = "hedgedoc";
+          # set via the `environmentFile`
+          # clientSecret = "$CMD_OAUTH2_CLIENT_SECRET";
+        };
+
         uploadsPath = "/var/lib/hedgedoc/uploads";
       };
 
@@ -268,6 +303,108 @@ in {
     systemd.services.lldap.serviceConfig.User = config.users.users.lldap.name;
     systemd.services.lldap.serviceConfig.Group = config.users.groups.lldap.name;
     systemd.services.lldap.serviceConfig.DynamicUser = lib.mkForce false;
+
+    # combine a path watcher with a service that transfers the certs by caddy to kanidm
+    systemd.paths.kanidm-tls-watch = {
+      enable = true;
+      requiredBy = ["kanidm.service"];
+      pathConfig = {
+        PathChanged = [
+          "${config.services.caddy.dataDir}/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/${config.services.kanidm.serverSettings.domain}/${config.services.kanidm.serverSettings.domain}.key"
+          "${config.services.caddy.dataDir}/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/${config.services.kanidm.serverSettings.domain}/${config.services.kanidm.serverSettings.domain}.crt"
+        ];
+        Unit = "kanidm-tls-update.service";
+      };
+    };
+    systemd.services.kanidm-tls-update = let
+      dbDir =
+        builtins.dirOf
+        config.services.kanidm.serverSettings.db_path;
+    in {
+      enable = true;
+      requiredBy = ["kanidm.service"];
+      unitConfig = {
+        # ConditionPathExists = [
+        #   "${config.services.caddy.dataDir}/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/${config.services.kanidm.serverSettings.domain}/${config.services.kanidm.serverSettings.domain}.key"
+        #   "${config.services.caddy.dataDir}/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/${config.services.kanidm.serverSettings.domain}/${config.services.kanidm.serverSettings.domain}.crt"
+        # ];
+      };
+      serviceConfig.Type = "oneshot";
+      script = let
+        tlsDir = builtins.dirOf config.services.kanidm.serverSettings.tls_key;
+      in ''
+        set -xe
+
+        cat "${config.services.caddy.dataDir}/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/${config.services.kanidm.serverSettings.domain}/${config.services.kanidm.serverSettings.domain}.key" > tls.key
+        cat "${config.services.caddy.dataDir}/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/${config.services.kanidm.serverSettings.domain}/${config.services.kanidm.serverSettings.domain}.crt" > tls.chain
+
+        chown ${config.systemd.services.kanidm.serviceConfig.User}:${config.systemd.services.kanidm.serviceConfig.Group} tls.{key,chain}
+        chmod 400 tls.{key,chain}
+
+        # create the kanidm directory in case it's missing
+        if [[ ! -d ${tlsDir} ]]; then
+          mkdir -p ${tlsDir}
+          chown -R ${config.systemd.services.kanidm.serviceConfig.User}:${config.systemd.services.kanidm.serviceConfig.Group} ${tlsDir}
+          chmod 700 ${tlsDir}
+        fi
+
+        mv tls.key ${config.services.kanidm.serverSettings.tls_key}
+        mv tls.chain ${config.services.kanidm.serverSettings.tls_chain}
+
+        if [[ ! -d ${dbDir} ]]; then
+          mkdir -p ${dbDir}
+          chown -R ${config.systemd.services.kanidm.serviceConfig.User}:${config.systemd.services.kanidm.serviceConfig.Group} ${dbDir}
+          chmod 700 ${dbDir}
+        fi
+      '';
+    };
+
+    systemd.services.kanidm.serviceConfig = let
+      dbDir =
+        builtins.dirOf
+        config.services.kanidm.serverSettings.db_path;
+      # stateDir = "/var/lib/${config.systemd.services.kanidm.serviceConfig.StateDirectory}";
+    in {
+      # ExecStartPre = ''
+      #   mkdir -p ${dbDir}
+      # '';
+      BindPaths = [
+        dbDir
+        # stateDir
+      ];
+    };
+
+    services.kanidm = let
+      dataDir = "/var/lib/kanidm";
+    in {
+      package = repoFlake.inputs.nixpkgs-kanidm.legacyPackages.${pkgs.system}.kanidm;
+
+      enablePam = false;
+      enableClient = false;
+
+      enableServer = true;
+      serverSettings = {
+        role = "WriteReplica";
+        log_level = "debug";
+
+        domain = "kanidm.${domain}";
+        origin = "https://kanidm.${domain}";
+
+        db_path = "${dataDir}/db/kanidm.db";
+
+        bindaddress = "127.0.0.1:8444";
+
+        # don't expose ldap
+        # ldapbindaddress = "[::1]:6636";
+
+        tls_key = "${dataDir}/tls/tls.key";
+        tls_chain = "${dataDir}/tls/tls.chain";
+
+        online_backup = {
+          schedule = "00 06 * * *";
+        };
+      };
+    };
   };
 
   inherit autoStart;
@@ -306,6 +443,11 @@ in {
       hostPath = "/var/lib/container-volumes/webserver/var-lib-forgejo";
       isReadOnly = false;
     };
+
+    "/var/lib/kanidm" = {
+      hostPath = "/var/lib/container-volumes/webserver/var-lib-kanidm";
+      isReadOnly = false;
+    };
   };
 
   privateNetwork = true;

nix/os/containers/webserver_secrets.yaml

@@ -1,4 +1,4 @@
-hedgedoc_environment_file: ENC[AES256_GCM,data:ciVnpDXq5CZltHcAHJQNeKfelQlKhyXfGkUeuvwFBq8QUQDNEgLOVZ5X7Yw3kPGAvXEozK2Nz3aFfOpbGt76OmNdJ2TQNxOEpcHDJEvAoYSc/XTcctfDQmqga6MMWWAjIO3LXpFa9UD9riP6yUFNwGOB7waIvV7yD+D+QILwUyNda0/iVHtC/6HO8Yaj3nK6Fp1IDclppobIQ/MdzG+cy+yN7h0XUNOzMh91DGAC3ePIB5DX90wlXTzsox9HWWAUTh6Lpss=,iv:X7fROtc0Fn9AnZkWHAs8XFwIInBowQZzRJuLWSKSGWM=,tag:gKysRtqBhTtwLnxDv2QGBA==,type:str]
+hedgedoc_environment_file: ENC[AES256_GCM,data:gPTokPMGBAN/lGGeUs95vg45yVrrSmFCKWTjlMV4V+YnflcqiaZvifX9+0fe3DELwNL4kY4st4N0MadhLkTiSieyp46fP8Dujk4Prhi7JWweBDsN4WtxcwJfAdowgh5LTzqM3zggC/J9NGR/zgJGLYraOqsFueXycxDxntE+8MlepYFGsND4WbFHNRvsVd7xUWerZZD+JFhws2sjwC9DqoJ+mBX4u9J2faSrL3okBGwRpEZlJhe6/8pT0l1aVxI0b/9UsLUL/him/vVqY8ygMP8O95gzuDEaCtwSXw08ylhb3g3YHdMh9ZOe9dPNVocVFrB15HfxeY4KzRCVfvgmBsSiUrgUAZQ8aav2ZWHPKQ==,iv:AVtx/43MK5KVxP59olEmbkUzLhd0cBjPpVeiAJGELfM=,tag:Hd3edeUzLgHnwAwPiMGp4A==,type:str]
 authelia_storageEncryptionKey: ENC[AES256_GCM,data:BLj2+w0jUOiiqljd4WcYt90I1Y/tm6vYRn9IPvv7k+ykjeWeafDcb9LpAbYZ82iAbCDZ6cDnTIJSH50TYFM1Bw==,iv:DfrFuTMS6rzWS1F7hIloD7Eb9N6LV6MDsPGEidTCIkM=,tag:4TxiUuWj8LLCAe+3SQWthA==,type:str]
 authelia_jwtSecret: ENC[AES256_GCM,data:CtvhPJDlM8kZDh3MWI6jBEIK5P/Rc0ZY3JfQ/1qn2FyZ0kOE9CMznRQdzi125Mlh+LgfhS00aeY/esuSJ1YKnw==,iv:o29+Ja4E4USpxD5Bgg4nGpj1WdK+uJJeuyqa3c92Pys=,tag:Ko/6PqIM7FyvqMJAepTNKw==,type:str]
 lldap_jwtSecret: ENC[AES256_GCM,data:Jd05K5zZZcdFY9P+PM1ycpl4eKr7THXzlEC2gxhfq0kZ0c7NrmqpnxLFyMOoq0oZwFXQ3aE29N+dYYCbWgEs0g==,iv:k9Y4XZcFSUaaOvooMzH1AlHaI/RiZZ0cpSqTU6jzsUU=,tag:EHwIMQ00Z+IrXAKgugMbmQ==,type:str]
@@ -23,8 +23,8 @@ sops:
             eWdXVHg5MTlQQU9GeElPeUZXUlBlaTgKceDu3tLbQM/DxY0tJYJTPy2Dl/SBYaoc
             KfMZOkc322/NvgWu/3Ke0hV1/eMk8EICwXbSwHhXr5a0+cwPZ9xV4A==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-10-13T17:41:14Z"
+    lastmodified: "2024-10-16T12:28:51Z"
-    mac: ENC[AES256_GCM,data:1mqRRPa4tP1OFxC3Oo5uJhk3H79jxObUeIsIab8fOrafsrw9tbrqpb9lRgziR3C0ssDagb0deA6PAGH6YWvSU716Ayr3p+Ih2sXOkbkp8wV/u3AULsDUzSUglshgM5f1Hf5jvL7xoWBOzek8eMGIkFFFwu0VmkqwpqOalXY0Kxk=,iv:cC4hRQZlLuOyktS0pER6Ef0f7qVxMXfS8w9Q5p7AlTA=,tag:/maJgYz/Ks3iaQZr+WSUUA==,type:str]
+    mac: ENC[AES256_GCM,data:nrd2czzJlBcFfwn6lzh4qqco+/XsU2J6BqvQqMtskh3mL4Xx25IAzxiCno0KlNGr6o4YsuZP5anOX9RvrDq76Us3JQ7pDi3iQGPhmg+SE9u3Rwqn1/3YConvdfPV2DNB+tuyG3UVoRqpA4d+HdcYjN9n1UKk54R6UdSm9UrA+zc=,iv:Juupyet09zUAMu7bmVxq+/Q0bXJAzR0wAyt6vKNns3w=,tag:owdUWuXrQcDdiWi+1geY9A==,type:str]
     pgp:
         - created_at: "2023-07-09T17:51:27Z"
           enc: |-

nix/os/devices/router0-dmz0/configuration.nix

@@ -1260,6 +1260,7 @@ in {
         "jitsi.www.stefanjunker.de,${dmzExposedHost}"
         "lldap.www.stefanjunker.de,${dmzExposedHost}"
         "forgejo.www.stefanjunker.de,${dmzExposedHost}"
+        "kanidm.www.stefanjunker.de,${dmzExposedHost}"
       ];
     };
   };

nix/os/devices/sj-srv1/flake.lock

@@ -23,11 +23,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1728328465,
+        "lastModified": 1728909085,
-        "narHash": "sha256-a0a0M1TmXMK34y3M0cugsmpJ4FJPT/xsblhpiiX1CXo=",
+        "narHash": "sha256-WLxED18lodtQiayIPDE5zwAfkPJSjHJ35UhZ8h3cJUg=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "1bfbbbe5bbf888d675397c66bfdb275d0b99361c",
+        "rev": "c0b1da36f7c34a7146501f684e9ebdf15d2bebf8",
         "type": "github"
       },
       "original": {
@@ -39,11 +39,11 @@
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1728543552,
+        "lastModified": 1729086167,
-        "narHash": "sha256-3OR+2XHHo+USlAz7T30VKnPxR7k3GeErkXM0Wm/Ctzw=",
+        "narHash": "sha256-Vh6kOiQHefsr6Zin4Xi+VH06leuNZuMyP8YkkGo/Naw=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "f4f573fde42d181f22c95e10822856399c24feeb",
+        "rev": "6b1ffdb0976ac367aeea173b8e69de62828a4ca7",
         "type": "github"
       },
       "original": {
@@ -55,11 +55,11 @@
     },
     "nixpkgs-unstable": {
       "locked": {
-        "lastModified": 1728534991,
+        "lastModified": 1729077633,
-        "narHash": "sha256-wLUZyvtOOowAz0kTrU2MoC4nXWniFaVezGyzuEt5HPc=",
+        "narHash": "sha256-6sIuRVqVMHq9ZwcEVdpf2BuZeuLIUgvFznhIfsc75Jo=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "6b955bdbb9efe4a5c047746323951fe1bdf8d01b",
+        "rev": "8f1d45587bd9af3dbf5146aa8a1347e20421597b",
         "type": "github"
       },
       "original": {

nix/os/devices/sj-srv1/system.nix

@@ -11,6 +11,23 @@
 in {
   imports = [
     ../../snippets/systemd-resolved.nix
+    {
+      # make sure it uses the DNS that comes in via DHCP
+      networking.nameservers = lib.mkForce [];
+      services.resolved.enable = true;
+
+      # provide DNS to the containers
+      services.resolved.extraConfig = ''
+        DNSStubListenerExtra=${hostBridgeAddress}
+      '';
+      networking.firewall.interfaces.br0.allowedTCPPorts = [53];
+      networking.firewall.interfaces.br0.allowedUDPPorts = [53];
+    }
+  ];
+
+  programs.wireshark.enable = true;
+  environment.systemPackages = [
+    pkgs.dnsutils
   ];
 
   networking.firewall.enable = true;
@@ -83,6 +100,9 @@ in {
     enable = true;
     matchConfig.Name = "dmz0";
     DHCP = "yes";
+
+    dhcpV4Config.UseDNS = true;
+    dhcpV6Config.UseDNS = true;
   };
 
   boot.kernel.sysctl = {
@@ -134,6 +154,7 @@ in {
     mailserver = import ../../containers/mailserver.nix {
       specialArgs = {
         inherit repoFlake nodeFlake;
+        hostAddress = hostBridgeAddress;
       };
 
       autoStart = true;
@@ -151,6 +172,7 @@ in {
       {
         specialArgs = {
           inherit repoFlake nodeFlake;
+          hostAddress = hostBridgeAddress;
         };
 
         autoStart = true;
@@ -167,6 +189,7 @@ in {
     syncthing = import ../../containers/syncthing.nix {
       specialArgs = {
         inherit repoFlake nodeFlake;
+        hostAddress = hostBridgeAddress;
       };
       autoStart = true;
 

nix/os/profiles/containers/configuration.nix

@@ -1,16 +1,29 @@
-{pkgs, ...}: {
+{
+  hostAddress,
+  pkgs,
+  lib,
+  ...
+}: {
   networking.useHostResolvConf = false;
 
   networking.firewall.enable = true;
   networking.nftables.enable = true;
   networking.nftables.flushRuleset = true;
 
+  networking.nameservers = lib.mkForce [hostAddress];
+
   environment.systemPackages = [
     pkgs.dnsutils
   ];
 
   imports = [
-    ../../snippets/systemd-resolved.nix
+    {
+      # keep DNS set up to a minimum: only query the container host
+      services.resolved.enable = lib.mkForce false;
+      networking.nameservers = [
+        hostAddress
+      ];
+    }
     ../../snippets/nix-settings.nix
     # ../../modules/ddclient-ovh.nix
     # ../../modules/ddclient-hetzner.nix

nix/os/snippets/k3s-w-nix-snapshotter.nix

@@ -5,37 +5,56 @@
   pkgs,
   lib,
   system,
+  config,
   ...
-}: {
+}: let
+  cfg = config.steveej.k3s;
+
+  # TODO: make this configurable
+  homeUser = "steveej";
+in {
+  options.steveej.k3s = {
+    enable = lib.mkOption {
+      description = "steveej's k3s distro";
+      type = lib.types.bool;
+      default = true;
+    };
+  };
+
   # (1) Import nixos module.
   imports = [
     nodeFlake.inputs.nix-snapshotter.nixosModules.default
   ];
 
-  # (2) Add overlay.
+  config = lib.mkIf cfg.enable {
-  nixpkgs.overlays = [nodeFlake.inputs.nix-snapshotter.overlays.default];
+    # (2) Add overlay.
+    nixpkgs.overlays = [nodeFlake.inputs.nix-snapshotter.overlays.default];
 
-  # (3) Enable service.
+    # (3) Enable service.
-  virtualisation.containerd = {
+    virtualisation.containerd = {
-    enable = true;
+      enable = true;
-    k3sIntegration = false;
+      nixSnapshotterIntegration = true;
-    nixSnapshotterIntegration = true;
 
-    # TODO: understand if this has an influence on the systemd LoadCredential issue
+      # TODO: understand if this has an influence on the systemd LoadCredential issue
-    settings.plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options.SystemdCgroup = lib.mkForce true;
+      # settings.plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options.SystemdCgroup = lib.mkForce true;
-  };
+    };
-  services.nix-snapshotter = {
+    services.nix-snapshotter = {
-    enable = true;
+      enable = true;
-  };
+    };
 
-  # (4) Add a containerd CLI like nerdctl.
+    # (4) Add a containerd CLI like nerdctl.
-  environment.systemPackages = [
+    environment.systemPackages = [
-    pkgs.nerdctl
+      pkgs.nerdctl
-    nodeFlake.inputs.nix-snapshotter.packages.${system}.default
+      nodeFlake.inputs.nix-snapshotter.packages.${system}.default
-  ];
+    ];
 
-  services.k3s = {
+    services.k3s = {
-    enable = false;
+      enable = false;
-    setKubeConfig = true;
+      setKubeConfig = true;
+    };
+
+    # home-manager.users."${homeUser}" = _: {
+    #   home.sessionVariables.CONTAINERD_ADDRESS = "/run/user/1000/containerd/containerd.sock";
+    # };
   };
 }

nix/os/snippets/systemd-resolved.nix

@@ -1,4 +1,4 @@
-{
+{lib, ...}: {
   networking.nameservers = [
     # https://dnsforge.de/
     "176.9.93.198"
@@ -16,5 +16,7 @@
 
     # TODO: figure out why "true" doesn't work
     dnsovertls = "opportunistic";
+
+    fallbackDns = lib.mkForce [];
   };
 }