WIP k3s

feat(router0-dmz0, sj-srv1/containers/webserver): set up kanidm
fix(sj-srv1,containers, systemd-resolved): resolve via dhcp, host, and never use fallbacks
2024-10-16 23:03:38 +02:00 · 2024-10-16 23:03:38 +02:00 · 2024-10-16 23:03:38 +02:00
13 changed files with 308 additions and 80 deletions
--- a/flake.lock
+++ b/flake.lock
@ -585,6 +585,22 @@
        "type": "github"
      }
    },
+    "nixpkgs-kanidm": {
+      "locked": {
+        "lastModified": 1729071019,
+        "narHash": "sha256-c4J/ZiMbjMf98FawO5XJaTWqvrvIXpxnIpxu4OV3CGA=",
+        "owner": "steveej-forks",
+        "repo": "nixpkgs",
+        "rev": "984b1d5a286d3a072b840b30ec49d96878d01e64",
+        "type": "github"
+      },
+      "original": {
+        "owner": "steveej-forks",
+        "ref": "kanidm",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
    "nixpkgs-lib": {
      "locked": {
        "dir": "lib",
@ -794,6 +810,7 @@
        "nixpkgs-2305": "nixpkgs-2305",
        "nixpkgs-2311": "nixpkgs-2311",
        "nixpkgs-2405": "nixpkgs-2405",
+        "nixpkgs-kanidm": "nixpkgs-kanidm",
        "nixpkgs-unstable": "nixpkgs-unstable",
        "nixpkgs-vscodium": "nixpkgs-vscodium",
        "nixpkgs-wayland": "nixpkgs-wayland",
--- a/flake.nix
+++ b/flake.nix
@ -125,6 +125,8 @@
      url = "git+https://git.codelinaro.org/clo/ath-firmware/ath11k-firmware.git";
      flake = false;
    };
+
+    nixpkgs-kanidm.url = "github:steveej-forks/nixpkgs/kanidm";
  };

  outputs = inputs @ {
@ -360,6 +362,7 @@
        devShells = let
          all = import ./nix/devShells.nix {
            inherit
+              self
              self'
              inputs'
              pkgs
--- a/nix/devShells.nix
+++ b/nix/devShells.nix
@ -1,4 +1,5 @@
 {
+  self,
  self',
  inputs',
  pkgs,
@ -82,9 +83,13 @@ in {
      wireguard-tools

      screen
+
+      inputs'.nixpkgs-kanidm.legacyPackages.kanidm
    ];

    # Set Environment Variables
    RUST_BACKTRACE = 1;
+
+    KANIDM_URL = self.nixosConfigurations.sj-srv1.config.containers.webserver.config.services.kanidm.serverSettings.origin;
  };
 }
--- a/nix/os/containers/mycelium/flake.lock
+++ b/nix/os/containers/mycelium/flake.lock
@ -46,11 +46,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1715438114,
-        "narHash": "sha256-btb702TXuhDg0D6tW0dCOy4+II9Wl6BJ0LvpT+O9wrs=",
+        "lastModified": 1723875769,
+        "narHash": "sha256-66GofByLJ+S4ZZphIC+vJKeL9VJ2bzH2VbcJ3OqteMM=",
        "owner": "pdtpartners",
        "repo": "nix-snapshotter",
-        "rev": "7b251c9356bc7bb383ebeedcd0045b3ae431bff7",
+        "rev": "6eaadfd8f89e5e7d79b2013626bbd36e388159da",
        "type": "github"
      },
      "original": {
@ -61,11 +61,11 @@
    },
    "nixlib": {
      "locked": {
-        "lastModified": 1712450863,
-        "narHash": "sha256-K6IkdtMtq9xktmYPj0uaYc8NsIqHuaAoRBaMgu9Fvrw=",
+        "lastModified": 1728781282,
+        "narHash": "sha256-hUP9oxmnOmNnKcDOf5Y55HQ+NnoT0+bLWHLQWLLw9Ks=",
        "owner": "nix-community",
        "repo": "nixpkgs.lib",
-        "rev": "3c62b6a12571c9a7f65ab037173ee153d539905f",
+        "rev": "16340f605f4e8e5cf07fd74dcbe692eee2d4f51b",
        "type": "github"
      },
      "original": {
@ -82,11 +82,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1718025593,
-        "narHash": "sha256-WZ1gdKq/9u1Ns/oXuNsDm+W0salonVA0VY1amw8urJ4=",
+        "lastModified": 1728867876,
+        "narHash": "sha256-NCyOA8WZNoojmXH+kBDrQj3LwvakYNzSc0h+LTXkmPE=",
        "owner": "nix-community",
        "repo": "nixos-generators",
-        "rev": "35c20ba421dfa5059e20e0ef2343c875372bdcf3",
+        "rev": "fdf142111597f6c6283cf5ffe092b6293a3911d0",
        "type": "github"
      },
      "original": {
@ -97,42 +97,25 @@
    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1718086528,
-        "narHash": "sha256-hoB7B7oPgypePz16cKWawPfhVvMSXj4G/qLsfFuhFjw=",
+        "lastModified": 1728897630,
+        "narHash": "sha256-0utJPs4o2Mody8GDwo4hnGuxc8dJqju4u9lLJY4d/Lw=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "47b604b07d1e8146d5398b42d3306fdebd343986",
+        "rev": "c9f0b4a395289ce18727e2a8e43cae6796693ccc",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "nixos-24.05",
+        "ref": "nixos-unstable-small",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
-    "nixpkgs-systemd256": {
-      "locked": {
-        "lastModified": 1718397913,
-        "narHash": "sha256-omV+dq3GdXQQTaewxhkBgxM4Bbwqa4D9FVS4dTITxOQ=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "962cf03fb8c782c5e00f465397e03dc84284acc9",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "962cf03fb8c782c5e00f465397e03dc84284acc9",
-        "type": "github"
-      }
-    },
    "root": {
      "inputs": {
        "nix-snapshotter": "nix-snapshotter",
        "nixos-generators": "nixos-generators",
-        "nixpkgs": "nixpkgs",
-        "nixpkgs-systemd256": "nixpkgs-systemd256"
+        "nixpkgs": "nixpkgs"
      }
    }
  },
--- a/nix/os/containers/mycelium/flake.nix
+++ b/nix/os/containers/mycelium/flake.nix
@ -1,7 +1,7 @@
 {
  inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05";
-    nixpkgs-systemd256.url = "github:NixOS/nixpkgs/962cf03fb8c782c5e00f465397e03dc84284acc9";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
+    # nixpkgs-systemd256.url = "github:NixOS/nixpkgs/962cf03fb8c782c5e00f465397e03dc84284acc9";
    nixos-generators = {
      url = "github:nix-community/nixos-generators";
      inputs.nixpkgs.follows = "nixpkgs";
@ -102,7 +102,7 @@
            imports = [
              (modulesPath + "/profiles/minimal.nix")
            ];
-            system.stateVersion = "24.05";
+            system.stateVersion = "24.11";

            # https://github.com/hercules-ci/arion/blob/c24c185e67f093298a081900b49ca18716077dec/src/nix/modules/nixos/container-systemd.nix
            boot.isContainer = true;
@ -116,22 +116,37 @@
            services.nscd.enable = false;
            system.nssModules = lib.mkForce [];
            systemd.services.systemd-logind.enable = false;
-            systemd.services.console-getty.enable = true;
+            systemd.services.console-getty.enable = false;

            systemd.sockets.nix-daemon.enable = false;
            systemd.services.nix-daemon.enable = false;
            systemd.oomd.enable = false;
            networking.useDHCP = false;
            networking.firewall.enable = false;
+
+            # system.build.earlyMountScript =
+            #   lib.mkForce ''
+            #   '';
+            # system.activationScripts.specialfs =
+            #   lib.mkForce ''
+            #   '';
            boot.postBootCommands = ''
              ls -lha /run
              mkdir -p /run/wrappers
            '';
+
+            boot.kernelParams = [
+              "systemd.log_level=debug"
+            ];
+
            # services.udev.enable = false;

            # TODO: this is only needed because `/run/current-system` is missing
            # environment.variables.PATH = "${lib.makeBinPath config.environment.systemPackages}:$PATH";

+            systemd.mounts = lib.mkForce [];
+            fileSystems = lib.mkForce {};
+
            services.mycelium.enable = false;
            services.mycelium.keyFile = "/var/lib/secrets/mycelium-keyfile";
            systemd.services.mycelium.serviceConfig.DynamicUser = lib.mkForce false;
@ -151,17 +166,18 @@
              serviceConfig = {
                SyslogIdentifier = "testing-credential";
                StateDirectory = "testing-credentials";
-                # DynamicUser = true;
+                DynamicUser = true;
                # User = "tc";
                # ProtectHome = true;
-                # ProtectSystem = true;
-                LoadCredential = [
-                  "mycelium-keyfile:${self.nixosConfigurations.default.config.services.mycelium.keyFile}"
-                  "hosts:/etc/hosts"
-                ];
-                SetCredential = "nosecret:not secret string";
+                ProtectSystem = true;
+                # LoadCredential = [
+                #   "mycelium-keyfile:${self.nixosConfigurations.default.config.services.mycelium.keyFile}"
+                #   "hosts:/etc/hosts"
+                # ];
+                SetCredential = "mycelium-keyfile:not secret string";
                ExecStart = lib.mkForce (pkgs.writeShellScript "mycelium" ''
                  cd $STATE_DIRECTORY
+                  pwd
                  env
                  while true; do
                    ls -lha $CREDENTIALS_DIRECTORY
@ -228,6 +244,9 @@
          ];
          volumes = {
            # "/var/lib/private/mycelium/key.bin" = {};
+            # "/run" = {};
+            # "/tmp" = {};
+            # "/etc" = {};
          };
          copyToRoot = [
            # self.nixosConfigurations.default.config.system.build.toplevel
@ -312,6 +331,7 @@
          nix build --impure .#image
          sudo nix2container load ./result
          sudo -E nerdctl run  --name ${name} --privileged -dt \
+            --cgroup-manager cgroupfs \
            --volume "$PWD/key.bin.crypt:${self.nixosConfigurations.default.config.services.mycelium.keyFile}:ro" \
            "nix:0$(readlink result):latest"
        '';
--- a/nix/os/containers/webserver.nix
+++ b/nix/os/containers/webserver.nix
@ -17,16 +17,19 @@ in {
    lib,
    repoFlake,
    nodeFlake,
+    system,
    ...
  }: {
    system.stateVersion = "22.05"; # Did you read the comment?

    disabledModules = [
      "services/misc/forgejo.nix"
+      "services/security/kanidm.nix"
    ];

    imports = [
      "${nodeFlake.inputs.nixpkgs-unstable}/nixos/modules/services/misc/forgejo.nix"
+      "${repoFlake.inputs.nixpkgs-kanidm}/nixos/modules/services/security/kanidm.nix"

      ../profiles/containers/configuration.nix

@ -90,6 +93,16 @@ in {
          reverse_proxy http://127.0.0.1:${builtins.toString config.services.forgejo.settings.server.HTTP_PORT}
        '';
      };
+
+      virtualHosts."kanidm.${domain}" = {
+        extraConfig = ''
+          reverse_proxy https://${builtins.toString config.services.kanidm.serverSettings.bindaddress} {
+            transport http {
+                tls_server_name ${config.services.kanidm.serverSettings.domain}
+            }
+          }
+        '';
+      };
    };

    services.hedgedoc = {
@ -116,12 +129,34 @@ in {
          url = "ldap://127.0.0.1:${builtins.toString config.services.lldap.settings.ldap_port}";
          bindDn = "uid=admin,ou=people,dc=stefanjunker,dc=de";
          # these are set via the `environmentFile`
-          bindCredentials = "$LDAP_ADMIN_PASSWORD";
+          # bindCredentials = "$LDAP_ADMIN_PASSWORD";
          searchBase = "ou=people,dc=stefanjunker,dc=de";
          searchFilter = "(&(memberOf=cn=hedgedoc,ou=groups,dc=stefanjunker,dc=de)(uid={{username}}))";
          useridField = "uid";
        };

+        oauth2 = let
+          originURL = config.services.kanidm.serverSettings.origin;
+        in {
+          providerName = "kanidm (${originURL})";
+
+          authorizationURL = "${originURL}/ui/oauth2";
+          tokenURL = "${originURL}/oauth2/token";
+          userProfileURL = "${originURL}/oauth2/openid/hedgedoc/userinfo";
+
+          scope = "openid email profile";
+          # rolesClaim = "roles";
+          # accessRole = "role/hedgedoc";
+
+          userProfileUsernameAttr = "name";
+          userProfileDisplayNameAttr = "displayname";
+          userProfileEmailAttr = "email";
+
+          clientID = "hedgedoc";
+          # set via the `environmentFile`
+          # clientSecret = "$CMD_OAUTH2_CLIENT_SECRET";
+        };
+
        uploadsPath = "/var/lib/hedgedoc/uploads";
      };

@ -268,6 +303,108 @@ in {
    systemd.services.lldap.serviceConfig.User = config.users.users.lldap.name;
    systemd.services.lldap.serviceConfig.Group = config.users.groups.lldap.name;
    systemd.services.lldap.serviceConfig.DynamicUser = lib.mkForce false;
+
+    # combine a path watcher with a service that transfers the certs by caddy to kanidm
+    systemd.paths.kanidm-tls-watch = {
+      enable = true;
+      requiredBy = ["kanidm.service"];
+      pathConfig = {
+        PathChanged = [
+          "${config.services.caddy.dataDir}/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/${config.services.kanidm.serverSettings.domain}/${config.services.kanidm.serverSettings.domain}.key"
+          "${config.services.caddy.dataDir}/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/${config.services.kanidm.serverSettings.domain}/${config.services.kanidm.serverSettings.domain}.crt"
+        ];
+        Unit = "kanidm-tls-update.service";
+      };
+    };
+    systemd.services.kanidm-tls-update = let
+      dbDir =
+        builtins.dirOf
+        config.services.kanidm.serverSettings.db_path;
+    in {
+      enable = true;
+      requiredBy = ["kanidm.service"];
+      unitConfig = {
+        # ConditionPathExists = [
+        #   "${config.services.caddy.dataDir}/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/${config.services.kanidm.serverSettings.domain}/${config.services.kanidm.serverSettings.domain}.key"
+        #   "${config.services.caddy.dataDir}/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/${config.services.kanidm.serverSettings.domain}/${config.services.kanidm.serverSettings.domain}.crt"
+        # ];
+      };
+      serviceConfig.Type = "oneshot";
+      script = let
+        tlsDir = builtins.dirOf config.services.kanidm.serverSettings.tls_key;
+      in ''
+        set -xe
+
+        cat "${config.services.caddy.dataDir}/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/${config.services.kanidm.serverSettings.domain}/${config.services.kanidm.serverSettings.domain}.key" > tls.key
+        cat "${config.services.caddy.dataDir}/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/${config.services.kanidm.serverSettings.domain}/${config.services.kanidm.serverSettings.domain}.crt" > tls.chain
+
+        chown ${config.systemd.services.kanidm.serviceConfig.User}:${config.systemd.services.kanidm.serviceConfig.Group} tls.{key,chain}
+        chmod 400 tls.{key,chain}
+
+        # create the kanidm directory in case it's missing
+        if [[ ! -d ${tlsDir} ]]; then
+          mkdir -p ${tlsDir}
+          chown -R ${config.systemd.services.kanidm.serviceConfig.User}:${config.systemd.services.kanidm.serviceConfig.Group} ${tlsDir}
+          chmod 700 ${tlsDir}
+        fi
+
+        mv tls.key ${config.services.kanidm.serverSettings.tls_key}
+        mv tls.chain ${config.services.kanidm.serverSettings.tls_chain}
+
+        if [[ ! -d ${dbDir} ]]; then
+          mkdir -p ${dbDir}
+          chown -R ${config.systemd.services.kanidm.serviceConfig.User}:${config.systemd.services.kanidm.serviceConfig.Group} ${dbDir}
+          chmod 700 ${dbDir}
+        fi
+      '';
+    };
+
+    systemd.services.kanidm.serviceConfig = let
+      dbDir =
+        builtins.dirOf
+        config.services.kanidm.serverSettings.db_path;
+      # stateDir = "/var/lib/${config.systemd.services.kanidm.serviceConfig.StateDirectory}";
+    in {
+      # ExecStartPre = ''
+      #   mkdir -p ${dbDir}
+      # '';
+      BindPaths = [
+        dbDir
+        # stateDir
+      ];
+    };
+
+    services.kanidm = let
+      dataDir = "/var/lib/kanidm";
+    in {
+      package = repoFlake.inputs.nixpkgs-kanidm.legacyPackages.${pkgs.system}.kanidm;
+
+      enablePam = false;
+      enableClient = false;
+
+      enableServer = true;
+      serverSettings = {
+        role = "WriteReplica";
+        log_level = "debug";
+
+        domain = "kanidm.${domain}";
+        origin = "https://kanidm.${domain}";
+
+        db_path = "${dataDir}/db/kanidm.db";
+
+        bindaddress = "127.0.0.1:8444";
+
+        # don't expose ldap
+        # ldapbindaddress = "[::1]:6636";
+
+        tls_key = "${dataDir}/tls/tls.key";
+        tls_chain = "${dataDir}/tls/tls.chain";
+
+        online_backup = {
+          schedule = "00 06 * * *";
+        };
+      };
+    };
  };

  inherit autoStart;
@ -306,6 +443,11 @@ in {
      hostPath = "/var/lib/container-volumes/webserver/var-lib-forgejo";
      isReadOnly = false;
    };
+
+    "/var/lib/kanidm" = {
+      hostPath = "/var/lib/container-volumes/webserver/var-lib-kanidm";
+      isReadOnly = false;
+    };
  };

  privateNetwork = true;
--- a/nix/os/containers/webserver_secrets.yaml
+++ b/nix/os/containers/webserver_secrets.yaml
@ -1,4 +1,4 @@
-hedgedoc_environment_file: ENC[AES256_GCM,data:ciVnpDXq5CZltHcAHJQNeKfelQlKhyXfGkUeuvwFBq8QUQDNEgLOVZ5X7Yw3kPGAvXEozK2Nz3aFfOpbGt76OmNdJ2TQNxOEpcHDJEvAoYSc/XTcctfDQmqga6MMWWAjIO3LXpFa9UD9riP6yUFNwGOB7waIvV7yD+D+QILwUyNda0/iVHtC/6HO8Yaj3nK6Fp1IDclppobIQ/MdzG+cy+yN7h0XUNOzMh91DGAC3ePIB5DX90wlXTzsox9HWWAUTh6Lpss=,iv:X7fROtc0Fn9AnZkWHAs8XFwIInBowQZzRJuLWSKSGWM=,tag:gKysRtqBhTtwLnxDv2QGBA==,type:str]
+hedgedoc_environment_file: ENC[AES256_GCM,data:gPTokPMGBAN/lGGeUs95vg45yVrrSmFCKWTjlMV4V+YnflcqiaZvifX9+0fe3DELwNL4kY4st4N0MadhLkTiSieyp46fP8Dujk4Prhi7JWweBDsN4WtxcwJfAdowgh5LTzqM3zggC/J9NGR/zgJGLYraOqsFueXycxDxntE+8MlepYFGsND4WbFHNRvsVd7xUWerZZD+JFhws2sjwC9DqoJ+mBX4u9J2faSrL3okBGwRpEZlJhe6/8pT0l1aVxI0b/9UsLUL/him/vVqY8ygMP8O95gzuDEaCtwSXw08ylhb3g3YHdMh9ZOe9dPNVocVFrB15HfxeY4KzRCVfvgmBsSiUrgUAZQ8aav2ZWHPKQ==,iv:AVtx/43MK5KVxP59olEmbkUzLhd0cBjPpVeiAJGELfM=,tag:Hd3edeUzLgHnwAwPiMGp4A==,type:str]
 authelia_storageEncryptionKey: ENC[AES256_GCM,data:BLj2+w0jUOiiqljd4WcYt90I1Y/tm6vYRn9IPvv7k+ykjeWeafDcb9LpAbYZ82iAbCDZ6cDnTIJSH50TYFM1Bw==,iv:DfrFuTMS6rzWS1F7hIloD7Eb9N6LV6MDsPGEidTCIkM=,tag:4TxiUuWj8LLCAe+3SQWthA==,type:str]
 authelia_jwtSecret: ENC[AES256_GCM,data:CtvhPJDlM8kZDh3MWI6jBEIK5P/Rc0ZY3JfQ/1qn2FyZ0kOE9CMznRQdzi125Mlh+LgfhS00aeY/esuSJ1YKnw==,iv:o29+Ja4E4USpxD5Bgg4nGpj1WdK+uJJeuyqa3c92Pys=,tag:Ko/6PqIM7FyvqMJAepTNKw==,type:str]
 lldap_jwtSecret: ENC[AES256_GCM,data:Jd05K5zZZcdFY9P+PM1ycpl4eKr7THXzlEC2gxhfq0kZ0c7NrmqpnxLFyMOoq0oZwFXQ3aE29N+dYYCbWgEs0g==,iv:k9Y4XZcFSUaaOvooMzH1AlHaI/RiZZ0cpSqTU6jzsUU=,tag:EHwIMQ00Z+IrXAKgugMbmQ==,type:str]
@ -23,8 +23,8 @@ sops:
            eWdXVHg5MTlQQU9GeElPeUZXUlBlaTgKceDu3tLbQM/DxY0tJYJTPy2Dl/SBYaoc
            KfMZOkc322/NvgWu/3Ke0hV1/eMk8EICwXbSwHhXr5a0+cwPZ9xV4A==
            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-10-13T17:41:14Z"
-    mac: ENC[AES256_GCM,data:1mqRRPa4tP1OFxC3Oo5uJhk3H79jxObUeIsIab8fOrafsrw9tbrqpb9lRgziR3C0ssDagb0deA6PAGH6YWvSU716Ayr3p+Ih2sXOkbkp8wV/u3AULsDUzSUglshgM5f1Hf5jvL7xoWBOzek8eMGIkFFFwu0VmkqwpqOalXY0Kxk=,iv:cC4hRQZlLuOyktS0pER6Ef0f7qVxMXfS8w9Q5p7AlTA=,tag:/maJgYz/Ks3iaQZr+WSUUA==,type:str]
+    lastmodified: "2024-10-16T12:28:51Z"
+    mac: ENC[AES256_GCM,data:nrd2czzJlBcFfwn6lzh4qqco+/XsU2J6BqvQqMtskh3mL4Xx25IAzxiCno0KlNGr6o4YsuZP5anOX9RvrDq76Us3JQ7pDi3iQGPhmg+SE9u3Rwqn1/3YConvdfPV2DNB+tuyG3UVoRqpA4d+HdcYjN9n1UKk54R6UdSm9UrA+zc=,iv:Juupyet09zUAMu7bmVxq+/Q0bXJAzR0wAyt6vKNns3w=,tag:owdUWuXrQcDdiWi+1geY9A==,type:str]
    pgp:
        - created_at: "2023-07-09T17:51:27Z"
          enc: |-
--- a/nix/os/devices/router0-dmz0/configuration.nix
+++ b/nix/os/devices/router0-dmz0/configuration.nix
@ -1260,6 +1260,7 @@ in {
        "jitsi.www.stefanjunker.de,${dmzExposedHost}"
        "lldap.www.stefanjunker.de,${dmzExposedHost}"
        "forgejo.www.stefanjunker.de,${dmzExposedHost}"
+        "kanidm.www.stefanjunker.de,${dmzExposedHost}"
      ];
    };
  };
--- a/nix/os/devices/sj-srv1/flake.lock
+++ b/nix/os/devices/sj-srv1/flake.lock
@ -23,11 +23,11 @@
    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1728328465,
-        "narHash": "sha256-a0a0M1TmXMK34y3M0cugsmpJ4FJPT/xsblhpiiX1CXo=",
+        "lastModified": 1728909085,
+        "narHash": "sha256-WLxED18lodtQiayIPDE5zwAfkPJSjHJ35UhZ8h3cJUg=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "1bfbbbe5bbf888d675397c66bfdb275d0b99361c",
+        "rev": "c0b1da36f7c34a7146501f684e9ebdf15d2bebf8",
        "type": "github"
      },
      "original": {
@ -39,11 +39,11 @@
    },
    "nixpkgs-master": {
      "locked": {
-        "lastModified": 1728543552,
-        "narHash": "sha256-3OR+2XHHo+USlAz7T30VKnPxR7k3GeErkXM0Wm/Ctzw=",
+        "lastModified": 1729086167,
+        "narHash": "sha256-Vh6kOiQHefsr6Zin4Xi+VH06leuNZuMyP8YkkGo/Naw=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "f4f573fde42d181f22c95e10822856399c24feeb",
+        "rev": "6b1ffdb0976ac367aeea173b8e69de62828a4ca7",
        "type": "github"
      },
      "original": {
@ -55,11 +55,11 @@
    },
    "nixpkgs-unstable": {
      "locked": {
-        "lastModified": 1728534991,
-        "narHash": "sha256-wLUZyvtOOowAz0kTrU2MoC4nXWniFaVezGyzuEt5HPc=",
+        "lastModified": 1729077633,
+        "narHash": "sha256-6sIuRVqVMHq9ZwcEVdpf2BuZeuLIUgvFznhIfsc75Jo=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "6b955bdbb9efe4a5c047746323951fe1bdf8d01b",
+        "rev": "8f1d45587bd9af3dbf5146aa8a1347e20421597b",
        "type": "github"
      },
      "original": {
--- a/nix/os/devices/sj-srv1/system.nix
+++ b/nix/os/devices/sj-srv1/system.nix
@ -11,6 +11,23 @@
 in {
  imports = [
    ../../snippets/systemd-resolved.nix
+    {
+      # make sure it uses the DNS that comes in via DHCP
+      networking.nameservers = lib.mkForce [];
+      services.resolved.enable = true;
+
+      # provide DNS to the containers
+      services.resolved.extraConfig = ''
+        DNSStubListenerExtra=${hostBridgeAddress}
+      '';
+      networking.firewall.interfaces.br0.allowedTCPPorts = [53];
+      networking.firewall.interfaces.br0.allowedUDPPorts = [53];
+    }
+  ];
+
+  programs.wireshark.enable = true;
+  environment.systemPackages = [
+    pkgs.dnsutils
  ];

  networking.firewall.enable = true;
@ -83,6 +100,9 @@ in {
    enable = true;
    matchConfig.Name = "dmz0";
    DHCP = "yes";
+
+    dhcpV4Config.UseDNS = true;
+    dhcpV6Config.UseDNS = true;
  };

  boot.kernel.sysctl = {
@ -134,6 +154,7 @@ in {
    mailserver = import ../../containers/mailserver.nix {
      specialArgs = {
        inherit repoFlake nodeFlake;
+        hostAddress = hostBridgeAddress;
      };

      autoStart = true;
@ -151,6 +172,7 @@ in {
      {
        specialArgs = {
          inherit repoFlake nodeFlake;
+          hostAddress = hostBridgeAddress;
        };

        autoStart = true;
@ -167,6 +189,7 @@ in {
    syncthing = import ../../containers/syncthing.nix {
      specialArgs = {
        inherit repoFlake nodeFlake;
+        hostAddress = hostBridgeAddress;
      };
      autoStart = true;

--- a/nix/os/profiles/containers/configuration.nix
+++ b/nix/os/profiles/containers/configuration.nix
@ -1,16 +1,29 @@
-{pkgs, ...}: {
+{
+  hostAddress,
+  pkgs,
+  lib,
+  ...
+}: {
  networking.useHostResolvConf = false;

  networking.firewall.enable = true;
  networking.nftables.enable = true;
  networking.nftables.flushRuleset = true;

+  networking.nameservers = lib.mkForce [hostAddress];
+
  environment.systemPackages = [
    pkgs.dnsutils
  ];

  imports = [
-    ../../snippets/systemd-resolved.nix
+    {
+      # keep DNS set up to a minimum: only query the container host
+      services.resolved.enable = lib.mkForce false;
+      networking.nameservers = [
+        hostAddress
+      ];
+    }
    ../../snippets/nix-settings.nix
    # ../../modules/ddclient-ovh.nix
    # ../../modules/ddclient-hetzner.nix
--- a/nix/os/snippets/k3s-w-nix-snapshotter.nix
+++ b/nix/os/snippets/k3s-w-nix-snapshotter.nix
@ -5,24 +5,38 @@
  pkgs,
  lib,
  system,
+  config,
  ...
-}: {
+}: let
+  cfg = config.steveej.k3s;
+
+  # TODO: make this configurable
+  homeUser = "steveej";
+in {
+  options.steveej.k3s = {
+    enable = lib.mkOption {
+      description = "steveej's k3s distro";
+      type = lib.types.bool;
+      default = true;
+    };
+  };
+
  # (1) Import nixos module.
  imports = [
    nodeFlake.inputs.nix-snapshotter.nixosModules.default
  ];

+  config = lib.mkIf cfg.enable {
    # (2) Add overlay.
    nixpkgs.overlays = [nodeFlake.inputs.nix-snapshotter.overlays.default];

    # (3) Enable service.
    virtualisation.containerd = {
      enable = true;
-    k3sIntegration = false;
      nixSnapshotterIntegration = true;

      # TODO: understand if this has an influence on the systemd LoadCredential issue
-    settings.plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options.SystemdCgroup = lib.mkForce true;
+      # settings.plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options.SystemdCgroup = lib.mkForce true;
    };
    services.nix-snapshotter = {
      enable = true;
@ -38,4 +52,9 @@
      enable = false;
      setKubeConfig = true;
    };
+
+    # home-manager.users."${homeUser}" = _: {
+    #   home.sessionVariables.CONTAINERD_ADDRESS = "/run/user/1000/containerd/containerd.sock";
+    # };
+  };
 }
--- a/nix/os/snippets/systemd-resolved.nix
+++ b/nix/os/snippets/systemd-resolved.nix
@ -1,4 +1,4 @@
-{
+{lib, ...}: {
  networking.nameservers = [
    # https://dnsforge.de/
    "176.9.93.198"
@ -16,5 +16,7 @@

    # TODO: figure out why "true" doesn't work
    dnsovertls = "opportunistic";
+
+    fallbackDns = lib.mkForce [];
  };
 }
Author	SHA1	Message	Date
Stefan Junker	bd8630681b	WIP k3s	2024-10-16 23:03:38 +02:00
Stefan Junker	4c71887ea6	feat(router0-dmz0, sj-srv1/containers/webserver): set up kanidm	2024-10-16 23:03:38 +02:00
Stefan Junker	7f97ee3d47	fix(sj-srv1,containers, systemd-resolved): resolve via dhcp, host, and never use fallbacks	2024-10-16 23:03:38 +02:00