From f986a8300db1068da4155a0d321caf6ab5891f78 Mon Sep 17 00:00:00 2001 From: Stefan Junker Date: Thu, 13 Jun 2024 16:42:55 +0200 Subject: [PATCH 1/6] experiment with k3s + nix-snapshotter --- nix/os/containers/mycelium/configuration.nix | 0 nix/os/containers/mycelium/flake.lock | 141 ++++++++ nix/os/containers/mycelium/flake.nix | 340 ++++++++++++++++++ nix/os/devices/steveej-x13s/configuration.nix | 48 ++- nix/os/devices/steveej-x13s/flake.lock | 62 +++- nix/os/devices/steveej-x13s/flake.nix | 6 + 6 files changed, 593 insertions(+), 4 deletions(-) create mode 100644 nix/os/containers/mycelium/configuration.nix create mode 100644 nix/os/containers/mycelium/flake.lock create mode 100644 nix/os/containers/mycelium/flake.nix diff --git a/nix/os/containers/mycelium/configuration.nix b/nix/os/containers/mycelium/configuration.nix new file mode 100644 index 0000000..e69de29 diff --git a/nix/os/containers/mycelium/flake.lock b/nix/os/containers/mycelium/flake.lock new file mode 100644 index 0000000..899ee98 --- /dev/null +++ b/nix/os/containers/mycelium/flake.lock @@ -0,0 +1,141 @@ +{ + "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nix-snapshotter", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1704152458, + "narHash": "sha256-DS+dGw7SKygIWf9w4eNBUZsK+4Ug27NwEWmn2tnbycg=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "88a2cd8166694ba0b6cb374700799cec53aef527", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "nix-snapshotter": { + "inputs": { + "flake-compat": "flake-compat", + "flake-parts": "flake-parts", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1715438114, + "narHash": "sha256-btb702TXuhDg0D6tW0dCOy4+II9Wl6BJ0LvpT+O9wrs=", + "owner": "pdtpartners", + "repo": "nix-snapshotter", + "rev": "7b251c9356bc7bb383ebeedcd0045b3ae431bff7", + "type": "github" + }, + "original": { + "owner": "pdtpartners", + "repo": "nix-snapshotter", + "type": "github" + } + }, + "nixlib": { + "locked": { + "lastModified": 1712450863, + "narHash": "sha256-K6IkdtMtq9xktmYPj0uaYc8NsIqHuaAoRBaMgu9Fvrw=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "3c62b6a12571c9a7f65ab037173ee153d539905f", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "nixos-generators": { + "inputs": { + "nixlib": "nixlib", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1718025593, + "narHash": "sha256-WZ1gdKq/9u1Ns/oXuNsDm+W0salonVA0VY1amw8urJ4=", + "owner": "nix-community", + "repo": "nixos-generators", + "rev": "35c20ba421dfa5059e20e0ef2343c875372bdcf3", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixos-generators", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1718086528, + "narHash": "sha256-hoB7B7oPgypePz16cKWawPfhVvMSXj4G/qLsfFuhFjw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "47b604b07d1e8146d5398b42d3306fdebd343986", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-systemd256": { + "locked": { + "lastModified": 1718397913, + "narHash": "sha256-omV+dq3GdXQQTaewxhkBgxM4Bbwqa4D9FVS4dTITxOQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "962cf03fb8c782c5e00f465397e03dc84284acc9", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "962cf03fb8c782c5e00f465397e03dc84284acc9", + "type": "github" + } + }, + "root": { + "inputs": { + "nix-snapshotter": "nix-snapshotter", + "nixos-generators": "nixos-generators", + "nixpkgs": "nixpkgs", + "nixpkgs-systemd256": "nixpkgs-systemd256" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/nix/os/containers/mycelium/flake.nix b/nix/os/containers/mycelium/flake.nix new file mode 100644 index 0000000..6f247a1 --- /dev/null +++ b/nix/os/containers/mycelium/flake.nix @@ -0,0 +1,340 @@ +{ + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; + nixpkgs-systemd256.url = "github:NixOS/nixpkgs/962cf03fb8c782c5e00f465397e03dc84284acc9"; + nixos-generators = { + url = "github:nix-community/nixos-generators"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + nix-snapshotter = { + url = "github:pdtpartners/nix-snapshotter"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + outputs = { + self, + nixpkgs, + nixos-generators, + ... + }: let + systems = [ + "aarch64-linux" + "x86_64-linux" + ]; + forAllSystems = nixpkgs.lib.genAttrs systems; + in { + nixosConfigurations.default = + nixpkgs.lib.nixosSystem + { + system = "aarch64-linux"; + + specialArgs = {}; + + modules = [ + ({ + config, + modulesPath, + pkgs, + lib, + ... + }: { + nixpkgs.overlays = [ + (final: previous: { + # inherit (self.inputs.nixpkgs-systemd256.legacyPackages.${pkgs.system}) systemd systemdMinimal; + # systemd = + # self.inputs.nixpkgs-systemd256.legacyPackages.${pkgs.system}.systemd.overrideAttrs (prevAttrs: { + # src = /home/steveej/src/others/systemd; + + # withAppArmor = false; + # withRepart = false; + # withHomed = false; + # withAcl = false; + # withEfi = false; + # withBootloader = false; + # withCryptsetup = false; + # withLibBPF = false; + # withOomd = false; + # withFido2 = false; + # withApparmor = false; + # withDocumentation = false; + # withUtmp = false; + # withQrencode = false; + # withVmspawn = false; + # withMachined = false; + # withLogTrace = true; + # withArchive = false; + # # don't need these but cause errors for exampel files not found + # # withLogind = false; + # }) + # pkgs.systemdMinimal.override { + # # getting errors with these disabled + # withCoredump = true; + # withCompression = true; + # withLogind = true; + # withSysusers = true; + # withUserDb = true; + # } + # pkgs.systemdMinimal + # pkgs.systemd.override { + # withRepart = false; + # withHomed = false; + # withAcl = false; + # withEfi = false; + # withBootloader = false; + # withCryptsetup = false; + # withLibBPF = false; + # withOomd = false; + # withFido2 = false; + # withApparmor = false; + # withDocumentation = false; + # withUtmp = false; + # withQrencode = false; + # withVmspawn = false; + # withMachined = false; + # withLogTrace = true; + # # don't need these but cause errors for exampel files not found + # # withLogind = false; + # } + # ; + }) + ]; + + imports = [ + (modulesPath + "/profiles/minimal.nix") + ]; + system.stateVersion = "24.05"; + + # https://github.com/hercules-ci/arion/blob/c24c185e67f093298a081900b49ca18716077dec/src/nix/modules/nixos/container-systemd.nix + boot.isContainer = true; + # boot.tmp.useTmpfs = true; + boot.loader.grub.enable = lib.mkForce false; + boot.loader.systemd-boot.enable = lib.mkForce false; + services.journald.console = "/dev/console"; + services.journald.storage = "none"; + # boot.specialFileSystems = lib.mkForce {}; + + services.nscd.enable = false; + system.nssModules = lib.mkForce []; + systemd.services.systemd-logind.enable = false; + systemd.services.console-getty.enable = true; + + systemd.sockets.nix-daemon.enable = false; + systemd.services.nix-daemon.enable = false; + systemd.oomd.enable = false; + networking.useDHCP = false; + networking.firewall.enable = false; + boot.postBootCommands = '' + ls -lha /run + mkdir -p /run/wrappers + ''; + # services.udev.enable = false; + + # TODO: this is only needed because `/run/current-system` is missing + # environment.variables.PATH = "${lib.makeBinPath config.environment.systemPackages}:$PATH"; + + services.mycelium.enable = false; + services.mycelium.keyFile = "/var/lib/secrets/mycelium-keyfile"; + systemd.services.mycelium.serviceConfig.DynamicUser = lib.mkForce false; + systemd.services.mycelium.serviceConfig.User = lib.mkForce "root"; + systemd.services.mycelium.serviceConfig.ExecStart = lib.mkForce (pkgs.writeShellScript "mycelium" '' + while true; do + ls -lha $CREDENTIALS_DIRECTORY + sleep 5 + done + ''); + + systemd.services.testing-credentials = { + description = "testing credentials"; + wantedBy = ["multi-user.target"]; + path = [pkgs.coreutils]; + + serviceConfig = { + SyslogIdentifier = "testing-credential"; + StateDirectory = "testing-credentials"; + # DynamicUser = true; + # User = "tc"; + # ProtectHome = true; + # ProtectSystem = true; + LoadCredential = [ + "mycelium-keyfile:${self.nixosConfigurations.default.config.services.mycelium.keyFile}" + "hosts:/etc/hosts" + ]; + SetCredential = "nosecret:not secret string"; + ExecStart = lib.mkForce (pkgs.writeShellScript "mycelium" '' + cd $STATE_DIRECTORY + env + while true; do + ls -lha $CREDENTIALS_DIRECTORY + sleep 5 + done + ''); + }; + }; + + services.caddy = { + enable = true; + globalConfig = '' + auto_https off + ''; + virtualHosts.":80" = { + extraConfig = '' + respond "hello from ${config.networking.hostName}" + ''; + }; + }; + }) + ]; + }; + packages = forAllSystems (system: let + name = "mycelium"; + inherit (self.inputs) nix-snapshotter; + + config = { + entrypoint = "${self.nixosConfigurations.default.config.system.build.toplevel}/init"; + # port = 2379; + args = [ + ]; + # nodePort = 30001; + }; + + myceliumPorts = { + tcp = [9651]; + udp = [9650 9651]; + }; + + inherit + (config) + entrypoint + # port + + args + # nodePort + + ; + + pkgs = import nixpkgs { + overlays = [nix-snapshotter.overlays.default]; + }; + + image = pkgs.nix-snapshotter.buildImage { + inherit name; + resolvedByNix = true; + config = { + entrypoint = [entrypoint]; + env = [ + # this is read by the `/init` script and prevents various incompatible commands like mount, etc. + # the value of this doesn't seem to matter as long as it's not an empty string. + "container=nerd" + ]; + volumes = { + # "/var/lib/private/mycelium/key.bin" = {}; + }; + copyToRoot = [ + # self.nixosConfigurations.default.config.system.build.toplevel + ]; + }; + }; + in { + k8s = let + pod = pkgs.writeText "${name}-pod.json" (builtins.toJSON { + apiVersion = "v1"; + kind = "Pod"; + metadata = { + inherit name; + labels = {inherit name;}; + }; + spec.containers = [ + { + inherit name args; + image = "nix:0${image}"; + ports = [ + { + name = "mycelium-tcp-0"; + containerPort = builtins.elemAt myceliumPorts.tcp 0; + } + { + name = "mycelium-udp-0"; + protocol = "UDP"; + containerPort = builtins.elemAt myceliumPorts.udp 0; + } + { + name = "mycelium-udp-1"; + protocol = "UDP"; + containerPort = builtins.elemAt myceliumPorts.udp 1; + } + ]; + } + ]; + }); + + service = pkgs.writeText "${name}-service.json" (builtins.toJSON { + apiVersion = "v1"; + kind = "Service"; + metadata.name = "${name}-service"; + spec = { + type = "NodePort"; + selector = {inherit name;}; + ports = [ + { + name = "mycelium-tcp-0"; + port = builtins.elemAt myceliumPorts.tcp 0 + 50000; + targetPort = "mycelium-tcp-0"; + } + { + name = "mycelium-udp-0"; + protocol = "UDP"; + port = builtins.elemAt myceliumPorts.udp 0 + 50000; + targetPort = "mycelium-udp-0"; + } + { + name = "mycelium-udp-1"; + protocol = "UDP"; + port = builtins.elemAt myceliumPorts.udp 1 + 50000; + targetPort = "mycelium-udp-1"; + } + ]; + }; + }); + in + pkgs.runCommand "declarative-k8s" {} '' + mkdir -p $out/share/k8s + cp ${pod} $out/share/k8s/ + cp ${service} $out/share/k8s/ + ''; + + inherit image; + + start = pkgs.writeShellApplication { + name = "start"; + text = '' + set -x + rm -rf ./result + nix build --impure .#image + sudo nix2container load ./result + sudo -E nerdctl run --name ${name} --privileged -dt \ + --volume "$PWD/key.bin.crypt:${self.nixosConfigurations.default.config.services.mycelium.keyFile}:ro" \ + "nix:0$(readlink result):latest" + ''; + }; + + stop = pkgs.writeShellApplication { + name = "stop"; + text = '' + set +e + sudo -E nerdctl stop -t 60 ${name} + sudo -E nerdctl rm --force ${name} + sudo -E nerdctl system prune --all --force + sudo systemctl stop nix-snapshotter + sudo systemctl stop containerd + mount | rg -No '(/var/lib/container|/tmp/initial)[^ ]+' | tac | xargs sudo umount -l + sudo systemctl start containerd + sudo systemctl start nix-snapshotter + ''; + + # tmpfs on /run/credentials/mycelium.service type tmpfs (ro,nosuid,nodev,noexec,relatime,nosymfollow,size=1024k,nr_inodes=1024,mode=700,noswap) + + # mount -t tmpfs tmpfs /run/credentials/mycelium.service -o ro,nosuid,nodev,noexec,relatime,nosymfollow,size=1024k,nr_inodes=1024,mode=700,noswap + }; + }); + }; +} diff --git a/nix/os/devices/steveej-x13s/configuration.nix b/nix/os/devices/steveej-x13s/configuration.nix index d538058..5ecc54b 100644 --- a/nix/os/devices/steveej-x13s/configuration.nix +++ b/nix/os/devices/steveej-x13s/configuration.nix @@ -9,6 +9,12 @@ system, ... }: { + nixpkgs.overlays = [ + (final: previous: { + # inherit (nodeFlake.inputs.nixpkgs-systemd256.legacyPackages.${system}) systemd systemdMinimal; + }) + ]; + nixos-x13s = { enable = true; # TODO: use hardware address @@ -86,6 +92,8 @@ sops.secrets.builder-private-key = {}; nix.distributedBuilds = true; nix.buildMachines = [ + # test these with: sudo nix store ping --store 'ssh-ng://nix-remote-builder@sj-bm-hostkey0.dev.infra.holochain.org?ssh-key=/run/secrets/builder-private-key' + { hostName = "sj-bm-hostkey0.dev.infra.holochain.org"; sshUser = "nix-remote-builder"; @@ -147,6 +155,40 @@ ../../snippets/radicale.nix ../../snippets/holo-zerotier.nix + + { + # (1) Import nixos module. + imports = [ + nodeFlake.inputs.nix-snapshotter.nixosModules.default + ]; + + # (2) Add overlay. + nixpkgs.overlays = [nodeFlake.inputs.nix-snapshotter.overlays.default]; + + # (3) Enable service. + virtualisation.containerd = { + enable = true; + k3sIntegration = false; + nixSnapshotterIntegration = true; + + # TODO: understand if this has an influence on the systemd LoadCredential issue + settings.plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options.SystemdCgroup = lib.mkForce true; + }; + services.nix-snapshotter = { + enable = true; + }; + + # (4) Add a containerd CLI like nerdctl. + environment.systemPackages = [ + pkgs.nerdctl + nodeFlake.inputs.nix-snapshotter.packages.${system}.nix-snapshotter + ]; + + services.k3s = { + enable = false; + setKubeConfig = true; + }; + } ]; networking.hostName = nodeName; @@ -228,11 +270,11 @@ # android on linux virtualisation.waydroid.enable = false; - virtualisation.podman.enable = true; - virtualisation.podman.dockerCompat = true; - hardware.ledger.enable = true; + virtualisation.containers.enable = true; + virtualisation.podman.enable = true; + nix.settings.substituters = [ "https://nixos-x13s.cachix.org" ]; diff --git a/nix/os/devices/steveej-x13s/flake.lock b/nix/os/devices/steveej-x13s/flake.lock index 1335373..4cb431c 100644 --- a/nix/os/devices/steveej-x13s/flake.lock +++ b/nix/os/devices/steveej-x13s/flake.lock @@ -70,7 +70,44 @@ "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz" } }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nix-snapshotter", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1704152458, + "narHash": "sha256-DS+dGw7SKygIWf9w4eNBUZsK+4Ug27NwEWmn2tnbycg=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "88a2cd8166694ba0b6cb374700799cec53aef527", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_2": { "inputs": { "nixpkgs-lib": "nixpkgs-lib" }, @@ -194,9 +231,31 @@ "type": "github" } }, + "nix-snapshotter": { + "inputs": { + "flake-compat": "flake-compat_2", + "flake-parts": "flake-parts", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1717948701, + "narHash": "sha256-G7SXaZ7J4yO4OQEKSZPVWcccfV87uyLech0jEOU350g=", + "owner": "yu-re-ka", + "repo": "nix-snapshotter", + "rev": "c10b066a4b1bb3451507c141636014e3335e579e", + "type": "github" + }, + "original": { + "owner": "yu-re-ka", + "repo": "nix-snapshotter", + "type": "github" + } + }, "nixos-x13s": { "inputs": { - "flake-parts": "flake-parts", + "flake-parts": "flake-parts_2", "nixpkgs": "nixpkgs_2" }, "locked": { @@ -297,6 +356,7 @@ "home-manager": "home-manager", "mobile-nixos": "mobile-nixos", "mycelium": "mycelium", + "nix-snapshotter": "nix-snapshotter", "nixos-x13s": "nixos-x13s", "nixpkgs": "nixpkgs_3", "nixpkgs-unstable": "nixpkgs-unstable" diff --git a/nix/os/devices/steveej-x13s/flake.nix b/nix/os/devices/steveej-x13s/flake.nix index 8d1328a..251ff89 100644 --- a/nix/os/devices/steveej-x13s/flake.nix +++ b/nix/os/devices/steveej-x13s/flake.nix @@ -27,6 +27,12 @@ url = "git+https://git.codelinaro.org/clo/ath-firmware/ath11k-firmware.git"; flake = false; }; + + nix-snapshotter = { + url = "github:yu-re-ka/nix-snapshotter"; + # url = "github:pdtpartners/nix-snapshotter"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; outputs = { From fd706db158728c7f1d95be03a9878b4735afb7b1 Mon Sep 17 00:00:00 2001 From: Stefan Junker Date: Fri, 2 Aug 2024 09:25:09 +0200 Subject: [PATCH 2/6] nix/os/devices/steveej-x13s: bump versions --- nix/os/devices/steveej-x13s/flake.lock | 44 +++++++++++++------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/nix/os/devices/steveej-x13s/flake.lock b/nix/os/devices/steveej-x13s/flake.lock index 1335373..4d3d60a 100644 --- a/nix/os/devices/steveej-x13s/flake.lock +++ b/nix/os/devices/steveej-x13s/flake.lock @@ -44,11 +44,11 @@ ] }, "locked": { - "lastModified": 1721266288, - "narHash": "sha256-MsyTzXu9CJVcBr44ct8ILKF/Ro7VlF+tVZTylzAoXSs=", + "lastModified": 1722476845, + "narHash": "sha256-7gZ8uf3qOox8Vrwd+p9EhUHHLhhK8lis/5KcXGmIaow=", "owner": "nix-community", "repo": "disko", - "rev": "e8e8d9a3a9c1d0e654ccda7834bf0288a9d15c47", + "rev": "7e1b215a0a96efb306ad6440bf706d2b307dc267", "type": "github" }, "original": { @@ -144,11 +144,11 @@ "mobile-nixos": { "flake": false, "locked": { - "lastModified": 1721250279, - "narHash": "sha256-S47+MFnArDvR7OFCXU0BQoyLCBwSZws+t+bany7Ol6w=", + "lastModified": 1722056346, + "narHash": "sha256-50fcuCppaLMfSOTFO4IkCBs4folToCwlhTgc6IdZFHg=", "owner": "NixOS", "repo": "mobile-nixos", - "rev": "472073a51745cca03257cf625582252cdd04ec21", + "rev": "717ce90cfadffa449480bae2e155185c651e9993", "type": "github" }, "original": { @@ -166,11 +166,11 @@ "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1721210676, - "narHash": "sha256-6hwlFcRPbfW+tOrKnnx3AL/yWjXqkXHoiIpbFsPMEAg=", + "lastModified": 1722421086, + "narHash": "sha256-Po8GvfDt4O646w1IDIcRdiRIxnqVLv/OwBhr5nOazLw=", "owner": "threefoldtech", "repo": "mycelium", - "rev": "767060eb8bd5abcc8e375bdb8f0984bcb27936da", + "rev": "e4d7be8556b259a2204f19d6a0537da3aa7d7a3e", "type": "github" }, "original": { @@ -200,11 +200,11 @@ "nixpkgs": "nixpkgs_2" }, "locked": { - "lastModified": 1721131656, - "narHash": "sha256-0EoouVAaS8GHZJitpzQ9S4s4Gps4hkzr8yQcgYkLxwo=", + "lastModified": 1722396240, + "narHash": "sha256-t4C/I1CttOGFLYzzjd6zNOPvPleykqbPwO0YT19S6jI=", "ref": "refs/heads/main", - "rev": "f1a2bace41471300aa2b0c5c98e6e37087532e16", - "revCount": 91, + "rev": "a59ae21e47538cecddadc220ea16a7b8e2485f66", + "revCount": 97, "type": "git", "url": "https://codeberg.org/adamcstephens/nixos-x13s" }, @@ -243,11 +243,11 @@ }, "nixpkgs-unstable": { "locked": { - "lastModified": 1720957393, - "narHash": "sha256-oedh2RwpjEa+TNxhg5Je9Ch6d3W1NKi7DbRO1ziHemA=", + "lastModified": 1722421184, + "narHash": "sha256-/DJBI6trCeVnasdjUo9pbnodCLZcFqnVZiLUfqLH4jA=", "owner": "nixos", "repo": "nixpkgs", - "rev": "693bc46d169f5af9c992095736e82c3488bf7dbb", + "rev": "9f918d616c5321ad374ae6cb5ea89c9e04bf3e58", "type": "github" }, "original": { @@ -259,11 +259,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1720957393, - "narHash": "sha256-oedh2RwpjEa+TNxhg5Je9Ch6d3W1NKi7DbRO1ziHemA=", + "lastModified": 1722062969, + "narHash": "sha256-QOS0ykELUmPbrrUGmegAUlpmUFznDQeR4q7rFhl8eQg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "693bc46d169f5af9c992095736e82c3488bf7dbb", + "rev": "b73c2221a46c13557b1b3be9c2070cc42cf01eb3", "type": "github" }, "original": { @@ -275,11 +275,11 @@ }, "nixpkgs_3": { "locked": { - "lastModified": 1720954236, - "narHash": "sha256-1mEKHp4m9brvfQ0rjCca8P1WHpymK3TOr3v34ydv9bs=", + "lastModified": 1722372011, + "narHash": "sha256-B2xRiC3NEJy/82ugtareBkRqEkPGpMyjaLxaR8LBxNs=", "owner": "nixos", "repo": "nixpkgs", - "rev": "53e81e790209e41f0c1efa9ff26ff2fd7ab35e27", + "rev": "cf05eeada35e122770c5c14add958790fcfcbef5", "type": "github" }, "original": { From fa45a3a1e3af0cf1ab0255713d236576e3005f0d Mon Sep 17 00:00:00 2001 From: Stefan Junker Date: Sat, 3 Aug 2024 18:13:53 +0200 Subject: [PATCH 3/6] chromium: switch back to full chromium for extensions support --- nix/home-manager/programs/chromium.nix | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/nix/home-manager/programs/chromium.nix b/nix/home-manager/programs/chromium.nix index be021d5..e48456e 100644 --- a/nix/home-manager/programs/chromium.nix +++ b/nix/home-manager/programs/chromium.nix @@ -42,22 +42,29 @@ # unhook {id = "khncfooichmfjbepaaaebmommgaepoid";} - - # polkadotjs - {id = "mopnmbcafieddcagagdcbnhejhlodfdd";} ] ++ (lib.lists.optionals ((builtins.match "^steveej.*" name) != null) [ + # polkadotjs + {id = "mopnmbcafieddcagagdcbnhejhlodfdd";} + + # rabby wallet + {id = "acmacodkjbdgmoleebolmdjonilkdbch";} + # Vimium C {id = "hfjbmagddngcpeloejdejnfgbamkjaeg";} # always right {id = "npjpaghfnndnnmjiliibnkmdfgbojokj";} + + # shazam music + {id = "mmioliijnhnoblpgimnlajmefafdfilb";} ]); in { programs.chromium = { enable = true; inherit extensions; - package = pkgs.ungoogled-chromium; + # TODO: extensions currently don't work with ungoogled-chromium + package = pkgs.chromium; }; programs.brave = { @@ -67,6 +74,4 @@ in { && !pkgs.stdenv.targetPlatform.isAarch64; inherit extensions; }; - - programs.browserpass = {browsers = ["chromium" "brave"];}; } From c64db438a28dff412b2c3635bd6e8944ee871cd9 Mon Sep 17 00:00:00 2001 From: Stefan Junker Date: Sat, 3 Aug 2024 18:14:18 +0200 Subject: [PATCH 4/6] router0-dmz0: further wifi tweaking with these settings i did not get a single disconnect after 3-4 hours of testing. --- nix/os/devices/router0-dmz0/configuration.nix | 149 ++---------------- 1 file changed, 10 insertions(+), 139 deletions(-) diff --git a/nix/os/devices/router0-dmz0/configuration.nix b/nix/os/devices/router0-dmz0/configuration.nix index 1486713..0691c01 100644 --- a/nix/os/devices/router0-dmz0/configuration.nix +++ b/nix/os/devices/router0-dmz0/configuration.nix @@ -909,7 +909,7 @@ in { channel = 0; # 0 would mean Automatic Channel Selection settings = { - # TODO: with 1 the x13s on windows can't connect, however this slows the AP down. + # TODO: this would be faster but x13s on windows can't connect when it's enabled. # ieee80211n = 1; # Exclude DFS channels from ACS @@ -921,6 +921,11 @@ in { # indications of connection loss. This depends on the driver capabilities and # may not be available with all drivers. disassoc_low_ack = 0; + + skip_inactivity_poll = 1; + + # TODO: check if this is required. multicast can be more efficient so it'd be nice to disable this. + multicast_to_unicast = 1; }; # use 'iw phy#1 info' to determine your VHT capabilities @@ -1038,7 +1043,8 @@ in { wmm_enabled = 1; # IEEE 802.11i (authentication) related configuration - # Encrypt management frames to protect against deauthentication and similar attacks + # Encrypt management frames to protect against deauthentication and similar attacks. + # 0 := disabled; 1 := optional; 2 := required ieee80211w = 1; # sae_require_mfp = 1; # sae_groups = "19 20 21"; @@ -1046,147 +1052,12 @@ in { # [ENABLE-TLSv1.3] = enable TLSv1.3 (experimental - disabled by default) tls_flags = "[ENABLE-TLSv1.3]"; - ieee8021x = 1; - eap_server = 1; - - multicast_to_unicast = 0; - - skip_inactivity_poll = 0; - disassoc_low_ack = 0; + ieee8021x = 0; + eap_server = 0; }; }; - - # wlan0-1 = { - # ssid = "mlsia-testing"; - # authentication = { - # mode = "wpa3-sae-transition"; - # }; - - # bssid = mkBssid 1; - # settings = { - # bridge = bridgeInterfaceName; - # }; - # }; - - # wlan0-1 = { - # ssid = "justtestingwifi-wpa3"; - # authentication = { - # mode = "wpa3-sae"; - # saePasswordsFile = config.sops.secrets.wlan0_1_saePasswordFile.path; - # }; - - # bssid = mkBssid 1; - # settings = { - # bridge = bridgeInterfaceName; - # }; - # }; - - # Uncomment when needed otherwise remove - # wlan0-1 = { - # ssid = "koteczkowo3"; - # authentication = { - # mode = "none"; # this is overriden by settings - # }; - # managementFrameProtection = "optional"; - # bssid = "e6:02:43:07:00:00"; - # settings = { - # bridge = bridgeInterfaceName; - # wpa = lib.mkForce 2; - # wpa_key_mgmt = "WPA-PSK"; - # wpa_pairwise = "CCMP"; - # wpa_psk_file = config.sops.secrets.legacyWifiPassword.path; - # }; - # }; }; }; - - # wlan1 = { - # band = "5g"; - # # channels with 160 MHz width in Poland: 36, 52, 100 i 116 - # channel = 0; # ACS - # countryCode = "PL"; - - # # use 'iw phy#1 info' to determine your VHT capabilities - # wifi4 = { - # enable = true; - # capabilities = ["HT40+" "LDPC" "SHORT-GI-20" "SHORT-GI-40" "TX-STBC" "RX-STBC1" "MAX-AMSDU-7935"]; - # }; - # wifi5 = { - # enable = true; - # operatingChannelWidth = "160"; - # capabilities = ["RXLDPC" "SHORT-GI-80" "SHORT-GI-160" "TX-STBC-2BY1" "SU-BEAMFORMER" "SU-BEAMFORMEE" "MU-BEAMFORMER" "MU-BEAMFORMEE" "RX-ANTENNA-PATTERN" "TX-ANTENNA-PATTERN" "RX-STBC-1" "SOUNDING-DIMENSION-4" "BF-ANTENNA-4" "VHT160" "MAX-MPDU-11454" "MAX-A-MPDU-LEN-EXP7"]; - # }; - # wifi6 = { - # enable = true; - # singleUserBeamformer = true; - # singleUserBeamformee = true; - # multiUserBeamformer = true; - # operatingChannelWidth = "160"; - # }; - # settings = { - # # these two are mandatory for wifi 5 & 6 to work - # vht_oper_centr_freq_seg0_idx = 50; - # he_oper_centr_freq_seg0_idx = 50; - - # # The "tx_queue_data2_burst" parameter in Linux refers to the burst size for - # # transmitting data packets from the second data queue of a network interface. - # # It determines the number of packets that can be sent in a burst. - # # Adjusting this parameter can impact network throughput and latency. - # tx_queue_data2_burst = 2; - - # # The "he_bss_color" parameter in Wi-Fi 6 (802.11ax) refers to the BSS Color field in the HE (High Efficiency) MAC header. - # # BSS Color is a mechanism introduced in Wi-Fi 6 to mitigate interference and improve network efficiency in dense deployment scenarios. - # # It allows multiple overlapping Basic Service Sets (BSS) to differentiate and coexist in the same area without causing excessive interference. - # he_bss_color = 63; # was set to 128 by openwrt but range of possible values in 2.10 is 1-63 - - # # Magic values that were set by openwrt but I didn't bother inspecting every single one - # he_spr_sr_control = 3; - # he_default_pe_duration = 4; - # he_rts_threshold = 1023; - - # he_mu_edca_qos_info_param_count = 0; - # he_mu_edca_qos_info_q_ack = 0; - # he_mu_edca_qos_info_queue_request = 0; - # he_mu_edca_qos_info_txop_request = 0; - - # # he_mu_edca_ac_be_aci=0; missing in 2.10 - # he_mu_edca_ac_be_aifsn = 8; - # he_mu_edca_ac_be_ecwmin = 9; - # he_mu_edca_ac_be_ecwmax = 10; - # he_mu_edca_ac_be_timer = 255; - - # he_mu_edca_ac_bk_aifsn = 15; - # he_mu_edca_ac_bk_aci = 1; - # he_mu_edca_ac_bk_ecwmin = 9; - # he_mu_edca_ac_bk_ecwmax = 10; - # he_mu_edca_ac_bk_timer = 255; - - # he_mu_edca_ac_vi_ecwmin = 5; - # he_mu_edca_ac_vi_ecwmax = 7; - # he_mu_edca_ac_vi_aifsn = 5; - # he_mu_edca_ac_vi_aci = 2; - # he_mu_edca_ac_vi_timer = 255; - - # he_mu_edca_ac_vo_aifsn = 5; - # he_mu_edca_ac_vo_aci = 3; - # he_mu_edca_ac_vo_ecwmin = 5; - # he_mu_edca_ac_vo_ecwmax = 7; - # he_mu_edca_ac_vo_timer = 255; - # }; - # networks = { - # wlan1 = { - # ssid = "koteczkowo5"; - # authentication = { - # mode = "wpa3-sae"; - # saePasswordsFile = config.sops.secrets.wifiPassword.path; # Use saePasswordsFile if possible. - # }; - # bssid = "36:b9:02:21:08:a2"; - # settings = { - # bridge = bridgeInterfaceName; - # }; - # }; - # }; - # }; }; }; From 186488456c64cdb4a952aa110239bc2f482dbeeb Mon Sep 17 00:00:00 2001 From: Stefan Junker Date: Sat, 3 Aug 2024 18:15:16 +0200 Subject: [PATCH 5/6] steveej-x13s: re-enable syncthing and cleanup --- nix/os/devices/steveej-x13s/configuration.nix | 20 ------------------- nix/os/devices/steveej-x13s/flake.nix | 3 --- 2 files changed, 23 deletions(-) diff --git a/nix/os/devices/steveej-x13s/configuration.nix b/nix/os/devices/steveej-x13s/configuration.nix index d538058..63f932e 100644 --- a/nix/os/devices/steveej-x13s/configuration.nix +++ b/nix/os/devices/steveej-x13s/configuration.nix @@ -176,8 +176,6 @@ ../../../home-manager/configuration/graphical-fullblown.nix ]; - services.syncthing.enable = lib.mkForce false; - home.sessionVariables = {}; home.packages = with pkgs; [ @@ -185,7 +183,6 @@ # TODO: currently unsupported services.gammastep.enable = lib.mkForce false; - # programs.chromium.enable = lib.mkForce false; }; boot = { @@ -193,23 +190,6 @@ loader.efi.canTouchEfiVariables = lib.mkForce false; loader.efi.efiSysMountPoint = "/boot"; blacklistedKernelModules = ["wwan"]; - - # kernelParams = let - # dtbName = "sc8280xp-lenovo-thinkpad-x13s.dtb"; - # in lib.mkForce [ - # # needed to boot - # "dtb=${dtbName}" - - # # jhovold recommended - # "efi=noruntime" - # "clk_ignore_unused" - # "pd_ignore_unused" - # # "regulator_ignore_unused" - # "arm64.nopauth" - - # # blacklist graphics in initrd so the firmware can load from disk - # "rd.driver.blacklist=msm" - # ]; }; hardware.firmware = lib.mkBefore [ diff --git a/nix/os/devices/steveej-x13s/flake.nix b/nix/os/devices/steveej-x13s/flake.nix index 8d1328a..8a30dbe 100644 --- a/nix/os/devices/steveej-x13s/flake.nix +++ b/nix/os/devices/steveej-x13s/flake.nix @@ -9,9 +9,6 @@ disko.inputs.nixpkgs.follows = "nixpkgs"; - mobile-nixos.url = "github:NixOS/mobile-nixos"; - mobile-nixos.flake = false; - home-manager = { url = "github:nix-community/home-manager/release-24.05"; inputs.nixpkgs.follows = "nixpkgs"; From e6c10aad010f84cc04a953f9e632d91d9fe73039 Mon Sep 17 00:00:00 2001 From: Stefan Junker Date: Thu, 13 Jun 2024 16:42:55 +0200 Subject: [PATCH 6/6] experiment with k3s + nix-snapshotter --- nix/os/containers/mycelium/configuration.nix | 0 nix/os/containers/mycelium/flake.lock | 141 ++++++++ nix/os/containers/mycelium/flake.nix | 340 ++++++++++++++++++ nix/os/devices/steveej-x13s/configuration.nix | 49 ++- nix/os/devices/steveej-x13s/flake.lock | 62 +++- nix/os/devices/steveej-x13s/flake.nix | 6 + 6 files changed, 594 insertions(+), 4 deletions(-) create mode 100644 nix/os/containers/mycelium/configuration.nix create mode 100644 nix/os/containers/mycelium/flake.lock create mode 100644 nix/os/containers/mycelium/flake.nix diff --git a/nix/os/containers/mycelium/configuration.nix b/nix/os/containers/mycelium/configuration.nix new file mode 100644 index 0000000..e69de29 diff --git a/nix/os/containers/mycelium/flake.lock b/nix/os/containers/mycelium/flake.lock new file mode 100644 index 0000000..899ee98 --- /dev/null +++ b/nix/os/containers/mycelium/flake.lock @@ -0,0 +1,141 @@ +{ + "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nix-snapshotter", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1704152458, + "narHash": "sha256-DS+dGw7SKygIWf9w4eNBUZsK+4Ug27NwEWmn2tnbycg=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "88a2cd8166694ba0b6cb374700799cec53aef527", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "nix-snapshotter": { + "inputs": { + "flake-compat": "flake-compat", + "flake-parts": "flake-parts", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1715438114, + "narHash": "sha256-btb702TXuhDg0D6tW0dCOy4+II9Wl6BJ0LvpT+O9wrs=", + "owner": "pdtpartners", + "repo": "nix-snapshotter", + "rev": "7b251c9356bc7bb383ebeedcd0045b3ae431bff7", + "type": "github" + }, + "original": { + "owner": "pdtpartners", + "repo": "nix-snapshotter", + "type": "github" + } + }, + "nixlib": { + "locked": { + "lastModified": 1712450863, + "narHash": "sha256-K6IkdtMtq9xktmYPj0uaYc8NsIqHuaAoRBaMgu9Fvrw=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "3c62b6a12571c9a7f65ab037173ee153d539905f", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "nixos-generators": { + "inputs": { + "nixlib": "nixlib", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1718025593, + "narHash": "sha256-WZ1gdKq/9u1Ns/oXuNsDm+W0salonVA0VY1amw8urJ4=", + "owner": "nix-community", + "repo": "nixos-generators", + "rev": "35c20ba421dfa5059e20e0ef2343c875372bdcf3", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixos-generators", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1718086528, + "narHash": "sha256-hoB7B7oPgypePz16cKWawPfhVvMSXj4G/qLsfFuhFjw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "47b604b07d1e8146d5398b42d3306fdebd343986", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-systemd256": { + "locked": { + "lastModified": 1718397913, + "narHash": "sha256-omV+dq3GdXQQTaewxhkBgxM4Bbwqa4D9FVS4dTITxOQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "962cf03fb8c782c5e00f465397e03dc84284acc9", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "962cf03fb8c782c5e00f465397e03dc84284acc9", + "type": "github" + } + }, + "root": { + "inputs": { + "nix-snapshotter": "nix-snapshotter", + "nixos-generators": "nixos-generators", + "nixpkgs": "nixpkgs", + "nixpkgs-systemd256": "nixpkgs-systemd256" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/nix/os/containers/mycelium/flake.nix b/nix/os/containers/mycelium/flake.nix new file mode 100644 index 0000000..6f247a1 --- /dev/null +++ b/nix/os/containers/mycelium/flake.nix @@ -0,0 +1,340 @@ +{ + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; + nixpkgs-systemd256.url = "github:NixOS/nixpkgs/962cf03fb8c782c5e00f465397e03dc84284acc9"; + nixos-generators = { + url = "github:nix-community/nixos-generators"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + nix-snapshotter = { + url = "github:pdtpartners/nix-snapshotter"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + outputs = { + self, + nixpkgs, + nixos-generators, + ... + }: let + systems = [ + "aarch64-linux" + "x86_64-linux" + ]; + forAllSystems = nixpkgs.lib.genAttrs systems; + in { + nixosConfigurations.default = + nixpkgs.lib.nixosSystem + { + system = "aarch64-linux"; + + specialArgs = {}; + + modules = [ + ({ + config, + modulesPath, + pkgs, + lib, + ... + }: { + nixpkgs.overlays = [ + (final: previous: { + # inherit (self.inputs.nixpkgs-systemd256.legacyPackages.${pkgs.system}) systemd systemdMinimal; + # systemd = + # self.inputs.nixpkgs-systemd256.legacyPackages.${pkgs.system}.systemd.overrideAttrs (prevAttrs: { + # src = /home/steveej/src/others/systemd; + + # withAppArmor = false; + # withRepart = false; + # withHomed = false; + # withAcl = false; + # withEfi = false; + # withBootloader = false; + # withCryptsetup = false; + # withLibBPF = false; + # withOomd = false; + # withFido2 = false; + # withApparmor = false; + # withDocumentation = false; + # withUtmp = false; + # withQrencode = false; + # withVmspawn = false; + # withMachined = false; + # withLogTrace = true; + # withArchive = false; + # # don't need these but cause errors for exampel files not found + # # withLogind = false; + # }) + # pkgs.systemdMinimal.override { + # # getting errors with these disabled + # withCoredump = true; + # withCompression = true; + # withLogind = true; + # withSysusers = true; + # withUserDb = true; + # } + # pkgs.systemdMinimal + # pkgs.systemd.override { + # withRepart = false; + # withHomed = false; + # withAcl = false; + # withEfi = false; + # withBootloader = false; + # withCryptsetup = false; + # withLibBPF = false; + # withOomd = false; + # withFido2 = false; + # withApparmor = false; + # withDocumentation = false; + # withUtmp = false; + # withQrencode = false; + # withVmspawn = false; + # withMachined = false; + # withLogTrace = true; + # # don't need these but cause errors for exampel files not found + # # withLogind = false; + # } + # ; + }) + ]; + + imports = [ + (modulesPath + "/profiles/minimal.nix") + ]; + system.stateVersion = "24.05"; + + # https://github.com/hercules-ci/arion/blob/c24c185e67f093298a081900b49ca18716077dec/src/nix/modules/nixos/container-systemd.nix + boot.isContainer = true; + # boot.tmp.useTmpfs = true; + boot.loader.grub.enable = lib.mkForce false; + boot.loader.systemd-boot.enable = lib.mkForce false; + services.journald.console = "/dev/console"; + services.journald.storage = "none"; + # boot.specialFileSystems = lib.mkForce {}; + + services.nscd.enable = false; + system.nssModules = lib.mkForce []; + systemd.services.systemd-logind.enable = false; + systemd.services.console-getty.enable = true; + + systemd.sockets.nix-daemon.enable = false; + systemd.services.nix-daemon.enable = false; + systemd.oomd.enable = false; + networking.useDHCP = false; + networking.firewall.enable = false; + boot.postBootCommands = '' + ls -lha /run + mkdir -p /run/wrappers + ''; + # services.udev.enable = false; + + # TODO: this is only needed because `/run/current-system` is missing + # environment.variables.PATH = "${lib.makeBinPath config.environment.systemPackages}:$PATH"; + + services.mycelium.enable = false; + services.mycelium.keyFile = "/var/lib/secrets/mycelium-keyfile"; + systemd.services.mycelium.serviceConfig.DynamicUser = lib.mkForce false; + systemd.services.mycelium.serviceConfig.User = lib.mkForce "root"; + systemd.services.mycelium.serviceConfig.ExecStart = lib.mkForce (pkgs.writeShellScript "mycelium" '' + while true; do + ls -lha $CREDENTIALS_DIRECTORY + sleep 5 + done + ''); + + systemd.services.testing-credentials = { + description = "testing credentials"; + wantedBy = ["multi-user.target"]; + path = [pkgs.coreutils]; + + serviceConfig = { + SyslogIdentifier = "testing-credential"; + StateDirectory = "testing-credentials"; + # DynamicUser = true; + # User = "tc"; + # ProtectHome = true; + # ProtectSystem = true; + LoadCredential = [ + "mycelium-keyfile:${self.nixosConfigurations.default.config.services.mycelium.keyFile}" + "hosts:/etc/hosts" + ]; + SetCredential = "nosecret:not secret string"; + ExecStart = lib.mkForce (pkgs.writeShellScript "mycelium" '' + cd $STATE_DIRECTORY + env + while true; do + ls -lha $CREDENTIALS_DIRECTORY + sleep 5 + done + ''); + }; + }; + + services.caddy = { + enable = true; + globalConfig = '' + auto_https off + ''; + virtualHosts.":80" = { + extraConfig = '' + respond "hello from ${config.networking.hostName}" + ''; + }; + }; + }) + ]; + }; + packages = forAllSystems (system: let + name = "mycelium"; + inherit (self.inputs) nix-snapshotter; + + config = { + entrypoint = "${self.nixosConfigurations.default.config.system.build.toplevel}/init"; + # port = 2379; + args = [ + ]; + # nodePort = 30001; + }; + + myceliumPorts = { + tcp = [9651]; + udp = [9650 9651]; + }; + + inherit + (config) + entrypoint + # port + + args + # nodePort + + ; + + pkgs = import nixpkgs { + overlays = [nix-snapshotter.overlays.default]; + }; + + image = pkgs.nix-snapshotter.buildImage { + inherit name; + resolvedByNix = true; + config = { + entrypoint = [entrypoint]; + env = [ + # this is read by the `/init` script and prevents various incompatible commands like mount, etc. + # the value of this doesn't seem to matter as long as it's not an empty string. + "container=nerd" + ]; + volumes = { + # "/var/lib/private/mycelium/key.bin" = {}; + }; + copyToRoot = [ + # self.nixosConfigurations.default.config.system.build.toplevel + ]; + }; + }; + in { + k8s = let + pod = pkgs.writeText "${name}-pod.json" (builtins.toJSON { + apiVersion = "v1"; + kind = "Pod"; + metadata = { + inherit name; + labels = {inherit name;}; + }; + spec.containers = [ + { + inherit name args; + image = "nix:0${image}"; + ports = [ + { + name = "mycelium-tcp-0"; + containerPort = builtins.elemAt myceliumPorts.tcp 0; + } + { + name = "mycelium-udp-0"; + protocol = "UDP"; + containerPort = builtins.elemAt myceliumPorts.udp 0; + } + { + name = "mycelium-udp-1"; + protocol = "UDP"; + containerPort = builtins.elemAt myceliumPorts.udp 1; + } + ]; + } + ]; + }); + + service = pkgs.writeText "${name}-service.json" (builtins.toJSON { + apiVersion = "v1"; + kind = "Service"; + metadata.name = "${name}-service"; + spec = { + type = "NodePort"; + selector = {inherit name;}; + ports = [ + { + name = "mycelium-tcp-0"; + port = builtins.elemAt myceliumPorts.tcp 0 + 50000; + targetPort = "mycelium-tcp-0"; + } + { + name = "mycelium-udp-0"; + protocol = "UDP"; + port = builtins.elemAt myceliumPorts.udp 0 + 50000; + targetPort = "mycelium-udp-0"; + } + { + name = "mycelium-udp-1"; + protocol = "UDP"; + port = builtins.elemAt myceliumPorts.udp 1 + 50000; + targetPort = "mycelium-udp-1"; + } + ]; + }; + }); + in + pkgs.runCommand "declarative-k8s" {} '' + mkdir -p $out/share/k8s + cp ${pod} $out/share/k8s/ + cp ${service} $out/share/k8s/ + ''; + + inherit image; + + start = pkgs.writeShellApplication { + name = "start"; + text = '' + set -x + rm -rf ./result + nix build --impure .#image + sudo nix2container load ./result + sudo -E nerdctl run --name ${name} --privileged -dt \ + --volume "$PWD/key.bin.crypt:${self.nixosConfigurations.default.config.services.mycelium.keyFile}:ro" \ + "nix:0$(readlink result):latest" + ''; + }; + + stop = pkgs.writeShellApplication { + name = "stop"; + text = '' + set +e + sudo -E nerdctl stop -t 60 ${name} + sudo -E nerdctl rm --force ${name} + sudo -E nerdctl system prune --all --force + sudo systemctl stop nix-snapshotter + sudo systemctl stop containerd + mount | rg -No '(/var/lib/container|/tmp/initial)[^ ]+' | tac | xargs sudo umount -l + sudo systemctl start containerd + sudo systemctl start nix-snapshotter + ''; + + # tmpfs on /run/credentials/mycelium.service type tmpfs (ro,nosuid,nodev,noexec,relatime,nosymfollow,size=1024k,nr_inodes=1024,mode=700,noswap) + + # mount -t tmpfs tmpfs /run/credentials/mycelium.service -o ro,nosuid,nodev,noexec,relatime,nosymfollow,size=1024k,nr_inodes=1024,mode=700,noswap + }; + }); + }; +} diff --git a/nix/os/devices/steveej-x13s/configuration.nix b/nix/os/devices/steveej-x13s/configuration.nix index 63f932e..16727f0 100644 --- a/nix/os/devices/steveej-x13s/configuration.nix +++ b/nix/os/devices/steveej-x13s/configuration.nix @@ -9,6 +9,12 @@ system, ... }: { + nixpkgs.overlays = [ + (final: previous: { + # inherit (nodeFlake.inputs.nixpkgs-systemd256.legacyPackages.${system}) systemd systemdMinimal; + }) + ]; + nixos-x13s = { enable = true; # TODO: use hardware address @@ -86,6 +92,8 @@ sops.secrets.builder-private-key = {}; nix.distributedBuilds = true; nix.buildMachines = [ + # test these with: sudo nix store ping --store 'ssh-ng://nix-remote-builder@sj-bm-hostkey0.dev.infra.holochain.org?ssh-key=/run/secrets/builder-private-key' + { hostName = "sj-bm-hostkey0.dev.infra.holochain.org"; sshUser = "nix-remote-builder"; @@ -147,6 +155,41 @@ ../../snippets/radicale.nix ../../snippets/holo-zerotier.nix + + # experiment with k3s, nix-snapshotter, and nixos images + { + # (1) Import nixos module. + imports = [ + nodeFlake.inputs.nix-snapshotter.nixosModules.default + ]; + + # (2) Add overlay. + nixpkgs.overlays = [nodeFlake.inputs.nix-snapshotter.overlays.default]; + + # (3) Enable service. + virtualisation.containerd = { + enable = true; + k3sIntegration = false; + nixSnapshotterIntegration = true; + + # TODO: understand if this has an influence on the systemd LoadCredential issue + settings.plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options.SystemdCgroup = lib.mkForce true; + }; + services.nix-snapshotter = { + enable = true; + }; + + # (4) Add a containerd CLI like nerdctl. + environment.systemPackages = [ + pkgs.nerdctl + nodeFlake.inputs.nix-snapshotter.packages.${system}.nix-snapshotter + ]; + + services.k3s = { + enable = false; + setKubeConfig = true; + }; + } ]; networking.hostName = nodeName; @@ -208,11 +251,11 @@ # android on linux virtualisation.waydroid.enable = false; - virtualisation.podman.enable = true; - virtualisation.podman.dockerCompat = true; - hardware.ledger.enable = true; + virtualisation.containers.enable = true; + virtualisation.podman.enable = true; + nix.settings.substituters = [ "https://nixos-x13s.cachix.org" ]; diff --git a/nix/os/devices/steveej-x13s/flake.lock b/nix/os/devices/steveej-x13s/flake.lock index 4d3d60a..dcad237 100644 --- a/nix/os/devices/steveej-x13s/flake.lock +++ b/nix/os/devices/steveej-x13s/flake.lock @@ -70,7 +70,44 @@ "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz" } }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nix-snapshotter", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1704152458, + "narHash": "sha256-DS+dGw7SKygIWf9w4eNBUZsK+4Ug27NwEWmn2tnbycg=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "88a2cd8166694ba0b6cb374700799cec53aef527", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_2": { "inputs": { "nixpkgs-lib": "nixpkgs-lib" }, @@ -194,9 +231,31 @@ "type": "github" } }, + "nix-snapshotter": { + "inputs": { + "flake-compat": "flake-compat_2", + "flake-parts": "flake-parts", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1717948701, + "narHash": "sha256-G7SXaZ7J4yO4OQEKSZPVWcccfV87uyLech0jEOU350g=", + "owner": "yu-re-ka", + "repo": "nix-snapshotter", + "rev": "c10b066a4b1bb3451507c141636014e3335e579e", + "type": "github" + }, + "original": { + "owner": "yu-re-ka", + "repo": "nix-snapshotter", + "type": "github" + } + }, "nixos-x13s": { "inputs": { - "flake-parts": "flake-parts", + "flake-parts": "flake-parts_2", "nixpkgs": "nixpkgs_2" }, "locked": { @@ -297,6 +356,7 @@ "home-manager": "home-manager", "mobile-nixos": "mobile-nixos", "mycelium": "mycelium", + "nix-snapshotter": "nix-snapshotter", "nixos-x13s": "nixos-x13s", "nixpkgs": "nixpkgs_3", "nixpkgs-unstable": "nixpkgs-unstable" diff --git a/nix/os/devices/steveej-x13s/flake.nix b/nix/os/devices/steveej-x13s/flake.nix index 8a30dbe..4e3f1a9 100644 --- a/nix/os/devices/steveej-x13s/flake.nix +++ b/nix/os/devices/steveej-x13s/flake.nix @@ -24,6 +24,12 @@ url = "git+https://git.codelinaro.org/clo/ath-firmware/ath11k-firmware.git"; flake = false; }; + + nix-snapshotter = { + url = "github:yu-re-ka/nix-snapshotter"; + # url = "github:pdtpartners/nix-snapshotter"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; outputs = {