diff --git a/nix/os/containers/mycelium/configuration.nix b/nix/os/containers/mycelium/configuration.nix deleted file mode 100644 index e69de29..0000000 diff --git a/nix/os/containers/mycelium/flake.lock b/nix/os/containers/mycelium/flake.lock deleted file mode 100644 index 899ee98..0000000 --- a/nix/os/containers/mycelium/flake.lock +++ /dev/null @@ -1,141 +0,0 @@ -{ - "nodes": { - "flake-compat": { - "flake": false, - "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, - "flake-parts": { - "inputs": { - "nixpkgs-lib": [ - "nix-snapshotter", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1704152458, - "narHash": "sha256-DS+dGw7SKygIWf9w4eNBUZsK+4Ug27NwEWmn2tnbycg=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "88a2cd8166694ba0b6cb374700799cec53aef527", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" - } - }, - "nix-snapshotter": { - "inputs": { - "flake-compat": "flake-compat", - "flake-parts": "flake-parts", - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1715438114, - "narHash": "sha256-btb702TXuhDg0D6tW0dCOy4+II9Wl6BJ0LvpT+O9wrs=", - "owner": "pdtpartners", - "repo": "nix-snapshotter", - "rev": "7b251c9356bc7bb383ebeedcd0045b3ae431bff7", - "type": "github" - }, - "original": { - "owner": "pdtpartners", - "repo": "nix-snapshotter", - "type": "github" - } - }, - "nixlib": { - "locked": { - "lastModified": 1712450863, - "narHash": "sha256-K6IkdtMtq9xktmYPj0uaYc8NsIqHuaAoRBaMgu9Fvrw=", - "owner": "nix-community", - "repo": "nixpkgs.lib", - "rev": "3c62b6a12571c9a7f65ab037173ee153d539905f", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nixpkgs.lib", - "type": "github" - } - }, - "nixos-generators": { - "inputs": { - "nixlib": "nixlib", - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1718025593, - "narHash": "sha256-WZ1gdKq/9u1Ns/oXuNsDm+W0salonVA0VY1amw8urJ4=", - "owner": "nix-community", - "repo": "nixos-generators", - "rev": "35c20ba421dfa5059e20e0ef2343c875372bdcf3", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nixos-generators", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1718086528, - "narHash": "sha256-hoB7B7oPgypePz16cKWawPfhVvMSXj4G/qLsfFuhFjw=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "47b604b07d1e8146d5398b42d3306fdebd343986", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-24.05", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-systemd256": { - "locked": { - "lastModified": 1718397913, - "narHash": "sha256-omV+dq3GdXQQTaewxhkBgxM4Bbwqa4D9FVS4dTITxOQ=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "962cf03fb8c782c5e00f465397e03dc84284acc9", - "type": "github" - }, - "original": { - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "962cf03fb8c782c5e00f465397e03dc84284acc9", - "type": "github" - } - }, - "root": { - "inputs": { - "nix-snapshotter": "nix-snapshotter", - "nixos-generators": "nixos-generators", - "nixpkgs": "nixpkgs", - "nixpkgs-systemd256": "nixpkgs-systemd256" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/nix/os/containers/mycelium/flake.nix b/nix/os/containers/mycelium/flake.nix deleted file mode 100644 index 6f247a1..0000000 --- a/nix/os/containers/mycelium/flake.nix +++ /dev/null @@ -1,340 +0,0 @@ -{ - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; - nixpkgs-systemd256.url = "github:NixOS/nixpkgs/962cf03fb8c782c5e00f465397e03dc84284acc9"; - nixos-generators = { - url = "github:nix-community/nixos-generators"; - inputs.nixpkgs.follows = "nixpkgs"; - }; - nix-snapshotter = { - url = "github:pdtpartners/nix-snapshotter"; - inputs.nixpkgs.follows = "nixpkgs"; - }; - }; - outputs = { - self, - nixpkgs, - nixos-generators, - ... - }: let - systems = [ - "aarch64-linux" - "x86_64-linux" - ]; - forAllSystems = nixpkgs.lib.genAttrs systems; - in { - nixosConfigurations.default = - nixpkgs.lib.nixosSystem - { - system = "aarch64-linux"; - - specialArgs = {}; - - modules = [ - ({ - config, - modulesPath, - pkgs, - lib, - ... - }: { - nixpkgs.overlays = [ - (final: previous: { - # inherit (self.inputs.nixpkgs-systemd256.legacyPackages.${pkgs.system}) systemd systemdMinimal; - # systemd = - # self.inputs.nixpkgs-systemd256.legacyPackages.${pkgs.system}.systemd.overrideAttrs (prevAttrs: { - # src = /home/steveej/src/others/systemd; - - # withAppArmor = false; - # withRepart = false; - # withHomed = false; - # withAcl = false; - # withEfi = false; - # withBootloader = false; - # withCryptsetup = false; - # withLibBPF = false; - # withOomd = false; - # withFido2 = false; - # withApparmor = false; - # withDocumentation = false; - # withUtmp = false; - # withQrencode = false; - # withVmspawn = false; - # withMachined = false; - # withLogTrace = true; - # withArchive = false; - # # don't need these but cause errors for exampel files not found - # # withLogind = false; - # }) - # pkgs.systemdMinimal.override { - # # getting errors with these disabled - # withCoredump = true; - # withCompression = true; - # withLogind = true; - # withSysusers = true; - # withUserDb = true; - # } - # pkgs.systemdMinimal - # pkgs.systemd.override { - # withRepart = false; - # withHomed = false; - # withAcl = false; - # withEfi = false; - # withBootloader = false; - # withCryptsetup = false; - # withLibBPF = false; - # withOomd = false; - # withFido2 = false; - # withApparmor = false; - # withDocumentation = false; - # withUtmp = false; - # withQrencode = false; - # withVmspawn = false; - # withMachined = false; - # withLogTrace = true; - # # don't need these but cause errors for exampel files not found - # # withLogind = false; - # } - # ; - }) - ]; - - imports = [ - (modulesPath + "/profiles/minimal.nix") - ]; - system.stateVersion = "24.05"; - - # https://github.com/hercules-ci/arion/blob/c24c185e67f093298a081900b49ca18716077dec/src/nix/modules/nixos/container-systemd.nix - boot.isContainer = true; - # boot.tmp.useTmpfs = true; - boot.loader.grub.enable = lib.mkForce false; - boot.loader.systemd-boot.enable = lib.mkForce false; - services.journald.console = "/dev/console"; - services.journald.storage = "none"; - # boot.specialFileSystems = lib.mkForce {}; - - services.nscd.enable = false; - system.nssModules = lib.mkForce []; - systemd.services.systemd-logind.enable = false; - systemd.services.console-getty.enable = true; - - systemd.sockets.nix-daemon.enable = false; - systemd.services.nix-daemon.enable = false; - systemd.oomd.enable = false; - networking.useDHCP = false; - networking.firewall.enable = false; - boot.postBootCommands = '' - ls -lha /run - mkdir -p /run/wrappers - ''; - # services.udev.enable = false; - - # TODO: this is only needed because `/run/current-system` is missing - # environment.variables.PATH = "${lib.makeBinPath config.environment.systemPackages}:$PATH"; - - services.mycelium.enable = false; - services.mycelium.keyFile = "/var/lib/secrets/mycelium-keyfile"; - systemd.services.mycelium.serviceConfig.DynamicUser = lib.mkForce false; - systemd.services.mycelium.serviceConfig.User = lib.mkForce "root"; - systemd.services.mycelium.serviceConfig.ExecStart = lib.mkForce (pkgs.writeShellScript "mycelium" '' - while true; do - ls -lha $CREDENTIALS_DIRECTORY - sleep 5 - done - ''); - - systemd.services.testing-credentials = { - description = "testing credentials"; - wantedBy = ["multi-user.target"]; - path = [pkgs.coreutils]; - - serviceConfig = { - SyslogIdentifier = "testing-credential"; - StateDirectory = "testing-credentials"; - # DynamicUser = true; - # User = "tc"; - # ProtectHome = true; - # ProtectSystem = true; - LoadCredential = [ - "mycelium-keyfile:${self.nixosConfigurations.default.config.services.mycelium.keyFile}" - "hosts:/etc/hosts" - ]; - SetCredential = "nosecret:not secret string"; - ExecStart = lib.mkForce (pkgs.writeShellScript "mycelium" '' - cd $STATE_DIRECTORY - env - while true; do - ls -lha $CREDENTIALS_DIRECTORY - sleep 5 - done - ''); - }; - }; - - services.caddy = { - enable = true; - globalConfig = '' - auto_https off - ''; - virtualHosts.":80" = { - extraConfig = '' - respond "hello from ${config.networking.hostName}" - ''; - }; - }; - }) - ]; - }; - packages = forAllSystems (system: let - name = "mycelium"; - inherit (self.inputs) nix-snapshotter; - - config = { - entrypoint = "${self.nixosConfigurations.default.config.system.build.toplevel}/init"; - # port = 2379; - args = [ - ]; - # nodePort = 30001; - }; - - myceliumPorts = { - tcp = [9651]; - udp = [9650 9651]; - }; - - inherit - (config) - entrypoint - # port - - args - # nodePort - - ; - - pkgs = import nixpkgs { - overlays = [nix-snapshotter.overlays.default]; - }; - - image = pkgs.nix-snapshotter.buildImage { - inherit name; - resolvedByNix = true; - config = { - entrypoint = [entrypoint]; - env = [ - # this is read by the `/init` script and prevents various incompatible commands like mount, etc. - # the value of this doesn't seem to matter as long as it's not an empty string. - "container=nerd" - ]; - volumes = { - # "/var/lib/private/mycelium/key.bin" = {}; - }; - copyToRoot = [ - # self.nixosConfigurations.default.config.system.build.toplevel - ]; - }; - }; - in { - k8s = let - pod = pkgs.writeText "${name}-pod.json" (builtins.toJSON { - apiVersion = "v1"; - kind = "Pod"; - metadata = { - inherit name; - labels = {inherit name;}; - }; - spec.containers = [ - { - inherit name args; - image = "nix:0${image}"; - ports = [ - { - name = "mycelium-tcp-0"; - containerPort = builtins.elemAt myceliumPorts.tcp 0; - } - { - name = "mycelium-udp-0"; - protocol = "UDP"; - containerPort = builtins.elemAt myceliumPorts.udp 0; - } - { - name = "mycelium-udp-1"; - protocol = "UDP"; - containerPort = builtins.elemAt myceliumPorts.udp 1; - } - ]; - } - ]; - }); - - service = pkgs.writeText "${name}-service.json" (builtins.toJSON { - apiVersion = "v1"; - kind = "Service"; - metadata.name = "${name}-service"; - spec = { - type = "NodePort"; - selector = {inherit name;}; - ports = [ - { - name = "mycelium-tcp-0"; - port = builtins.elemAt myceliumPorts.tcp 0 + 50000; - targetPort = "mycelium-tcp-0"; - } - { - name = "mycelium-udp-0"; - protocol = "UDP"; - port = builtins.elemAt myceliumPorts.udp 0 + 50000; - targetPort = "mycelium-udp-0"; - } - { - name = "mycelium-udp-1"; - protocol = "UDP"; - port = builtins.elemAt myceliumPorts.udp 1 + 50000; - targetPort = "mycelium-udp-1"; - } - ]; - }; - }); - in - pkgs.runCommand "declarative-k8s" {} '' - mkdir -p $out/share/k8s - cp ${pod} $out/share/k8s/ - cp ${service} $out/share/k8s/ - ''; - - inherit image; - - start = pkgs.writeShellApplication { - name = "start"; - text = '' - set -x - rm -rf ./result - nix build --impure .#image - sudo nix2container load ./result - sudo -E nerdctl run --name ${name} --privileged -dt \ - --volume "$PWD/key.bin.crypt:${self.nixosConfigurations.default.config.services.mycelium.keyFile}:ro" \ - "nix:0$(readlink result):latest" - ''; - }; - - stop = pkgs.writeShellApplication { - name = "stop"; - text = '' - set +e - sudo -E nerdctl stop -t 60 ${name} - sudo -E nerdctl rm --force ${name} - sudo -E nerdctl system prune --all --force - sudo systemctl stop nix-snapshotter - sudo systemctl stop containerd - mount | rg -No '(/var/lib/container|/tmp/initial)[^ ]+' | tac | xargs sudo umount -l - sudo systemctl start containerd - sudo systemctl start nix-snapshotter - ''; - - # tmpfs on /run/credentials/mycelium.service type tmpfs (ro,nosuid,nodev,noexec,relatime,nosymfollow,size=1024k,nr_inodes=1024,mode=700,noswap) - - # mount -t tmpfs tmpfs /run/credentials/mycelium.service -o ro,nosuid,nodev,noexec,relatime,nosymfollow,size=1024k,nr_inodes=1024,mode=700,noswap - }; - }); - }; -} diff --git a/nix/os/devices/steveej-x13s/configuration.nix b/nix/os/devices/steveej-x13s/configuration.nix index 8fd0122..41a9b8c 100644 --- a/nix/os/devices/steveej-x13s/configuration.nix +++ b/nix/os/devices/steveej-x13s/configuration.nix @@ -10,12 +10,6 @@ packages', ... }: { - nixpkgs.overlays = [ - (final: previous: { - # inherit (nodeFlake.inputs.nixpkgs-systemd256.legacyPackages.${system}) systemd systemdMinimal; - }) - ]; - nixos-x13s = { enable = true; # TODO: use hardware address @@ -93,8 +87,6 @@ sops.secrets.builder-private-key = {}; nix.distributedBuilds = true; nix.buildMachines = [ - # test these with: sudo nix store ping --store 'ssh-ng://nix-remote-builder@sj-bm-hostkey0.dev.infra.holochain.org?ssh-key=/run/secrets/builder-private-key' - { hostName = "sj-bm-hostkey0.dev.infra.holochain.org"; sshUser = "nix-remote-builder"; @@ -156,8 +148,6 @@ ../../snippets/radicale.nix ../../snippets/holo-zerotier.nix - - ../../snippets/k3s-w-nix-snapshotter.nix ]; networking.hostName = nodeName; @@ -220,10 +210,10 @@ # android on linux virtualisation.waydroid.enable = false; - hardware.ledger.enable = true; - - virtualisation.containers.enable = true; virtualisation.podman.enable = true; + virtualisation.podman.dockerCompat = true; + + hardware.ledger.enable = true; nix.settings.substituters = [ "https://nixos-x13s.cachix.org" diff --git a/nix/os/devices/steveej-x13s/flake.lock b/nix/os/devices/steveej-x13s/flake.lock index 9633bbc..724c6e9 100644 --- a/nix/os/devices/steveej-x13s/flake.lock +++ b/nix/os/devices/steveej-x13s/flake.lock @@ -54,44 +54,7 @@ "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz" } }, - "flake-compat_2": { - "flake": false, - "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, "flake-parts": { - "inputs": { - "nixpkgs-lib": [ - "nix-snapshotter", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1704152458, - "narHash": "sha256-DS+dGw7SKygIWf9w4eNBUZsK+4Ug27NwEWmn2tnbycg=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "88a2cd8166694ba0b6cb374700799cec53aef527", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" - } - }, - "flake-parts_2": { "inputs": { "nixpkgs-lib": "nixpkgs-lib" }, @@ -199,31 +162,9 @@ "type": "github" } }, - "nix-snapshotter": { - "inputs": { - "flake-compat": "flake-compat_2", - "flake-parts": "flake-parts", - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1717948701, - "narHash": "sha256-G7SXaZ7J4yO4OQEKSZPVWcccfV87uyLech0jEOU350g=", - "owner": "yu-re-ka", - "repo": "nix-snapshotter", - "rev": "c10b066a4b1bb3451507c141636014e3335e579e", - "type": "github" - }, - "original": { - "owner": "yu-re-ka", - "repo": "nix-snapshotter", - "type": "github" - } - }, "nixos-x13s": { "inputs": { - "flake-parts": "flake-parts_2", + "flake-parts": "flake-parts", "nixpkgs": "nixpkgs_2" }, "locked": { @@ -322,7 +263,6 @@ "get-flake": "get-flake", "home-manager": "home-manager", "mycelium": "mycelium", - "nix-snapshotter": "nix-snapshotter", "nixos-x13s": "nixos-x13s", "nixpkgs": "nixpkgs_3", "nixpkgs-unstable": "nixpkgs-unstable" diff --git a/nix/os/devices/steveej-x13s/flake.nix b/nix/os/devices/steveej-x13s/flake.nix index 6b8ed7e..110c2ae 100644 --- a/nix/os/devices/steveej-x13s/flake.nix +++ b/nix/os/devices/steveej-x13s/flake.nix @@ -20,12 +20,6 @@ # nixos-x13s.inputs.nixpkgs.follows = "nixpkgs"; mycelium.url = "github:threefoldtech/mycelium"; - - nix-snapshotter = { - url = "github:yu-re-ka/nix-snapshotter"; - # url = "github:pdtpartners/nix-snapshotter"; - inputs.nixpkgs.follows = "nixpkgs"; - }; }; outputs = { diff --git a/nix/os/snippets/k3s-w-nix-snapshotter.nix b/nix/os/snippets/k3s-w-nix-snapshotter.nix deleted file mode 100644 index 0243018..0000000 --- a/nix/os/snippets/k3s-w-nix-snapshotter.nix +++ /dev/null @@ -1,41 +0,0 @@ -# experiment with k3s, nix-snapshotter, and nixos images -{ - nodeFlake, - nodeFlakeInputs', - pkgs, - lib, - system, - ... -}: { - # (1) Import nixos module. - imports = [ - nodeFlake.inputs.nix-snapshotter.nixosModules.default - ]; - - # (2) Add overlay. - nixpkgs.overlays = [nodeFlake.inputs.nix-snapshotter.overlays.default]; - - # (3) Enable service. - virtualisation.containerd = { - enable = true; - k3sIntegration = false; - nixSnapshotterIntegration = true; - - # TODO: understand if this has an influence on the systemd LoadCredential issue - settings.plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options.SystemdCgroup = lib.mkForce true; - }; - services.nix-snapshotter = { - enable = true; - }; - - # (4) Add a containerd CLI like nerdctl. - environment.systemPackages = [ - pkgs.nerdctl - nodeFlake.inputs.nix-snapshotter.packages.${system}.default - ]; - - services.k3s = { - enable = false; - setKubeConfig = true; - }; -}