diff --git a/nix/os/containers/mycelium/configuration.nix b/nix/os/containers/mycelium/configuration.nix
new file mode 100644
index 0000000..e69de29
diff --git a/nix/os/containers/mycelium/flake.lock b/nix/os/containers/mycelium/flake.lock
new file mode 100644
index 0000000..899ee98
--- /dev/null
+++ b/nix/os/containers/mycelium/flake.lock
@@ -0,0 +1,141 @@
+{
+  "nodes": {
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nix-snapshotter",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1704152458,
+        "narHash": "sha256-DS+dGw7SKygIWf9w4eNBUZsK+4Ug27NwEWmn2tnbycg=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "88a2cd8166694ba0b6cb374700799cec53aef527",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "nix-snapshotter": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "flake-parts": "flake-parts",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1715438114,
+        "narHash": "sha256-btb702TXuhDg0D6tW0dCOy4+II9Wl6BJ0LvpT+O9wrs=",
+        "owner": "pdtpartners",
+        "repo": "nix-snapshotter",
+        "rev": "7b251c9356bc7bb383ebeedcd0045b3ae431bff7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "pdtpartners",
+        "repo": "nix-snapshotter",
+        "type": "github"
+      }
+    },
+    "nixlib": {
+      "locked": {
+        "lastModified": 1712450863,
+        "narHash": "sha256-K6IkdtMtq9xktmYPj0uaYc8NsIqHuaAoRBaMgu9Fvrw=",
+        "owner": "nix-community",
+        "repo": "nixpkgs.lib",
+        "rev": "3c62b6a12571c9a7f65ab037173ee153d539905f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nixpkgs.lib",
+        "type": "github"
+      }
+    },
+    "nixos-generators": {
+      "inputs": {
+        "nixlib": "nixlib",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1718025593,
+        "narHash": "sha256-WZ1gdKq/9u1Ns/oXuNsDm+W0salonVA0VY1amw8urJ4=",
+        "owner": "nix-community",
+        "repo": "nixos-generators",
+        "rev": "35c20ba421dfa5059e20e0ef2343c875372bdcf3",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nixos-generators",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1718086528,
+        "narHash": "sha256-hoB7B7oPgypePz16cKWawPfhVvMSXj4G/qLsfFuhFjw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "47b604b07d1e8146d5398b42d3306fdebd343986",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-24.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-systemd256": {
+      "locked": {
+        "lastModified": 1718397913,
+        "narHash": "sha256-omV+dq3GdXQQTaewxhkBgxM4Bbwqa4D9FVS4dTITxOQ=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "962cf03fb8c782c5e00f465397e03dc84284acc9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "962cf03fb8c782c5e00f465397e03dc84284acc9",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nix-snapshotter": "nix-snapshotter",
+        "nixos-generators": "nixos-generators",
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-systemd256": "nixpkgs-systemd256"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
diff --git a/nix/os/containers/mycelium/flake.nix b/nix/os/containers/mycelium/flake.nix
new file mode 100644
index 0000000..6f247a1
--- /dev/null
+++ b/nix/os/containers/mycelium/flake.nix
@@ -0,0 +1,340 @@
+{
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05";
+    nixpkgs-systemd256.url = "github:NixOS/nixpkgs/962cf03fb8c782c5e00f465397e03dc84284acc9";
+    nixos-generators = {
+      url = "github:nix-community/nixos-generators";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    nix-snapshotter = {
+      url = "github:pdtpartners/nix-snapshotter";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+  };
+  outputs = {
+    self,
+    nixpkgs,
+    nixos-generators,
+    ...
+  }: let
+    systems = [
+      "aarch64-linux"
+      "x86_64-linux"
+    ];
+    forAllSystems = nixpkgs.lib.genAttrs systems;
+  in {
+    nixosConfigurations.default =
+      nixpkgs.lib.nixosSystem
+      {
+        system = "aarch64-linux";
+
+        specialArgs = {};
+
+        modules = [
+          ({
+            config,
+            modulesPath,
+            pkgs,
+            lib,
+            ...
+          }: {
+            nixpkgs.overlays = [
+              (final: previous: {
+                # inherit (self.inputs.nixpkgs-systemd256.legacyPackages.${pkgs.system}) systemd systemdMinimal;
+                # systemd =
+                # self.inputs.nixpkgs-systemd256.legacyPackages.${pkgs.system}.systemd.overrideAttrs (prevAttrs: {
+                #   src = /home/steveej/src/others/systemd;
+
+                #   withAppArmor = false;
+                #   withRepart = false;
+                #   withHomed = false;
+                #   withAcl = false;
+                #   withEfi = false;
+                #   withBootloader = false;
+                #   withCryptsetup = false;
+                #   withLibBPF = false;
+                #   withOomd = false;
+                #   withFido2 = false;
+                #   withApparmor = false;
+                #   withDocumentation = false;
+                #   withUtmp = false;
+                #   withQrencode = false;
+                #   withVmspawn = false;
+                #   withMachined = false;
+                #   withLogTrace = true;
+                #   withArchive = false;
+                #   # don't need these but cause errors for exampel files not found
+                #   # withLogind = false;
+                # })
+                # pkgs.systemdMinimal.override {
+                #   # getting errors with these disabled
+                #   withCoredump = true;
+                #   withCompression = true;
+                #   withLogind = true;
+                #   withSysusers = true;
+                #   withUserDb = true;
+                # }
+                # pkgs.systemdMinimal
+                # pkgs.systemd.override {
+                #   withRepart = false;
+                #   withHomed = false;
+                #   withAcl = false;
+                #   withEfi = false;
+                #   withBootloader = false;
+                #   withCryptsetup = false;
+                #   withLibBPF = false;
+                #   withOomd = false;
+                #   withFido2 = false;
+                #   withApparmor = false;
+                #   withDocumentation = false;
+                #   withUtmp = false;
+                #   withQrencode = false;
+                #   withVmspawn = false;
+                #   withMachined = false;
+                #   withLogTrace = true;
+                #   # don't need these but cause errors for exampel files not found
+                #   # withLogind = false;
+                # }
+                # ;
+              })
+            ];
+
+            imports = [
+              (modulesPath + "/profiles/minimal.nix")
+            ];
+            system.stateVersion = "24.05";
+
+            # https://github.com/hercules-ci/arion/blob/c24c185e67f093298a081900b49ca18716077dec/src/nix/modules/nixos/container-systemd.nix
+            boot.isContainer = true;
+            # boot.tmp.useTmpfs = true;
+            boot.loader.grub.enable = lib.mkForce false;
+            boot.loader.systemd-boot.enable = lib.mkForce false;
+            services.journald.console = "/dev/console";
+            services.journald.storage = "none";
+            # boot.specialFileSystems = lib.mkForce {};
+
+            services.nscd.enable = false;
+            system.nssModules = lib.mkForce [];
+            systemd.services.systemd-logind.enable = false;
+            systemd.services.console-getty.enable = true;
+
+            systemd.sockets.nix-daemon.enable = false;
+            systemd.services.nix-daemon.enable = false;
+            systemd.oomd.enable = false;
+            networking.useDHCP = false;
+            networking.firewall.enable = false;
+            boot.postBootCommands = ''
+              ls -lha /run
+              mkdir -p /run/wrappers
+            '';
+            # services.udev.enable = false;
+
+            # TODO: this is only needed because `/run/current-system` is missing
+            # environment.variables.PATH = "${lib.makeBinPath config.environment.systemPackages}:$PATH";
+
+            services.mycelium.enable = false;
+            services.mycelium.keyFile = "/var/lib/secrets/mycelium-keyfile";
+            systemd.services.mycelium.serviceConfig.DynamicUser = lib.mkForce false;
+            systemd.services.mycelium.serviceConfig.User = lib.mkForce "root";
+            systemd.services.mycelium.serviceConfig.ExecStart = lib.mkForce (pkgs.writeShellScript "mycelium" ''
+              while true; do
+                ls -lha $CREDENTIALS_DIRECTORY
+                sleep 5
+              done
+            '');
+
+            systemd.services.testing-credentials = {
+              description = "testing credentials";
+              wantedBy = ["multi-user.target"];
+              path = [pkgs.coreutils];
+
+              serviceConfig = {
+                SyslogIdentifier = "testing-credential";
+                StateDirectory = "testing-credentials";
+                # DynamicUser = true;
+                # User = "tc";
+                # ProtectHome = true;
+                # ProtectSystem = true;
+                LoadCredential = [
+                  "mycelium-keyfile:${self.nixosConfigurations.default.config.services.mycelium.keyFile}"
+                  "hosts:/etc/hosts"
+                ];
+                SetCredential = "nosecret:not secret string";
+                ExecStart = lib.mkForce (pkgs.writeShellScript "mycelium" ''
+                  cd $STATE_DIRECTORY
+                  env
+                  while true; do
+                    ls -lha $CREDENTIALS_DIRECTORY
+                    sleep 5
+                  done
+                '');
+              };
+            };
+
+            services.caddy = {
+              enable = true;
+              globalConfig = ''
+                auto_https off
+              '';
+              virtualHosts.":80" = {
+                extraConfig = ''
+                  respond "hello from ${config.networking.hostName}"
+                '';
+              };
+            };
+          })
+        ];
+      };
+    packages = forAllSystems (system: let
+      name = "mycelium";
+      inherit (self.inputs) nix-snapshotter;
+
+      config = {
+        entrypoint = "${self.nixosConfigurations.default.config.system.build.toplevel}/init";
+        # port = 2379;
+        args = [
+        ];
+        # nodePort = 30001;
+      };
+
+      myceliumPorts = {
+        tcp = [9651];
+        udp = [9650 9651];
+      };
+
+      inherit
+        (config)
+        entrypoint
+        # port
+        
+        args
+        # nodePort
+        
+        ;
+
+      pkgs = import nixpkgs {
+        overlays = [nix-snapshotter.overlays.default];
+      };
+
+      image = pkgs.nix-snapshotter.buildImage {
+        inherit name;
+        resolvedByNix = true;
+        config = {
+          entrypoint = [entrypoint];
+          env = [
+            # this is read by the `/init` script and prevents various incompatible commands like mount, etc.
+            # the value of this doesn't seem to matter as long as it's not an empty string.
+            "container=nerd"
+          ];
+          volumes = {
+            # "/var/lib/private/mycelium/key.bin" = {};
+          };
+          copyToRoot = [
+            # self.nixosConfigurations.default.config.system.build.toplevel
+          ];
+        };
+      };
+    in {
+      k8s = let
+        pod = pkgs.writeText "${name}-pod.json" (builtins.toJSON {
+          apiVersion = "v1";
+          kind = "Pod";
+          metadata = {
+            inherit name;
+            labels = {inherit name;};
+          };
+          spec.containers = [
+            {
+              inherit name args;
+              image = "nix:0${image}";
+              ports = [
+                {
+                  name = "mycelium-tcp-0";
+                  containerPort = builtins.elemAt myceliumPorts.tcp 0;
+                }
+                {
+                  name = "mycelium-udp-0";
+                  protocol = "UDP";
+                  containerPort = builtins.elemAt myceliumPorts.udp 0;
+                }
+                {
+                  name = "mycelium-udp-1";
+                  protocol = "UDP";
+                  containerPort = builtins.elemAt myceliumPorts.udp 1;
+                }
+              ];
+            }
+          ];
+        });
+
+        service = pkgs.writeText "${name}-service.json" (builtins.toJSON {
+          apiVersion = "v1";
+          kind = "Service";
+          metadata.name = "${name}-service";
+          spec = {
+            type = "NodePort";
+            selector = {inherit name;};
+            ports = [
+              {
+                name = "mycelium-tcp-0";
+                port = builtins.elemAt myceliumPorts.tcp 0 + 50000;
+                targetPort = "mycelium-tcp-0";
+              }
+              {
+                name = "mycelium-udp-0";
+                protocol = "UDP";
+                port = builtins.elemAt myceliumPorts.udp 0 + 50000;
+                targetPort = "mycelium-udp-0";
+              }
+              {
+                name = "mycelium-udp-1";
+                protocol = "UDP";
+                port = builtins.elemAt myceliumPorts.udp 1 + 50000;
+                targetPort = "mycelium-udp-1";
+              }
+            ];
+          };
+        });
+      in
+        pkgs.runCommand "declarative-k8s" {} ''
+          mkdir -p $out/share/k8s
+          cp ${pod} $out/share/k8s/
+          cp ${service} $out/share/k8s/
+        '';
+
+      inherit image;
+
+      start = pkgs.writeShellApplication {
+        name = "start";
+        text = ''
+          set -x
+          rm -rf ./result
+          nix build --impure .#image
+          sudo nix2container load ./result
+          sudo -E nerdctl run  --name ${name} --privileged -dt \
+            --volume "$PWD/key.bin.crypt:${self.nixosConfigurations.default.config.services.mycelium.keyFile}:ro" \
+            "nix:0$(readlink result):latest"
+        '';
+      };
+
+      stop = pkgs.writeShellApplication {
+        name = "stop";
+        text = ''
+          set +e
+          sudo -E nerdctl stop -t 60 ${name}
+          sudo -E nerdctl rm --force ${name}
+          sudo -E nerdctl system prune --all --force
+          sudo systemctl stop nix-snapshotter
+          sudo systemctl stop containerd
+          mount | rg -No '(/var/lib/container|/tmp/initial)[^ ]+' | tac | xargs sudo umount -l
+          sudo systemctl start containerd
+          sudo systemctl start nix-snapshotter
+        '';
+
+        # tmpfs on /run/credentials/mycelium.service type tmpfs (ro,nosuid,nodev,noexec,relatime,nosymfollow,size=1024k,nr_inodes=1024,mode=700,noswap)
+
+        # mount -t tmpfs tmpfs /run/credentials/mycelium.service -o ro,nosuid,nodev,noexec,relatime,nosymfollow,size=1024k,nr_inodes=1024,mode=700,noswap
+      };
+    });
+  };
+}
diff --git a/nix/os/devices/steveej-x13s/configuration.nix b/nix/os/devices/steveej-x13s/configuration.nix
index 41a9b8c..8fd0122 100644
--- a/nix/os/devices/steveej-x13s/configuration.nix
+++ b/nix/os/devices/steveej-x13s/configuration.nix
@@ -10,6 +10,12 @@
   packages',
   ...
 }: {
+  nixpkgs.overlays = [
+    (final: previous: {
+      # inherit (nodeFlake.inputs.nixpkgs-systemd256.legacyPackages.${system}) systemd systemdMinimal;
+    })
+  ];
+
   nixos-x13s = {
     enable = true;
     # TODO: use hardware address
@@ -87,6 +93,8 @@
       sops.secrets.builder-private-key = {};
       nix.distributedBuilds = true;
       nix.buildMachines = [
+        # test these with: sudo nix store ping --store 'ssh-ng://nix-remote-builder@sj-bm-hostkey0.dev.infra.holochain.org?ssh-key=/run/secrets/builder-private-key'
+
         {
           hostName = "sj-bm-hostkey0.dev.infra.holochain.org";
           sshUser = "nix-remote-builder";
@@ -148,6 +156,8 @@
     ../../snippets/radicale.nix
 
     ../../snippets/holo-zerotier.nix
+
+    ../../snippets/k3s-w-nix-snapshotter.nix
   ];
 
   networking.hostName = nodeName;
@@ -210,11 +220,11 @@
 
   # android on linux
   virtualisation.waydroid.enable = false;
-  virtualisation.podman.enable = true;
-  virtualisation.podman.dockerCompat = true;
-
   hardware.ledger.enable = true;
 
+  virtualisation.containers.enable = true;
+  virtualisation.podman.enable = true;
+
   nix.settings.substituters = [
     "https://nixos-x13s.cachix.org"
   ];
diff --git a/nix/os/devices/steveej-x13s/flake.lock b/nix/os/devices/steveej-x13s/flake.lock
index 724c6e9..9633bbc 100644
--- a/nix/os/devices/steveej-x13s/flake.lock
+++ b/nix/os/devices/steveej-x13s/flake.lock
@@ -54,7 +54,44 @@
         "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz"
       }
     },
+    "flake-compat_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
     "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nix-snapshotter",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1704152458,
+        "narHash": "sha256-DS+dGw7SKygIWf9w4eNBUZsK+4Ug27NwEWmn2tnbycg=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "88a2cd8166694ba0b6cb374700799cec53aef527",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_2": {
       "inputs": {
         "nixpkgs-lib": "nixpkgs-lib"
       },
@@ -162,9 +199,31 @@
         "type": "github"
       }
     },
+    "nix-snapshotter": {
+      "inputs": {
+        "flake-compat": "flake-compat_2",
+        "flake-parts": "flake-parts",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1717948701,
+        "narHash": "sha256-G7SXaZ7J4yO4OQEKSZPVWcccfV87uyLech0jEOU350g=",
+        "owner": "yu-re-ka",
+        "repo": "nix-snapshotter",
+        "rev": "c10b066a4b1bb3451507c141636014e3335e579e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "yu-re-ka",
+        "repo": "nix-snapshotter",
+        "type": "github"
+      }
+    },
     "nixos-x13s": {
       "inputs": {
-        "flake-parts": "flake-parts",
+        "flake-parts": "flake-parts_2",
         "nixpkgs": "nixpkgs_2"
       },
       "locked": {
@@ -263,6 +322,7 @@
         "get-flake": "get-flake",
         "home-manager": "home-manager",
         "mycelium": "mycelium",
+        "nix-snapshotter": "nix-snapshotter",
         "nixos-x13s": "nixos-x13s",
         "nixpkgs": "nixpkgs_3",
         "nixpkgs-unstable": "nixpkgs-unstable"
diff --git a/nix/os/devices/steveej-x13s/flake.nix b/nix/os/devices/steveej-x13s/flake.nix
index 110c2ae..6b8ed7e 100644
--- a/nix/os/devices/steveej-x13s/flake.nix
+++ b/nix/os/devices/steveej-x13s/flake.nix
@@ -20,6 +20,12 @@
     # nixos-x13s.inputs.nixpkgs.follows = "nixpkgs";
 
     mycelium.url = "github:threefoldtech/mycelium";
+
+    nix-snapshotter = {
+      url = "github:yu-re-ka/nix-snapshotter";
+      # url = "github:pdtpartners/nix-snapshotter";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
   outputs = {
diff --git a/nix/os/snippets/k3s-w-nix-snapshotter.nix b/nix/os/snippets/k3s-w-nix-snapshotter.nix
new file mode 100644
index 0000000..0243018
--- /dev/null
+++ b/nix/os/snippets/k3s-w-nix-snapshotter.nix
@@ -0,0 +1,41 @@
+# experiment with k3s, nix-snapshotter, and nixos images
+{
+  nodeFlake,
+  nodeFlakeInputs',
+  pkgs,
+  lib,
+  system,
+  ...
+}: {
+  # (1) Import nixos module.
+  imports = [
+    nodeFlake.inputs.nix-snapshotter.nixosModules.default
+  ];
+
+  # (2) Add overlay.
+  nixpkgs.overlays = [nodeFlake.inputs.nix-snapshotter.overlays.default];
+
+  # (3) Enable service.
+  virtualisation.containerd = {
+    enable = true;
+    k3sIntegration = false;
+    nixSnapshotterIntegration = true;
+
+    # TODO: understand if this has an influence on the systemd LoadCredential issue
+    settings.plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options.SystemdCgroup = lib.mkForce true;
+  };
+  services.nix-snapshotter = {
+    enable = true;
+  };
+
+  # (4) Add a containerd CLI like nerdctl.
+  environment.systemPackages = [
+    pkgs.nerdctl
+    nodeFlake.inputs.nix-snapshotter.packages.${system}.default
+  ];
+
+  services.k3s = {
+    enable = false;
+    setKubeConfig = true;
+  };
+}