{ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small"; # nixpkgs-systemd256.url = "github:NixOS/nixpkgs/962cf03fb8c782c5e00f465397e03dc84284acc9"; nixos-generators = { url = "github:nix-community/nixos-generators"; inputs.nixpkgs.follows = "nixpkgs"; }; nix-snapshotter = { url = "github:pdtpartners/nix-snapshotter"; inputs.nixpkgs.follows = "nixpkgs"; }; }; outputs = { self, nixpkgs, nixos-generators, ... }: let systems = [ "aarch64-linux" "x86_64-linux" ]; forAllSystems = nixpkgs.lib.genAttrs systems; in { nixosConfigurations.default = nixpkgs.lib.nixosSystem { system = "aarch64-linux"; specialArgs = {}; modules = [ ({ config, modulesPath, pkgs, lib, ... }: { nixpkgs.overlays = [ (final: previous: { # inherit (self.inputs.nixpkgs-systemd256.legacyPackages.${pkgs.system}) systemd systemdMinimal; # systemd = # self.inputs.nixpkgs-systemd256.legacyPackages.${pkgs.system}.systemd.overrideAttrs (prevAttrs: { # src = /home/steveej/src/others/systemd; # withAppArmor = false; # withRepart = false; # withHomed = false; # withAcl = false; # withEfi = false; # withBootloader = false; # withCryptsetup = false; # withLibBPF = false; # withOomd = false; # withFido2 = false; # withApparmor = false; # withDocumentation = false; # withUtmp = false; # withQrencode = false; # withVmspawn = false; # withMachined = false; # withLogTrace = true; # withArchive = false; # # don't need these but cause errors for exampel files not found # # withLogind = false; # }) # pkgs.systemdMinimal.override { # # getting errors with these disabled # withCoredump = true; # withCompression = true; # withLogind = true; # withSysusers = true; # withUserDb = true; # } # pkgs.systemdMinimal # pkgs.systemd.override { # withRepart = false; # withHomed = false; # withAcl = false; # withEfi = false; # withBootloader = false; # withCryptsetup = false; # withLibBPF = false; # withOomd = false; # withFido2 = false; # withApparmor = false; # withDocumentation = false; # withUtmp = false; # withQrencode = false; # withVmspawn = false; # withMachined = false; # withLogTrace = true; # # don't need these but cause errors for exampel files not found # # withLogind = false; # } # ; }) ]; imports = [ (modulesPath + "/profiles/minimal.nix") ]; system.stateVersion = "24.11"; # https://github.com/hercules-ci/arion/blob/c24c185e67f093298a081900b49ca18716077dec/src/nix/modules/nixos/container-systemd.nix boot.isContainer = true; # boot.tmp.useTmpfs = true; boot.loader.grub.enable = lib.mkForce false; boot.loader.systemd-boot.enable = lib.mkForce false; services.journald.console = "/dev/console"; services.journald.storage = "none"; # boot.specialFileSystems = lib.mkForce {}; services.nscd.enable = false; system.nssModules = lib.mkForce []; systemd.services.systemd-logind.enable = false; systemd.services.console-getty.enable = false; systemd.sockets.nix-daemon.enable = false; systemd.services.nix-daemon.enable = false; systemd.oomd.enable = false; networking.useDHCP = false; networking.firewall.enable = false; # system.build.earlyMountScript = # lib.mkForce '' # ''; # system.activationScripts.specialfs = # lib.mkForce '' # ''; boot.postBootCommands = '' ls -lha /run mkdir -p /run/wrappers ''; boot.kernelParams = [ "systemd.log_level=debug" ]; # services.udev.enable = false; # TODO: this is only needed because `/run/current-system` is missing # environment.variables.PATH = "${lib.makeBinPath config.environment.systemPackages}:$PATH"; systemd.mounts = lib.mkForce []; fileSystems = lib.mkForce {}; services.mycelium.enable = false; services.mycelium.keyFile = "/var/lib/secrets/mycelium-keyfile"; systemd.services.mycelium.serviceConfig.DynamicUser = lib.mkForce false; systemd.services.mycelium.serviceConfig.User = lib.mkForce "root"; systemd.services.mycelium.serviceConfig.ExecStart = lib.mkForce (pkgs.writeShellScript "mycelium" '' while true; do ls -lha $CREDENTIALS_DIRECTORY sleep 5 done ''); systemd.services.testing-credentials = { wantedBy = ["multi-user.target"]; path = [pkgs.coreutils]; serviceConfig = { # SyslogIdentifier = "testing-credentials"; # StateDirectory = "testing-credentials"; # DynamicUser = true; # User = "tc"; # ProtectHome = true; # ProtectSystem = true; # LoadCredential = [ # "mycelium-keyfile:${self.nixosConfigurations.default.config.services.mycelium.keyFile}" # "hosts:/etc/hosts" # ]; SetCredential = "mycelium-keyfile:not secret string"; ExecStart = lib.mkForce (pkgs.writeShellScript "mycelium" '' cd $STATE_DIRECTORY pwd env while true; do ls -lha $CREDENTIALS_DIRECTORY sleep 5 done ''); }; }; services.caddy = { enable = true; globalConfig = '' auto_https off ''; virtualHosts.":80" = { extraConfig = '' respond "hello from ${config.networking.hostName}" ''; }; }; }) ]; }; packages = forAllSystems (system: let name = "mycelium"; inherit (self.inputs) nix-snapshotter; config = { entrypoint = "${self.nixosConfigurations.default.config.system.build.toplevel}/init"; # port = 2379; args = [ ]; # nodePort = 30001; }; myceliumPorts = { tcp = [9651]; udp = [9650 9651]; }; inherit (config) entrypoint # port args # nodePort ; pkgs = import nixpkgs { overlays = [nix-snapshotter.overlays.default]; }; image = pkgs.nix-snapshotter.buildImage { inherit name; resolvedByNix = true; config = { entrypoint = [entrypoint]; env = [ # this is read by the `/init` script and prevents various incompatible commands like mount, etc. # the value of this doesn't seem to matter as long as it's not an empty string. "container=nerd" "SYSTEMD_LOG_LEVEL=debug" ]; volumes = { # "/var/lib/private/mycelium/key.bin" = {}; # "/run" = {}; # "/tmp" = {}; # "/etc" = {}; }; copyToRoot = [ # self.nixosConfigurations.default.config.system.build.toplevel ]; }; }; in { k8s = let pod = pkgs.writeText "${name}-pod.json" (builtins.toJSON { apiVersion = "v1"; kind = "Pod"; metadata = { inherit name; labels = {inherit name;}; }; spec.containers = [ { inherit name args; image = "nix:0${image}"; ports = [ { name = "mycelium-tcp-0"; containerPort = builtins.elemAt myceliumPorts.tcp 0; } { name = "mycelium-udp-0"; protocol = "UDP"; containerPort = builtins.elemAt myceliumPorts.udp 0; } { name = "mycelium-udp-1"; protocol = "UDP"; containerPort = builtins.elemAt myceliumPorts.udp 1; } ]; } ]; }); service = pkgs.writeText "${name}-service.json" (builtins.toJSON { apiVersion = "v1"; kind = "Service"; metadata.name = "${name}-service"; spec = { type = "NodePort"; selector = {inherit name;}; ports = [ { name = "mycelium-tcp-0"; port = builtins.elemAt myceliumPorts.tcp 0 + 50000; targetPort = "mycelium-tcp-0"; } { name = "mycelium-udp-0"; protocol = "UDP"; port = builtins.elemAt myceliumPorts.udp 0 + 50000; targetPort = "mycelium-udp-0"; } { name = "mycelium-udp-1"; protocol = "UDP"; port = builtins.elemAt myceliumPorts.udp 1 + 50000; targetPort = "mycelium-udp-1"; } ]; }; }); in pkgs.runCommand "declarative-k8s" {} '' mkdir -p $out/share/k8s cp ${pod} $out/share/k8s/ cp ${service} $out/share/k8s/ ''; inherit image; start = pkgs.writeShellApplication { name = "start"; text = '' set -x rm -rf ./result nix build --impure .#image sudo nix2container load ./result sudo -E nerdctl run --name ${name} --privileged -dt \ --cgroup-manager cgroupfs \ --volume "$PWD/key.bin.crypt:${self.nixosConfigurations.default.config.services.mycelium.keyFile}:ro" \ "nix:0$(readlink result):latest" ''; }; stop = pkgs.writeShellApplication { name = "stop"; text = '' set +e sudo -E nerdctl stop -t 60 ${name} sudo -E nerdctl rm --force ${name} sudo -E nerdctl system prune --all --force sudo systemctl stop nix-snapshotter sudo systemctl stop containerd mount | rg -No '(/var/lib/container|/tmp/initial)[^ ]+' | tac | xargs sudo umount -l sudo systemctl start containerd sudo systemctl start nix-snapshotter ''; # tmpfs on /run/credentials/mycelium.service type tmpfs (ro,nosuid,nodev,noexec,relatime,nosymfollow,size=1024k,nr_inodes=1024,mode=700,noswap) # mount -t tmpfs tmpfs /run/credentials/mycelium.service -o ro,nosuid,nodev,noexec,relatime,nosymfollow,size=1024k,nr_inodes=1024,mode=700,noswap }; }); }; }