340 lines
11 KiB
Nix
340 lines
11 KiB
Nix
{
|
|
inputs = {
|
|
nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05";
|
|
nixpkgs-systemd256.url = "github:NixOS/nixpkgs/962cf03fb8c782c5e00f465397e03dc84284acc9";
|
|
nixos-generators = {
|
|
url = "github:nix-community/nixos-generators";
|
|
inputs.nixpkgs.follows = "nixpkgs";
|
|
};
|
|
nix-snapshotter = {
|
|
url = "github:pdtpartners/nix-snapshotter";
|
|
inputs.nixpkgs.follows = "nixpkgs";
|
|
};
|
|
};
|
|
outputs = {
|
|
self,
|
|
nixpkgs,
|
|
nixos-generators,
|
|
...
|
|
}: let
|
|
systems = [
|
|
"aarch64-linux"
|
|
"x86_64-linux"
|
|
];
|
|
forAllSystems = nixpkgs.lib.genAttrs systems;
|
|
in {
|
|
nixosConfigurations.default =
|
|
nixpkgs.lib.nixosSystem
|
|
{
|
|
system = "aarch64-linux";
|
|
|
|
specialArgs = {};
|
|
|
|
modules = [
|
|
({
|
|
config,
|
|
modulesPath,
|
|
pkgs,
|
|
lib,
|
|
...
|
|
}: {
|
|
nixpkgs.overlays = [
|
|
(final: previous: {
|
|
# inherit (self.inputs.nixpkgs-systemd256.legacyPackages.${pkgs.system}) systemd systemdMinimal;
|
|
# systemd =
|
|
# self.inputs.nixpkgs-systemd256.legacyPackages.${pkgs.system}.systemd.overrideAttrs (prevAttrs: {
|
|
# src = /home/steveej/src/others/systemd;
|
|
|
|
# withAppArmor = false;
|
|
# withRepart = false;
|
|
# withHomed = false;
|
|
# withAcl = false;
|
|
# withEfi = false;
|
|
# withBootloader = false;
|
|
# withCryptsetup = false;
|
|
# withLibBPF = false;
|
|
# withOomd = false;
|
|
# withFido2 = false;
|
|
# withApparmor = false;
|
|
# withDocumentation = false;
|
|
# withUtmp = false;
|
|
# withQrencode = false;
|
|
# withVmspawn = false;
|
|
# withMachined = false;
|
|
# withLogTrace = true;
|
|
# withArchive = false;
|
|
# # don't need these but cause errors for exampel files not found
|
|
# # withLogind = false;
|
|
# })
|
|
# pkgs.systemdMinimal.override {
|
|
# # getting errors with these disabled
|
|
# withCoredump = true;
|
|
# withCompression = true;
|
|
# withLogind = true;
|
|
# withSysusers = true;
|
|
# withUserDb = true;
|
|
# }
|
|
# pkgs.systemdMinimal
|
|
# pkgs.systemd.override {
|
|
# withRepart = false;
|
|
# withHomed = false;
|
|
# withAcl = false;
|
|
# withEfi = false;
|
|
# withBootloader = false;
|
|
# withCryptsetup = false;
|
|
# withLibBPF = false;
|
|
# withOomd = false;
|
|
# withFido2 = false;
|
|
# withApparmor = false;
|
|
# withDocumentation = false;
|
|
# withUtmp = false;
|
|
# withQrencode = false;
|
|
# withVmspawn = false;
|
|
# withMachined = false;
|
|
# withLogTrace = true;
|
|
# # don't need these but cause errors for exampel files not found
|
|
# # withLogind = false;
|
|
# }
|
|
# ;
|
|
})
|
|
];
|
|
|
|
imports = [
|
|
(modulesPath + "/profiles/minimal.nix")
|
|
];
|
|
system.stateVersion = "24.05";
|
|
|
|
# https://github.com/hercules-ci/arion/blob/c24c185e67f093298a081900b49ca18716077dec/src/nix/modules/nixos/container-systemd.nix
|
|
boot.isContainer = true;
|
|
# boot.tmp.useTmpfs = true;
|
|
boot.loader.grub.enable = lib.mkForce false;
|
|
boot.loader.systemd-boot.enable = lib.mkForce false;
|
|
services.journald.console = "/dev/console";
|
|
services.journald.storage = "none";
|
|
# boot.specialFileSystems = lib.mkForce {};
|
|
|
|
services.nscd.enable = false;
|
|
system.nssModules = lib.mkForce [];
|
|
systemd.services.systemd-logind.enable = false;
|
|
systemd.services.console-getty.enable = true;
|
|
|
|
systemd.sockets.nix-daemon.enable = false;
|
|
systemd.services.nix-daemon.enable = false;
|
|
systemd.oomd.enable = false;
|
|
networking.useDHCP = false;
|
|
networking.firewall.enable = false;
|
|
boot.postBootCommands = ''
|
|
ls -lha /run
|
|
mkdir -p /run/wrappers
|
|
'';
|
|
# services.udev.enable = false;
|
|
|
|
# TODO: this is only needed because `/run/current-system` is missing
|
|
# environment.variables.PATH = "${lib.makeBinPath config.environment.systemPackages}:$PATH";
|
|
|
|
services.mycelium.enable = false;
|
|
services.mycelium.keyFile = "/var/lib/secrets/mycelium-keyfile";
|
|
systemd.services.mycelium.serviceConfig.DynamicUser = lib.mkForce false;
|
|
systemd.services.mycelium.serviceConfig.User = lib.mkForce "root";
|
|
systemd.services.mycelium.serviceConfig.ExecStart = lib.mkForce (pkgs.writeShellScript "mycelium" ''
|
|
while true; do
|
|
ls -lha $CREDENTIALS_DIRECTORY
|
|
sleep 5
|
|
done
|
|
'');
|
|
|
|
systemd.services.testing-credentials = {
|
|
description = "testing credentials";
|
|
wantedBy = ["multi-user.target"];
|
|
path = [pkgs.coreutils];
|
|
|
|
serviceConfig = {
|
|
SyslogIdentifier = "testing-credential";
|
|
StateDirectory = "testing-credentials";
|
|
# DynamicUser = true;
|
|
# User = "tc";
|
|
# ProtectHome = true;
|
|
# ProtectSystem = true;
|
|
LoadCredential = [
|
|
"mycelium-keyfile:${self.nixosConfigurations.default.config.services.mycelium.keyFile}"
|
|
"hosts:/etc/hosts"
|
|
];
|
|
SetCredential = "nosecret:not secret string";
|
|
ExecStart = lib.mkForce (pkgs.writeShellScript "mycelium" ''
|
|
cd $STATE_DIRECTORY
|
|
env
|
|
while true; do
|
|
ls -lha $CREDENTIALS_DIRECTORY
|
|
sleep 5
|
|
done
|
|
'');
|
|
};
|
|
};
|
|
|
|
services.caddy = {
|
|
enable = true;
|
|
globalConfig = ''
|
|
auto_https off
|
|
'';
|
|
virtualHosts.":80" = {
|
|
extraConfig = ''
|
|
respond "hello from ${config.networking.hostName}"
|
|
'';
|
|
};
|
|
};
|
|
})
|
|
];
|
|
};
|
|
packages = forAllSystems (system: let
|
|
name = "mycelium";
|
|
inherit (self.inputs) nix-snapshotter;
|
|
|
|
config = {
|
|
entrypoint = "${self.nixosConfigurations.default.config.system.build.toplevel}/init";
|
|
# port = 2379;
|
|
args = [
|
|
];
|
|
# nodePort = 30001;
|
|
};
|
|
|
|
myceliumPorts = {
|
|
tcp = [9651];
|
|
udp = [9650 9651];
|
|
};
|
|
|
|
inherit
|
|
(config)
|
|
entrypoint
|
|
# port
|
|
|
|
args
|
|
# nodePort
|
|
|
|
;
|
|
|
|
pkgs = import nixpkgs {
|
|
overlays = [nix-snapshotter.overlays.default];
|
|
};
|
|
|
|
image = pkgs.nix-snapshotter.buildImage {
|
|
inherit name;
|
|
resolvedByNix = true;
|
|
config = {
|
|
entrypoint = [entrypoint];
|
|
env = [
|
|
# this is read by the `/init` script and prevents various incompatible commands like mount, etc.
|
|
# the value of this doesn't seem to matter as long as it's not an empty string.
|
|
"container=nerd"
|
|
];
|
|
volumes = {
|
|
# "/var/lib/private/mycelium/key.bin" = {};
|
|
};
|
|
copyToRoot = [
|
|
# self.nixosConfigurations.default.config.system.build.toplevel
|
|
];
|
|
};
|
|
};
|
|
in {
|
|
k8s = let
|
|
pod = pkgs.writeText "${name}-pod.json" (builtins.toJSON {
|
|
apiVersion = "v1";
|
|
kind = "Pod";
|
|
metadata = {
|
|
inherit name;
|
|
labels = {inherit name;};
|
|
};
|
|
spec.containers = [
|
|
{
|
|
inherit name args;
|
|
image = "nix:0${image}";
|
|
ports = [
|
|
{
|
|
name = "mycelium-tcp-0";
|
|
containerPort = builtins.elemAt myceliumPorts.tcp 0;
|
|
}
|
|
{
|
|
name = "mycelium-udp-0";
|
|
protocol = "UDP";
|
|
containerPort = builtins.elemAt myceliumPorts.udp 0;
|
|
}
|
|
{
|
|
name = "mycelium-udp-1";
|
|
protocol = "UDP";
|
|
containerPort = builtins.elemAt myceliumPorts.udp 1;
|
|
}
|
|
];
|
|
}
|
|
];
|
|
});
|
|
|
|
service = pkgs.writeText "${name}-service.json" (builtins.toJSON {
|
|
apiVersion = "v1";
|
|
kind = "Service";
|
|
metadata.name = "${name}-service";
|
|
spec = {
|
|
type = "NodePort";
|
|
selector = {inherit name;};
|
|
ports = [
|
|
{
|
|
name = "mycelium-tcp-0";
|
|
port = builtins.elemAt myceliumPorts.tcp 0 + 50000;
|
|
targetPort = "mycelium-tcp-0";
|
|
}
|
|
{
|
|
name = "mycelium-udp-0";
|
|
protocol = "UDP";
|
|
port = builtins.elemAt myceliumPorts.udp 0 + 50000;
|
|
targetPort = "mycelium-udp-0";
|
|
}
|
|
{
|
|
name = "mycelium-udp-1";
|
|
protocol = "UDP";
|
|
port = builtins.elemAt myceliumPorts.udp 1 + 50000;
|
|
targetPort = "mycelium-udp-1";
|
|
}
|
|
];
|
|
};
|
|
});
|
|
in
|
|
pkgs.runCommand "declarative-k8s" {} ''
|
|
mkdir -p $out/share/k8s
|
|
cp ${pod} $out/share/k8s/
|
|
cp ${service} $out/share/k8s/
|
|
'';
|
|
|
|
inherit image;
|
|
|
|
start = pkgs.writeShellApplication {
|
|
name = "start";
|
|
text = ''
|
|
set -x
|
|
rm -rf ./result
|
|
nix build --impure .#image
|
|
sudo nix2container load ./result
|
|
sudo -E nerdctl run --name ${name} --privileged -dt \
|
|
--volume "$PWD/key.bin.crypt:${self.nixosConfigurations.default.config.services.mycelium.keyFile}:ro" \
|
|
"nix:0$(readlink result):latest"
|
|
'';
|
|
};
|
|
|
|
stop = pkgs.writeShellApplication {
|
|
name = "stop";
|
|
text = ''
|
|
set +e
|
|
sudo -E nerdctl stop -t 60 ${name}
|
|
sudo -E nerdctl rm --force ${name}
|
|
sudo -E nerdctl system prune --all --force
|
|
sudo systemctl stop nix-snapshotter
|
|
sudo systemctl stop containerd
|
|
mount | rg -No '(/var/lib/container|/tmp/initial)[^ ]+' | tac | xargs sudo umount -l
|
|
sudo systemctl start containerd
|
|
sudo systemctl start nix-snapshotter
|
|
'';
|
|
|
|
# tmpfs on /run/credentials/mycelium.service type tmpfs (ro,nosuid,nodev,noexec,relatime,nosymfollow,size=1024k,nr_inodes=1024,mode=700,noswap)
|
|
|
|
# mount -t tmpfs tmpfs /run/credentials/mycelium.service -o ro,nosuid,nodev,noexec,relatime,nosymfollow,size=1024k,nr_inodes=1024,mode=700,noswap
|
|
};
|
|
});
|
|
};
|
|
}
|