steveej/infra
1
0
Fork 0
Code Issues 2 Pull requests Projects Releases Packages Wiki Activity Actions

WIP k3s

Browse source
This commit is contained in:
steveej 2024-10-16 18:28:17 +02:00
parent 4c71887ea6
commit bd8630681b
3 changed files with 86 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

45
nix/os/containers/mycelium/flake.lock generated
View file

@ -46,11 +46,11 @@
] ]
}, },
"locked": { "locked": {
"lastModified": 1715438114, "lastModified": 1723875769,
"narHash": "sha256-btb702TXuhDg0D6tW0dCOy4+II9Wl6BJ0LvpT+O9wrs=", "narHash": "sha256-66GofByLJ+S4ZZphIC+vJKeL9VJ2bzH2VbcJ3OqteMM=",
"owner": "pdtpartners", "owner": "pdtpartners",
"repo": "nix-snapshotter", "repo": "nix-snapshotter",
"rev": "7b251c9356bc7bb383ebeedcd0045b3ae431bff7", "rev": "6eaadfd8f89e5e7d79b2013626bbd36e388159da",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -61,11 +61,11 @@
}, },
"nixlib": { "nixlib": {
"locked": { "locked": {
"lastModified": 1712450863, "lastModified": 1728781282,
"narHash": "sha256-K6IkdtMtq9xktmYPj0uaYc8NsIqHuaAoRBaMgu9Fvrw=", "narHash": "sha256-hUP9oxmnOmNnKcDOf5Y55HQ+NnoT0+bLWHLQWLLw9Ks=",
"owner": "nix-community", "owner": "nix-community",
"repo": "nixpkgs.lib", "repo": "nixpkgs.lib",
"rev": "3c62b6a12571c9a7f65ab037173ee153d539905f", "rev": "16340f605f4e8e5cf07fd74dcbe692eee2d4f51b",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -82,11 +82,11 @@
] ]
}, },
"locked": { "locked": {
"lastModified": 1718025593, "lastModified": 1728867876,
"narHash": "sha256-WZ1gdKq/9u1Ns/oXuNsDm+W0salonVA0VY1amw8urJ4=", "narHash": "sha256-NCyOA8WZNoojmXH+kBDrQj3LwvakYNzSc0h+LTXkmPE=",
"owner": "nix-community", "owner": "nix-community",
"repo": "nixos-generators", "repo": "nixos-generators",
"rev": "35c20ba421dfa5059e20e0ef2343c875372bdcf3", "rev": "fdf142111597f6c6283cf5ffe092b6293a3911d0",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -97,42 +97,25 @@
}, },
"nixpkgs": { "nixpkgs": {
"locked": { "locked": {
"lastModified": 1718086528, "lastModified": 1728897630,
"narHash": "sha256-hoB7B7oPgypePz16cKWawPfhVvMSXj4G/qLsfFuhFjw=", "narHash": "sha256-0utJPs4o2Mody8GDwo4hnGuxc8dJqju4u9lLJY4d/Lw=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "47b604b07d1e8146d5398b42d3306fdebd343986", "rev": "c9f0b4a395289ce18727e2a8e43cae6796693ccc",
"type": "github" "type": "github"
}, },
"original": { "original": {
"owner": "NixOS", "owner": "NixOS",
"ref": "nixos-24.05", "ref": "nixos-unstable-small",
"repo": "nixpkgs", "repo": "nixpkgs",
"type": "github" "type": "github"
} }
}, },
"nixpkgs-systemd256": {
"locked": {
"lastModified": 1718397913,
"narHash": "sha256-omV+dq3GdXQQTaewxhkBgxM4Bbwqa4D9FVS4dTITxOQ=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "962cf03fb8c782c5e00f465397e03dc84284acc9",
"type": "github"
},
"original": {
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "962cf03fb8c782c5e00f465397e03dc84284acc9",
"type": "github"
}
},
"root": { "root": {
"inputs": { "inputs": {
"nix-snapshotter": "nix-snapshotter", "nix-snapshotter": "nix-snapshotter",
"nixos-generators": "nixos-generators", "nixos-generators": "nixos-generators",
"nixpkgs": "nixpkgs", "nixpkgs": "nixpkgs"
"nixpkgs-systemd256": "nixpkgs-systemd256"
} }
} }
}, },

42
nix/os/containers/mycelium/flake.nix
View file

@ -1,7 +1,7 @@
{ {
inputs = { inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
nixpkgs-systemd256.url = "github:NixOS/nixpkgs/962cf03fb8c782c5e00f465397e03dc84284acc9"; # nixpkgs-systemd256.url = "github:NixOS/nixpkgs/962cf03fb8c782c5e00f465397e03dc84284acc9";
nixos-generators = { nixos-generators = {
url = "github:nix-community/nixos-generators"; url = "github:nix-community/nixos-generators";
inputs.nixpkgs.follows = "nixpkgs"; inputs.nixpkgs.follows = "nixpkgs";
@ -102,7 +102,7 @@
imports = [ imports = [
(modulesPath + "/profiles/minimal.nix") (modulesPath + "/profiles/minimal.nix")
]; ];
system.stateVersion = "24.05"; system.stateVersion = "24.11";
# https://github.com/hercules-ci/arion/blob/c24c185e67f093298a081900b49ca18716077dec/src/nix/modules/nixos/container-systemd.nix # https://github.com/hercules-ci/arion/blob/c24c185e67f093298a081900b49ca18716077dec/src/nix/modules/nixos/container-systemd.nix
boot.isContainer = true; boot.isContainer = true;
@ -116,22 +116,37 @@
services.nscd.enable = false; services.nscd.enable = false;
system.nssModules = lib.mkForce []; system.nssModules = lib.mkForce [];
systemd.services.systemd-logind.enable = false; systemd.services.systemd-logind.enable = false;
systemd.services.console-getty.enable = true; systemd.services.console-getty.enable = false;
systemd.sockets.nix-daemon.enable = false; systemd.sockets.nix-daemon.enable = false;
systemd.services.nix-daemon.enable = false; systemd.services.nix-daemon.enable = false;
systemd.oomd.enable = false; systemd.oomd.enable = false;
networking.useDHCP = false; networking.useDHCP = false;
networking.firewall.enable = false; networking.firewall.enable = false;
# system.build.earlyMountScript =
# lib.mkForce ''
# '';
# system.activationScripts.specialfs =
# lib.mkForce ''
# '';
boot.postBootCommands = '' boot.postBootCommands = ''
ls -lha /run ls -lha /run
mkdir -p /run/wrappers mkdir -p /run/wrappers
''; '';
boot.kernelParams = [
"systemd.log_level=debug"
];
# services.udev.enable = false; # services.udev.enable = false;
# TODO: this is only needed because `/run/current-system` is missing # TODO: this is only needed because `/run/current-system` is missing
# environment.variables.PATH = "${lib.makeBinPath config.environment.systemPackages}:$PATH"; # environment.variables.PATH = "${lib.makeBinPath config.environment.systemPackages}:$PATH";
systemd.mounts = lib.mkForce [];
fileSystems = lib.mkForce {};
services.mycelium.enable = false; services.mycelium.enable = false;
services.mycelium.keyFile = "/var/lib/secrets/mycelium-keyfile"; services.mycelium.keyFile = "/var/lib/secrets/mycelium-keyfile";
systemd.services.mycelium.serviceConfig.DynamicUser = lib.mkForce false; systemd.services.mycelium.serviceConfig.DynamicUser = lib.mkForce false;
@ -151,17 +166,18 @@
serviceConfig = { serviceConfig = {
SyslogIdentifier = "testing-credential"; SyslogIdentifier = "testing-credential";
StateDirectory = "testing-credentials"; StateDirectory = "testing-credentials";
# DynamicUser = true; DynamicUser = true;
# User = "tc"; # User = "tc";
# ProtectHome = true; # ProtectHome = true;
# ProtectSystem = true; ProtectSystem = true;
LoadCredential = [ # LoadCredential = [
"mycelium-keyfile:${self.nixosConfigurations.default.config.services.mycelium.keyFile}" # "mycelium-keyfile:${self.nixosConfigurations.default.config.services.mycelium.keyFile}"
"hosts:/etc/hosts" # "hosts:/etc/hosts"
]; # ];
SetCredential = "nosecret:not secret string"; SetCredential = "mycelium-keyfile:not secret string";
ExecStart = lib.mkForce (pkgs.writeShellScript "mycelium" '' ExecStart = lib.mkForce (pkgs.writeShellScript "mycelium" ''
cd $STATE_DIRECTORY cd $STATE_DIRECTORY
pwd
env env
while true; do while true; do
ls -lha $CREDENTIALS_DIRECTORY ls -lha $CREDENTIALS_DIRECTORY
@ -228,6 +244,9 @@
]; ];
volumes = { volumes = {
# "/var/lib/private/mycelium/key.bin" = {}; # "/var/lib/private/mycelium/key.bin" = {};
# "/run" = {};
# "/tmp" = {};
# "/etc" = {};
}; };
copyToRoot = [ copyToRoot = [
# self.nixosConfigurations.default.config.system.build.toplevel # self.nixosConfigurations.default.config.system.build.toplevel
@ -312,6 +331,7 @@
nix build --impure .#image nix build --impure .#image
sudo nix2container load ./result sudo nix2container load ./result
sudo -E nerdctl run --name ${name} --privileged -dt \ sudo -E nerdctl run --name ${name} --privileged -dt \
--cgroup-manager cgroupfs \
--volume "$PWD/key.bin.crypt:${self.nixosConfigurations.default.config.services.mycelium.keyFile}:ro" \ --volume "$PWD/key.bin.crypt:${self.nixosConfigurations.default.config.services.mycelium.keyFile}:ro" \
"nix:0$(readlink result):latest" "nix:0$(readlink result):latest"
''; '';

25
nix/os/snippets/k3s-w-nix-snapshotter.nix
View file

@ -5,24 +5,38 @@
pkgs, pkgs,
lib, lib,
system, system,
config,
... ...
}: { }: let
cfg = config.steveej.k3s;
# TODO: make this configurable
homeUser = "steveej";
in {
options.steveej.k3s = {
enable = lib.mkOption {
description = "steveej's k3s distro";
type = lib.types.bool;
default = true;
};
};
# (1) Import nixos module. # (1) Import nixos module.
imports = [ imports = [
nodeFlake.inputs.nix-snapshotter.nixosModules.default nodeFlake.inputs.nix-snapshotter.nixosModules.default
]; ];
config = lib.mkIf cfg.enable {
# (2) Add overlay. # (2) Add overlay.
nixpkgs.overlays = [nodeFlake.inputs.nix-snapshotter.overlays.default]; nixpkgs.overlays = [nodeFlake.inputs.nix-snapshotter.overlays.default];
# (3) Enable service. # (3) Enable service.
virtualisation.containerd = { virtualisation.containerd = {
enable = true; enable = true;
k3sIntegration = false;
nixSnapshotterIntegration = true; nixSnapshotterIntegration = true;
# TODO: understand if this has an influence on the systemd LoadCredential issue # TODO: understand if this has an influence on the systemd LoadCredential issue
settings.plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options.SystemdCgroup = lib.mkForce true; # settings.plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options.SystemdCgroup = lib.mkForce true;
}; };
services.nix-snapshotter = { services.nix-snapshotter = {
enable = true; enable = true;
@ -38,4 +52,9 @@
enable = false; enable = false;
setKubeConfig = true; setKubeConfig = true;
}; };
# home-manager.users."${homeUser}" = _: {
# home.sessionVariables.CONTAINERD_ADDRESS = "/run/user/1000/containerd/containerd.sock";
# };
};
} }