From 088e83dd41e1d22da44cf42275a404dabcefb5c2 Mon Sep 17 00:00:00 2001 From: Stefan Junker Date: Sun, 10 Feb 2019 13:51:21 +0100 Subject: [PATCH] nix/containers,vmd32387: add backup container --- nix/os/containers/backup.nix | 184 ++++++++++++++++++ .../vmd32387.contaboserver.net/system.nix | 6 + nix/variables/passwords.crypt.nix | Bin 721 -> 885 bytes 3 files changed, 190 insertions(+) create mode 100644 nix/os/containers/backup.nix diff --git a/nix/os/containers/backup.nix b/nix/os/containers/backup.nix new file mode 100644 index 0000000..db77065 --- /dev/null +++ b/nix/os/containers/backup.nix @@ -0,0 +1,184 @@ +{ config, ... } @ args: + +let + unstablepkgs = import { config = config.nixpkgs.config; }; + + passwords = import ../../variables/passwords.crypt.nix; + bucket = "bkp"; + subvolumeParentDir = "/var/lib"; + + subvolumeDir = "/var/lib/container-volumes"; + subvolumeSnapshot = "/var/lib/container-volumes.snapshot"; + + bkpSource = subvolumeSnapshot; + bkpDestination = "/container/backup"; + cacheDir = "/var/lib/rclone-cachedir"; + + wasabiRc = pkgs: pkgs.writeText "rc" '' + [wasabi-${bucket}] + type = s3 + provider = Wasabi + env_auth = false + + #bkp user + access_key_id = ${passwords.storage.wasabi.bkp.key} + secret_access_key = ${passwords.storage.wasabi.bkp.secret} + + region = us-east-1 + endpoint = s3.wasabisys.com + location_constraint = + acl = + server_side_encryption = + storage_class = + ''; + + + bkp-mount-rclone-manual = pkgs: { + enable = true; + description = "bkp-mount-rclone-manual service"; + path = with pkgs; [ unstablepkgs.rclone utillinux ]; + serviceConfig = { + Type = "notify"; + }; + script = '' + export PATH="$PATH:/run/wrappers/bin" + exec rclone --config ${wasabiRc pkgs} mount wasabi-${bucket}:${bucket} ${bkpDestination} \ + --stats=1m --stats-log-level=NOTICE \ + --cache-dir=${cacheDir} \ + --vfs-cache-mode=full + + ''; + preStart = '' + mkdir -p ${bkpDestination} + mkdir -p ${cacheDir} + ''; + postStop = '' + sync + umount ${bkpDestination} \ + || umount -l ${bkpDestination} \ + || : + + rmdir ${bkpDestination} + ''; + }; + + +in args // { + config = { pkgs, ... }: { + imports = [ + ../profiles/containers/configuration.nix + ]; + + environment.systemPackages = with pkgs; [ + btrfs-progs + rdup rdedup + iptraf-ng nethogs + rclone + ]; + + networking.firewall.enable = true; + + systemd.services."bkp-mount-rclone-manual" = bkp-mount-rclone-manual pkgs; + + systemd.services."bkp-sync-rclone" = { + enable = true; + description = "bkp-sync-rclone service"; + + serviceConfig = { + Type = "oneshot"; + }; + + after = [ + "bkp-run.service" + ]; + requires = [ + "bkp-run.service" + ]; + + path = with pkgs; [ unstablepkgs.rclone utillinux ]; + script = '' + set -x + echo Starting rclone sync... + rclone --config ${wasabiRc pkgs} sync \ + ${bkpDestination}/rdedup/ wasabi-${bucket}:${bucket}/rdedup/ \ + --stats=1m --stats-log-level=NOTICE + echo Finished rclone sync... + ''; + }; + + systemd.services."bkp-run" = { + enable = true; + description = "bkp-run"; + + serviceConfig = { + Type = "oneshot"; + }; + + partOf = [ + "bkp-sync-rclone.service" + ]; + + path = with pkgs; [ btrfs-progs rdup rdedup coreutils ]; + preStart = '' + echo Creating new btrfs snapshot of ${subvolumeDir} at ${subvolumeSnapshot} + btrfs subvolume snapshot -r ${subvolumeDir} ${subvolumeSnapshot} + ''; + script = '' + #! ${pkgs.bash}/bin/bash + export RUST_BACKTRACE=1 + export TIMESTAMP=$(date +"%Y%m%d.%H%M%S") + + echo Starting rdup/rdedup backup... + for d in `ls -1 ${bkpSource}`; do + echo Determining backup source size ${bkpSource}/$d... + du -hs ${bkpSource}/$d + set -x + rdup -x /dev/null ${bkpSource}/$d | rdedup -v -ttt --dir=${bkpDestination}/rdedup store $d-''${TIMESTAMP} + set +x + done + sync + echo Finished rdup/rdedup backup... + echo Determining backup destination size ${bkpDestination}/rdedup... + du -hs ${bkpDestination}/rdedup + ''; + postStop = '' + btrfs subvolume delete ${subvolumeSnapshot} + ''; + }; + + systemd.timers."bkp" = { + description = "Timer to trigger bkp periodically"; + enable = true; + wantedBy = [ "timer.target" "multi-user.target" ]; + timerConfig = { + OnCalendar = "23:00"; + Unit = "bkp-sync-rclone.service"; + # OnActiveSec="1s"; + # OnUnitInactiveSec="12h"; + # AccuracySec="5s"; + }; + }; + }; + + autoStart = true; + + bindMounts = { + "${subvolumeParentDir}" = { + hostPath = "/var/lib/"; + isReadOnly = false; + }; + + "/dev/fuse" = { + hostPath = "/dev/fuse"; + isReadOnly = false; + }; + }; + + allowedDevices = [ + { node = "/dev/fuse"; modifier = "rw"; } + ]; + + privateNetwork = true; + forwardPorts = [ + ]; +} diff --git a/nix/os/devices/vmd32387.contaboserver.net/system.nix b/nix/os/devices/vmd32387.contaboserver.net/system.nix index 6d4ff00..abf32e6 100644 --- a/nix/os/devices/vmd32387.contaboserver.net/system.nix +++ b/nix/os/devices/vmd32387.contaboserver.net/system.nix @@ -85,5 +85,11 @@ in { hostAddress = "192.168.100.14"; localAddress = "192.168.100.15"; }; + + backup = import ../../containers/backup.nix { + inherit config; + hostAddress = "192.168.100.16"; + localAddress = "192.168.100.17"; + }; }; } diff --git a/nix/variables/passwords.crypt.nix b/nix/variables/passwords.crypt.nix index f2d3a022b2561c7625bbc964c4e731ab5a66ed4b..92f89d2bfc12e71b27ee2c02617dc3292635bb3d 100644 GIT binary patch literal 885 zcmZQ@_Y83kiVO&0xSx<;y21QlRPvFGuCwA>SWkV5xo~gVxz9Y`Z$_o7N<|zwd*RP< zSKm3BT!Eq&m*h`#-}(Dw-3ehOjoPi;uWxNUDfq|GY}$;j^1z7N;2TlAzr?s7d)Hs^ zF?01!WBV{UGa@H{wjuxNrPC9V91Im*UVM);mb}0AvqW^qXJI|=wPn0|42$l}b+wHu zvp9F~-HO$<0{nr8lJ`WIE$!&J!~LXk z&RPFC{4hI&_tiOGu^yRprAacs4Na1Jd_wX}cp@7*H@*0L!~eGF8rQ}uhM((~7SC;5 zlKNF${g#us_<6U;uXl|qWgiqCj{X%@TkE){Y2rrLG$+mjUHS(?MU@tPt!{hsv)0y0 zHpJbfcE%kquj)xxABpoA%H3PpcQVR%73-B_^LI~~e)48c=8OPCmxZUksl74tnba)0 zr_5pX%#7xVY4fZ*kL%ejSQxUUp56eG+k@C& zTC0xF{GPqzV@+ZFW>s~=Us7cT84aB7Tr2!}=knzi+<)}0Y?J-UZR-SbHa!hKmJ!A^ zL2qGJ%s=+tO2TSNp`h^|q@{%FLRzllP$4 zE0@AM7CEtZuD=(_jb~otE;Tdh^5c5@&pY2&PAwI)|JNSHvG|Ufpz{5zXH0i(%hVQY zbC=o~{cp|=IirXNpDvh8ZPVN>_iV+zQmOw1F_WVfMHfFXG5J%q=R-Q*bIG*CP)>`> z@jExInij<$CTS!ZZ5!p;XXvSavOCuB&q_tJ^-72NlpAarG`=`=w!PcmBs4wAae>6v z|JyE1Dt^oJ`&mxqnK=#hH&iC^+lJQ)v0qx-c6Z&{SJU;LTv#FR_EvYt?%}YbXOrF(!J99)um89`uWsw&jX9FRl}rB%eydGXQs`S%owS@I zr{#U7hLp8}sio19Grl)N0xs~1H7%Gkef#wX>*Q;%KRNsP+}8OF;o=wCcL*=3`1I1k zZLxh}$d_sLic4RfySwUX zrJorP81i8DE*`lwV~escX}7I246=rk z1QusxhHX3jKq641#!zJW%omHY{#BdUGTSP*th~Kr-yVgB^By|Qn%I0)p=`#Ad&!Yn zswUG8M&Hhn`uAz>IYqZO9(z{J+xWTA%`<8G;eYoGm;H-AT<$ghUXeuwmxTA5xj&h{ z?D;F~^m$hPoc8Lb7Z1;@T%{lFyXs#_McQnen~&ujd$g4$pZ%pFI-Lhb8q6BhtpP0-0C$oWVOnhnoWJ;XFh*YDhPHcJE_RB z%>VLSF|)Z2CuZ9RocUOt!{6lDsHAd)lckMyQ;upA&w-BFi>}+8*;+M4R;cT)nRHm- zlxt^%-=;+9h0EWY`(?GDvtAL~Bt*3yp4*e|7X5LdfSq&Z zfoGqKjwn_{T+L&XVGib>@oUE#gOh8Q$TQ3=ydU7(Ai477yf3QrpE4*Mk-b)S{Bc#q pvU#$cL7U>b->9A`D|)0Z{3_$06#zJda3KHy