From 10d015bb63692dd48785796a28d5c10fc35648ea Mon Sep 17 00:00:00 2001 From: Stefan Junker Date: Sun, 18 Feb 2018 21:33:49 +0100 Subject: [PATCH 1/2] containers: start with base, s3ql, syncthing --- container-images/build.sh | 7 ++ container-images/default.nix | 157 +++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100755 container-images/build.sh create mode 100644 container-images/default.nix diff --git a/container-images/build.sh b/container-images/build.sh new file mode 100755 index 0000000..6cfab1a --- /dev/null +++ b/container-images/build.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -xe +[ ! -z "$NAME" ] + +nix-build . --show-trace -A "$NAME" +docker image rm "$NAME":latest --force +docker load -i result diff --git a/container-images/default.nix b/container-images/default.nix new file mode 100644 index 0000000..b2984c2 --- /dev/null +++ b/container-images/default.nix @@ -0,0 +1,157 @@ +{ pkgs ? import {} +}: + +let + baseEnv = [ + "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + ]; + + +in rec { + + base = let + minimalDocker = + { + imports = [ ]; + boot.isContainer = true; + environment.etc.machine-id.text = "00000000000000000000000000000000"; + }; + eval = + import { + modules = [ + minimalDocker + ]; + }; + system = + eval.config.system; + + in pkgs.dockerTools.buildImage rec { + name = "base"; + +# contents = pkgs.symlinkJoin { +# name = "${name}-contents"; +# paths = [ +# system.build.etc +# system.path +# ]; +# }; + + # Requires a VM to boot + runAsRoot = '' + #!${pkgs.stdenv.shell} + ${pkgs.dockerTools.shadowSetup} + groupadd users --gid 100 + useradd -g users -d /home/user -M --uid 1000 user + ''; + + config = { + Env = baseEnv; + WorkingDir = "/"; + }; + }; + + interactiveBase = pkgs.dockerTools.buildImage { + name = "interactiveBase"; + fromImage = base; + contents = with pkgs; [ + procps + zsh + coreutils + vim + ]; + + config = { + Cmd = [ "/bin/zsh" ]; + }; + }; + + s3ql = let + entrypoint = pkgs.writeScript "entrypoint" '' + #!${pkgs.stdenv.shell} + + if [ -z "$S3QL_BUCKET" ]; then + echo S3QL_BUCKET not set + exit 1 + fi + + mkdir -p /buckets/"$S3QL_BUCKET" + + set -x + exec mount.s3ql \ + --cachedir "$S3QL_CACHE_DIR" \ + --authfile "$S3QL_AUTHINFO2" \ + --cachesize "$S3QL_CACHESIZE" \ + --fg \ + --log none \ + --allow-root \ + s3c://e24files.com/steveej-backup \ + /buckets/"$S3QL_BUCKET" + + # FIXME: touch .isbucket after mount + ''; + in pkgs.dockerTools.buildImage { + name = "s3ql"; + fromImage = interactiveBase; + contents = [ + pkgs.s3ql + pkgs.fuse + ]; + + runAsRoot = '' + #!${pkgs.stdenv.shell} + mkdir -p /usr/bin + cp -a ${pkgs.fuse}/bin/fusermount /usr/bin + chmod +s /usr/bin/fusermount + echo user_allow_other >> /etc/fuse.conf + ''; + + config = { + Env = baseEnv ++ [ + "HOME=/home/s3ql" + "S3QL_CACHE_DIR=/var/cache/s3ql" + "S3QL_AUTHINFO2=/etc/s3ql/authinfo2" + "S3QL_CACHESIZE=0" + "CONTAINER_ENTRYPOINT=${entrypoint}" + ]; + Cmd = [ entrypoint ]; + Volumes = { + "/var/cache/s3ql" = {}; + "/etc/s3ql/authinfo2" = {}; + "/buckets" = {}; + "/tmp" = {}; + }; + }; + }; + + syncthing = let + entrypoint = pkgs.writeScript "entrypoint" '' + #!${pkgs.stdenv.shell} + set -x + if [ ! -e /data/.isbucket ]; then + echo ERROR: Bucket not mounted at /data + exit 1 + fi + + exec syncthing \ + -home $SYNCTHING_HOME \ + -gui-address=$SYNCTHING_GUI_ADDRESS \ + -no-browser + ''; + in pkgs.dockerTools.buildImage { + name = "syncthing"; + fromImage = interactiveBase; + contents = pkgs.syncthing; + + config = { + Env = baseEnv ++ [ + "SYNCTHING_HOME=/home/syncthing" + "SYNCTHING_GUI_ADDRESS=0.0.0.0:8384" + ]; + Cmd = [ entrypoint ]; + Volumes = { + "/home/syncthing" = {}; + "/data" = {}; + }; + }; + }; +} From d36545c4191eefd3805b350509976ed92915555f Mon Sep 17 00:00:00 2001 From: Stefan Junker Date: Fri, 4 May 2018 08:53:25 +0200 Subject: [PATCH 2/2] containers: work on synchting container --- container-images/default.nix | 71 ++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/container-images/default.nix b/container-images/default.nix index b2984c2..c85ba1a 100644 --- a/container-images/default.nix +++ b/container-images/default.nix @@ -9,33 +9,9 @@ let in rec { - base = let - minimalDocker = - { - imports = [ ]; - boot.isContainer = true; - environment.etc.machine-id.text = "00000000000000000000000000000000"; - }; - eval = - import { - modules = [ - minimalDocker - ]; - }; - system = - eval.config.system; - - in pkgs.dockerTools.buildImage rec { + base = pkgs.dockerTools.buildImage rec { name = "base"; -# contents = pkgs.symlinkJoin { -# name = "${name}-contents"; -# paths = [ -# system.build.etc -# system.path -# ]; -# }; - # Requires a VM to boot runAsRoot = '' #!${pkgs.stdenv.shell} @@ -50,8 +26,8 @@ in rec { }; }; - interactiveBase = pkgs.dockerTools.buildImage { - name = "interactiveBase"; + interactive_base = pkgs.dockerTools.buildImage { + name = "interactive_base"; fromImage = base; contents = with pkgs; [ procps @@ -74,24 +50,43 @@ in rec { exit 1 fi - mkdir -p /buckets/"$S3QL_BUCKET" + if [ -z "$S3QL_STORAGE_URL" ]; then + echo S3QL_STORAGE_URL not set + exit 1 + fi + + if [ -z "$S3QL_CACHESIZE" ]; then + echo S3QL_CACHESIZE not set + exit 1 + fi set -x + + if [ "$S3QL_SKIP_FSCK" != "1" ]; then + fsck.s3ql \ + --authfile $S3QL_AUTHINFO2 \ + --log none \ + --cachedir $S3QL_CACHE_DIR \ + $S3QL_STORAGE_URL + fi + exec mount.s3ql \ --cachedir "$S3QL_CACHE_DIR" \ --authfile "$S3QL_AUTHINFO2" \ --cachesize "$S3QL_CACHESIZE" \ --fg \ + --compress lzma-6 \ + --threads 4 \ --log none \ --allow-root \ - s3c://e24files.com/steveej-backup \ - /buckets/"$S3QL_BUCKET" + "$S3QL_STORAGE_URL" \ + /bucket # FIXME: touch .isbucket after mount ''; in pkgs.dockerTools.buildImage { name = "s3ql"; - fromImage = interactiveBase; + fromImage = interactive_base; contents = [ pkgs.s3ql pkgs.fuse @@ -110,7 +105,6 @@ in rec { "HOME=/home/s3ql" "S3QL_CACHE_DIR=/var/cache/s3ql" "S3QL_AUTHINFO2=/etc/s3ql/authinfo2" - "S3QL_CACHESIZE=0" "CONTAINER_ENTRYPOINT=${entrypoint}" ]; Cmd = [ entrypoint ]; @@ -132,6 +126,15 @@ in rec { exit 1 fi + if [ -z "$SYNCTHING_GUI_ADDRESS" ]; then + echo ERROR: SYNCTHING_GUI_ADDRESS is not set + exit 1 + fi + + if [ ! -w "$SYNCTHING_HOME" ]; then + echo ERROR : SYNCTHING_HOME is not writable + fi + exec syncthing \ -home $SYNCTHING_HOME \ -gui-address=$SYNCTHING_GUI_ADDRESS \ @@ -139,17 +142,15 @@ in rec { ''; in pkgs.dockerTools.buildImage { name = "syncthing"; - fromImage = interactiveBase; + fromImage = interactive_base; contents = pkgs.syncthing; config = { Env = baseEnv ++ [ "SYNCTHING_HOME=/home/syncthing" - "SYNCTHING_GUI_ADDRESS=0.0.0.0:8384" ]; Cmd = [ entrypoint ]; Volumes = { - "/home/syncthing" = {}; "/data" = {}; }; };