From 5e5a3ad869ffbf91608a52ad117a858ee93eda2e Mon Sep 17 00:00:00 2001 From: clerie Date: Sat, 10 Apr 2021 14:52:55 +0200 Subject: [PATCH] Add borgbackup module to customize --- modules/borgbackup/default.nix | 675 +++++++++++++++++++++++++++++++++ modules/default.nix | 1 + 2 files changed, 676 insertions(+) create mode 100644 modules/borgbackup/default.nix diff --git a/modules/borgbackup/default.nix b/modules/borgbackup/default.nix new file mode 100644 index 0000000..db1edb3 --- /dev/null +++ b/modules/borgbackup/default.nix @@ -0,0 +1,675 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + + isLocalPath = x: + builtins.substring 0 1 x == "/" # absolute path + || builtins.substring 0 1 x == "." # relative path + || builtins.match "[.*:.*]" == null; # not machine:path + + mkExcludeFile = cfg: + # Write each exclude pattern to a new line + pkgs.writeText "excludefile" (concatStringsSep "\n" cfg.exclude); + + mkKeepArgs = cfg: + # If cfg.prune.keep e.g. has a yearly attribute, + # its content is passed on as --keep-yearly + concatStringsSep " " + (mapAttrsToList (x: y: "--keep-${x}=${toString y}") cfg.prune.keep); + + mkBackupScript = cfg: '' + on_exit() + { + exitStatus=$? + # Reset the EXIT handler, or else we're called again on 'exit' below + trap - EXIT + ${cfg.postHook} + exit $exitStatus + } + trap 'on_exit' INT TERM QUIT EXIT + + archiveName="${cfg.archiveBaseName}-$(date ${cfg.dateFormat})" + archiveSuffix="${optionalString cfg.appendFailedSuffix ".failed"}" + ${cfg.preHook} + '' + optionalString cfg.doInit '' + # Run borg init if the repo doesn't exist yet + if ! borg list $extraArgs > /dev/null; then + borg init $extraArgs \ + --encryption ${cfg.encryption.mode} \ + $extraInitArgs + ${cfg.postInit} + fi + '' + '' + borg create $extraArgs \ + --compression ${cfg.compression} \ + --exclude-from ${mkExcludeFile cfg} \ + $extraCreateArgs \ + "::$archiveName$archiveSuffix" \ + ${escapeShellArgs cfg.paths} + '' + optionalString cfg.appendFailedSuffix '' + borg rename $extraArgs \ + "::$archiveName$archiveSuffix" "$archiveName" + '' + '' + ${cfg.postCreate} + '' + optionalString (cfg.prune.keep != { }) '' + borg prune $extraArgs \ + ${mkKeepArgs cfg} \ + --prefix ${escapeShellArg cfg.prune.prefix} \ + $extraPruneArgs + ${cfg.postPrune} + ''; + + mkPassEnv = cfg: with cfg.encryption; + if passCommand != null then + { BORG_PASSCOMMAND = passCommand; } + else if passphrase != null then + { BORG_PASSPHRASE = passphrase; } + else { }; + + mkBackupService = name: cfg: + let + userHome = config.users.users.${cfg.user}.home; + in nameValuePair "borgbackup-job-${name}" { + description = "BorgBackup job ${name}"; + path = with pkgs; [ + borgbackup openssh + ]; + script = mkBackupScript cfg; + serviceConfig = { + User = cfg.user; + Group = cfg.group; + # Only run when no other process is using CPU or disk + CPUSchedulingPolicy = "idle"; + IOSchedulingClass = "idle"; + ProtectSystem = "strict"; + ReadWritePaths = + [ "${userHome}/.config/borg" "${userHome}/.cache/borg" ] + ++ cfg.readWritePaths + # Borg needs write access to repo if it is not remote + ++ optional (isLocalPath cfg.repo) cfg.repo; + PrivateTmp = cfg.privateTmp; + }; + environment = { + BORG_REPO = cfg.repo; + inherit (cfg) extraArgs extraInitArgs extraCreateArgs extraPruneArgs; + } // (mkPassEnv cfg) // cfg.environment; + inherit (cfg) startAt; + }; + + # utility function around makeWrapper + mkWrapperDrv = { + original, name, set ? {} + }: + pkgs.runCommandNoCC "${name}-wrapper" { + buildInputs = [ pkgs.makeWrapper ]; + } (with lib; '' + makeWrapper "${original}" "$out/bin/${name}" \ + ${concatStringsSep " \\\n " (mapAttrsToList (name: value: ''--set ${name} "${value}"'') set)} + ''); + + mkBorgWrapper = name: cfg: mkWrapperDrv { + original = "${pkgs.borgbackup}/bin/borg"; + name = "borg-job-${name}"; + set = { BORG_REPO = cfg.repo; } // (mkPassEnv cfg) // cfg.environment; + }; + + # Paths listed in ReadWritePaths must exist before service is started + mkActivationScript = name: cfg: + let + install = "install -o ${cfg.user} -g ${cfg.group}"; + in + nameValuePair "borgbackup-job-${name}" (stringAfter [ "users" ] ('' + # Ensure that the home directory already exists + # We can't assert createHome == true because that's not the case for root + cd "${config.users.users.${cfg.user}.home}" + ${install} -d .config/borg + ${install} -d .cache/borg + '' + optionalString (isLocalPath cfg.repo && !cfg.removableDevice) '' + ${install} -d ${escapeShellArg cfg.repo} + '')); + + mkPassAssertion = name: cfg: { + assertion = with cfg.encryption; + mode != "none" -> passCommand != null || passphrase != null; + message = + "passCommand or passphrase has to be specified because" + + '' borgbackup.jobs.${name}.encryption != "none"''; + }; + + mkRepoService = name: cfg: + nameValuePair "borgbackup-repo-${name}" { + description = "Create BorgBackup repository ${name} directory"; + script = '' + mkdir -p ${escapeShellArg cfg.path} + chown ${cfg.user}:${cfg.group} ${escapeShellArg cfg.path} + ''; + serviceConfig = { + # The service's only task is to ensure that the specified path exists + Type = "oneshot"; + }; + wantedBy = [ "multi-user.target" ]; + }; + + mkAuthorizedKey = cfg: appendOnly: key: + let + # Because of the following line, clients do not need to specify an absolute repo path + cdCommand = "cd ${escapeShellArg cfg.path}"; + restrictedArg = "--restrict-to-${if cfg.allowSubRepos then "path" else "repository"} ."; + appendOnlyArg = optionalString appendOnly "--append-only"; + quotaArg = optionalString (cfg.quota != null) "--storage-quota ${cfg.quota}"; + serveCommand = "borg serve ${restrictedArg} ${appendOnlyArg} ${quotaArg}"; + in + ''command="${cdCommand} && ${serveCommand}",restrict ${key}''; + + mkUsersConfig = name: cfg: { + users.${cfg.user} = { + openssh.authorizedKeys.keys = + (map (mkAuthorizedKey cfg false) cfg.authorizedKeys + ++ map (mkAuthorizedKey cfg true) cfg.authorizedKeysAppendOnly); + useDefaultShell = true; + }; + groups.${cfg.group} = { }; + }; + + mkKeysAssertion = name: cfg: { + assertion = cfg.authorizedKeys != [ ] || cfg.authorizedKeysAppendOnly != [ ]; + message = + "borgbackup.repos.${name} does not make sense" + + " without at least one public key"; + }; + + mkRemovableDeviceAssertions = name: cfg: { + assertion = !(isLocalPath cfg.repo) -> !cfg.removableDevice; + message = '' + borgbackup.repos.${name}: repo isn't a local path, thus it can't be a removable device! + ''; + }; + +in { + meta.maintainers = with maintainers; [ dotlambda ]; + meta.doc = ./borgbackup.xml; + + ###### interface + + options.clerie.borgbackup.jobs = mkOption { + description = '' + Deduplicating backups using BorgBackup. + Adding a job will cause a borg-job-NAME wrapper to be added + to your system path, so that you can perform maintenance easily. + See also the chapter about BorgBackup in the NixOS manual. + ''; + default = { }; + example = literalExample '' + { # for a local backup + rootBackup = { + paths = "/"; + exclude = [ "/nix" ]; + repo = "/path/to/local/repo"; + encryption = { + mode = "repokey"; + passphrase = "secret"; + }; + compression = "auto,lzma"; + startAt = "weekly"; + }; + } + { # Root backing each day up to a remote backup server. We assume that you have + # * created a password less key: ssh-keygen -N "" -t ed25519 -f /path/to/ssh_key + # best practices are: use -t ed25519, /path/to = /run/keys + # * the passphrase is in the file /run/keys/borgbackup_passphrase + # * you have initialized the repository manually + paths = [ "/etc" "/home" ]; + exclude = [ "/nix" "'**/.cache'" ]; + doInit = false; + repo = "user3@arep.repo.borgbase.com:repo"; + encryption = { + mode = "repokey-blake2"; + passCommand = "cat /path/to/passphrase"; + }; + environment = { BORG_RSH = "ssh -i /path/to/ssh_key"; }; + compression = "auto,lzma"; + startAt = "daily"; + }; + ''; + type = types.attrsOf (types.submodule (let globalConfig = config; in + { name, config, ... }: { + options = { + + paths = mkOption { + type = with types; coercedTo str lib.singleton (listOf str); + description = "Path(s) to back up."; + example = "/home/user"; + }; + + repo = mkOption { + type = types.str; + description = "Remote or local repository to back up to."; + example = "user@machine:/path/to/repo"; + }; + + removableDevice = mkOption { + type = types.bool; + default = false; + description = "Whether the repo (which must be local) is a removable device."; + }; + + archiveBaseName = mkOption { + type = types.strMatching "[^/{}]+"; + default = "${globalConfig.networking.hostName}-${name}"; + defaultText = "\${config.networking.hostName}-"; + description = '' + How to name the created archives. A timestamp, whose format is + determined by , will be appended. The full + name can be modified at runtime ($archiveName). + Placeholders like {hostname} must not be used. + ''; + }; + + dateFormat = mkOption { + type = types.str; + description = '' + Arguments passed to date + to create a timestamp suffix for the archive name. + ''; + default = "+%Y-%m-%dT%H:%M:%S"; + example = "-u +%s"; + }; + + startAt = mkOption { + type = with types; either str (listOf str); + default = "daily"; + description = '' + When or how often the backup should run. + Must be in the format described in + systemd.time + 7. + If you do not want the backup to start + automatically, use [ ]. + It will generate a systemd service borgbackup-job-NAME. + You may trigger it manually via systemctl restart borgbackup-job-NAME. + ''; + }; + + user = mkOption { + type = types.str; + description = '' + The user borg is run as. + User or group need read permission + for the specified . + ''; + default = "root"; + }; + + group = mkOption { + type = types.str; + description = '' + The group borg is run as. User or group needs read permission + for the specified . + ''; + default = "root"; + }; + + encryption.mode = mkOption { + type = types.enum [ + "repokey" "keyfile" + "repokey-blake2" "keyfile-blake2" + "authenticated" "authenticated-blake2" + "none" + ]; + description = '' + Encryption mode to use. Setting a mode + other than "none" requires + you to specify a + or a . + ''; + example = '' + encryption.mode = "repokey-blake2" ; + encryption.passphrase = "mySecretPassphrase" ; + ''; + }; + + encryption.passCommand = mkOption { + type = with types; nullOr str; + description = '' + A command which prints the passphrase to stdout. + Mutually exclusive with . + ''; + default = null; + example = "cat /path/to/passphrase_file"; + }; + + encryption.passphrase = mkOption { + type = with types; nullOr str; + description = '' + The passphrase the backups are encrypted with. + Mutually exclusive with . + If you do not want the passphrase to be stored in the + world-readable Nix store, use . + ''; + default = null; + }; + + compression = mkOption { + # "auto" is optional, + # compression mode must be given, + # compression level is optional + type = types.strMatching "none|(auto,)?(lz4|zstd|zlib|lzma)(,[[:digit:]]{1,2})?"; + description = '' + Compression method to use. Refer to + borg help compression + for all available options. + ''; + default = "lz4"; + example = "auto,lzma"; + }; + + exclude = mkOption { + type = with types; listOf str; + description = '' + Exclude paths matching any of the given patterns. See + borg help patterns for pattern syntax. + ''; + default = [ ]; + example = [ + "/home/*/.cache" + "/nix" + ]; + }; + + readWritePaths = mkOption { + type = with types; listOf path; + description = '' + By default, borg cannot write anywhere on the system but + $HOME/.config/borg and $HOME/.cache/borg. + If, for example, your preHook script needs to dump files + somewhere, put those directories here. + ''; + default = [ ]; + example = [ + "/var/backup/mysqldump" + ]; + }; + + privateTmp = mkOption { + type = types.bool; + description = '' + Set the PrivateTmp option for + the systemd-service. Set to false if you need sockets + or other files from global /tmp. + ''; + default = true; + }; + + doInit = mkOption { + type = types.bool; + description = '' + Run borg init if the + specified does not exist. + You should set this to false + if the repository is located on an external drive + that might not always be mounted. + ''; + default = true; + }; + + appendFailedSuffix = mkOption { + type = types.bool; + description = '' + Append a .failed suffix + to the archive name, which is only removed if + borg create has a zero exit status. + ''; + default = true; + }; + + prune.keep = mkOption { + # Specifying e.g. `prune.keep.yearly = -1` + # means there is no limit of yearly archives to keep + # The regex is for use with e.g. --keep-within 1y + type = with types; attrsOf (either int (strMatching "[[:digit:]]+[Hdwmy]")); + description = '' + Prune a repository by deleting all archives not matching any of the + specified retention options. See borg help prune + for the available options. + ''; + default = { }; + example = literalExample '' + { + within = "1d"; # Keep all archives from the last day + daily = 7; + weekly = 4; + monthly = -1; # Keep at least one archive for each month + } + ''; + }; + + prune.prefix = mkOption { + type = types.str; + description = '' + Only consider archive names starting with this prefix for pruning. + By default, only archives created by this job are considered. + Use "" to consider all archives. + ''; + default = config.archiveBaseName; + defaultText = "\${archiveBaseName}"; + }; + + environment = mkOption { + type = with types; attrsOf str; + description = '' + Environment variables passed to the backup script. + You can for example specify which SSH key to use. + ''; + default = { }; + example = { BORG_RSH = "ssh -i /path/to/key"; }; + }; + + preHook = mkOption { + type = types.lines; + description = '' + Shell commands to run before the backup. + This can for example be used to mount file systems. + ''; + default = ""; + example = '' + # To add excluded paths at runtime + extraCreateArgs="$extraCreateArgs --exclude /some/path" + ''; + }; + + postInit = mkOption { + type = types.lines; + description = '' + Shell commands to run after borg init. + ''; + default = ""; + }; + + postCreate = mkOption { + type = types.lines; + description = '' + Shell commands to run after borg create. The name + of the created archive is stored in $archiveName. + ''; + default = ""; + }; + + postPrune = mkOption { + type = types.lines; + description = '' + Shell commands to run after borg prune. + ''; + default = ""; + }; + + postHook = mkOption { + type = types.lines; + description = '' + Shell commands to run just before exit. They are executed + even if a previous command exits with a non-zero exit code. + The latter is available as $exitStatus. + ''; + default = ""; + }; + + extraArgs = mkOption { + type = types.str; + description = '' + Additional arguments for all borg calls the + service has. Handle with care. + ''; + default = ""; + example = "--remote-path=/path/to/borg"; + }; + + extraInitArgs = mkOption { + type = types.str; + description = '' + Additional arguments for borg init. + Can also be set at runtime using $extraInitArgs. + ''; + default = ""; + example = "--append-only"; + }; + + extraCreateArgs = mkOption { + type = types.str; + description = '' + Additional arguments for borg create. + Can also be set at runtime using $extraCreateArgs. + ''; + default = ""; + example = "--stats --checkpoint-interval 600"; + }; + + extraPruneArgs = mkOption { + type = types.str; + description = '' + Additional arguments for borg prune. + Can also be set at runtime using $extraPruneArgs. + ''; + default = ""; + example = "--save-space"; + }; + + }; + } + )); + }; + + options.clerie.borgbackup.repos = mkOption { + description = '' + Serve BorgBackup repositories to given public SSH keys, + restricting their access to the repository only. + See also the chapter about BorgBackup in the NixOS manual. + Also, clients do not need to specify the absolute path when accessing the repository, + i.e. user@machine:. is enough. (Note colon and dot.) + ''; + default = { }; + type = types.attrsOf (types.submodule ( + { ... }: { + options = { + path = mkOption { + type = types.path; + description = '' + Where to store the backups. Note that the directory + is created automatically, with correct permissions. + ''; + default = "/var/lib/borgbackup"; + }; + + user = mkOption { + type = types.str; + description = '' + The user borg serve is run as. + User or group needs write permission + for the specified . + ''; + default = "borg"; + }; + + group = mkOption { + type = types.str; + description = '' + The group borg serve is run as. + User or group needs write permission + for the specified . + ''; + default = "borg"; + }; + + authorizedKeys = mkOption { + type = with types; listOf str; + description = '' + Public SSH keys that are given full write access to this repository. + You should use a different SSH key for each repository you write to, because + the specified keys are restricted to running borg serve + and can only access this single repository. + ''; + default = [ ]; + }; + + authorizedKeysAppendOnly = mkOption { + type = with types; listOf str; + description = '' + Public SSH keys that can only be used to append new data (archives) to the repository. + Note that archives can still be marked as deleted and are subsequently removed from disk + upon accessing the repo with full write access, e.g. when pruning. + ''; + default = [ ]; + }; + + allowSubRepos = mkOption { + type = types.bool; + description = '' + Allow clients to create repositories in subdirectories of the + specified . These can be accessed using + user@machine:path/to/subrepo. Note that a + applies to repositories independently. + Therefore, if this is enabled, clients can create multiple + repositories and upload an arbitrary amount of data. + ''; + default = false; + }; + + quota = mkOption { + # See the definition of parse_file_size() in src/borg/helpers/parseformat.py + type = with types; nullOr (strMatching "[[:digit:].]+[KMGTP]?"); + description = '' + Storage quota for the repository. This quota is ensured for all + sub-repositories if is enabled + but not for the overall storage space used. + ''; + default = null; + example = "100G"; + }; + + }; + } + )); + }; + + ###### implementation + + config = mkIf (with config.clerie.borgbackup; jobs != { } || repos != { }) + (with config.clerie.borgbackup; { + assertions = + mapAttrsToList mkPassAssertion jobs + ++ mapAttrsToList mkKeysAssertion repos + ++ mapAttrsToList mkRemovableDeviceAssertions jobs; + + system.activationScripts = mapAttrs' mkActivationScript jobs; + + systemd.services = + # A job named "foo" is mapped to systemd.services.borgbackup-job-foo + mapAttrs' mkBackupService jobs + # A repo named "foo" is mapped to systemd.services.borgbackup-repo-foo + // mapAttrs' mkRepoService repos; + + users = mkMerge (mapAttrsToList mkUsersConfig repos); + + environment.systemPackages = with pkgs; [ borgbackup ] ++ (mapAttrsToList mkBorgWrapper jobs); + }); +} diff --git a/modules/default.nix b/modules/default.nix index c72d097..25322e2 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -4,6 +4,7 @@ imports = [ ./policyrouting ./anycast_healthchecker + ./borgbackup ./gitea ./gre-tunnel ./minecraft-server