diff options
| -rw-r--r-- | modules/backup/default.nix | 50 | ||||
| -rw-r--r-- | modules/backup/policy.nix | 301 | ||||
| -rw-r--r-- | modules/backup/repositories.nix | 144 | ||||
| -rw-r--r-- | modules/backup/snapshot.nix | 76 | ||||
| -rw-r--r-- | modules/backup/web.nix | 75 | ||||
| -rw-r--r-- | modules/default.nix | 1 |
6 files changed, 647 insertions, 0 deletions
diff --git a/modules/backup/default.nix b/modules/backup/default.nix new file mode 100644 index 0000000..ed319f2 --- /dev/null +++ b/modules/backup/default.nix @@ -0,0 +1,50 @@ +{ + lib, + pkgs, + ... +}: +let + instanceType = lib.types.submodule { + options = { + enable = lib.mkEnableOption "Enable Kopia instance"; + user = lib.mkOption { + type = lib.types.str; + default = "root"; + description = "User under which the Kopia instance runs."; + }; + environmentFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = "File containing environment variables for kopia like password."; + }; + }; + }; + mkInstanceServices = + instances: + serviceCreator: + lib.pipe instances [ + (lib.attrsets.mapAttrs' serviceCreator) + (lib.recursiveUpdate { }) + ]; +in +{ + imports = [ + { + _module.args.mkInstanceServices = mkInstanceServices; + imports = [ + ./repositories.nix + ./snapshot.nix + ./policy.nix + ./web.nix + ]; + } + ]; + + options.services.kopia = { + enable = lib.mkEnableOption "Enable Kopia backup"; + instances = lib.mkOption { + type = lib.types.attrsOf instanceType; + default = { }; + }; + }; +} diff --git a/modules/backup/policy.nix b/modules/backup/policy.nix new file mode 100644 index 0000000..5486cac --- /dev/null +++ b/modules/backup/policy.nix @@ -0,0 +1,301 @@ +{ + pkgs, + lib, + config, + mkInstanceServices, + ... +}: +let + + # kopia policy json definition + compressionType = lib.types.enum [ + "none" + "deflate-best-compression" + "deflate-best-speed" + "deflate-default" + "gzip" + "gzip-best-compression" + "gzip-best-speed" + "pgzip" + "pgzip-best-compression" + "pgzip-best-speed" + "s2-better" + "s2-default" + "s2-parallel-4" + "s2-parallel-8" + "zstd" + "zstd-better-compression" + "zstd-fastest" + ]; + + policyType = lib.mkOption { + type = lib.types.nullOr ( + lib.types.submodule { + options = { + retention = { + keepLatest = lib.mkOption { + type = lib.types.int; + default = 5; + description = "Number of latest snapshots to keep."; + }; + keepHourly = lib.mkOption { + type = lib.types.int; + default = 48; + description = "Number of hourly snapshots to keep."; + }; + keepDaily = lib.mkOption { + type = lib.types.int; + default = 7; + description = "Number of daily snapshots to keep."; + }; + keepWeekly = lib.mkOption { + type = lib.types.int; + default = 4; + description = "Number of weekly snapshots to keep."; + }; + keepMonthly = lib.mkOption { + type = lib.types.int; + default = 3; + description = "Number of monthly snapshots to keep."; + }; + keepAnnual = lib.mkOption { + type = lib.types.int; + default = 0; + description = "Number of yearly snapshots to keep."; + }; + }; + + files = { + ignoreDotFiles = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ + ".gitignore" + ".kopiaignore" + ]; + description = "List of files to source ignore lists from."; + }; + noParentDotFiles = lib.mkOption { + type = lib.types.nullOr lib.types.bool; + default = false; + description = "Do not use parent ignore dot files."; + }; + ignoreCacheDirs = lib.mkOption { + type = lib.types.nullOr lib.types.bool; + default = false; + description = "Ignore cache directories."; + }; + maxFileSize = lib.mkOption { + type = lib.types.nullOr lib.types.int; + default = null; + description = "Maximum file size to include in backup."; + }; + oneFileSystem = lib.mkOption { + type = lib.types.nullOr lib.types.bool; + default = null; + description = "Stay in parent filesystem when finding files."; + }; + }; + + errorHandling = { + ignoreFileErrors = lib.mkOption { + type = lib.types.nullOr lib.types.bool; + default = null; + description = "Ignore errors reading ignore files."; + }; + ignoreDirectoryErrors = lib.mkOption { + type = lib.types.nullOr lib.types.bool; + default = null; + description = "Ignore errors reading directories."; + }; + ignoreUnknownTypes = lib.mkOption { + type = lib.types.nullOr lib.types.bool; + default = null; + description = "Ignore unknown file types."; + }; + }; + + compression = { + compressorName = lib.mkOption { + type = compressionType; + default = "none"; + description = "Name of the compressor to use."; + }; + + onlyCompress = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "List of file extensions to compress."; + }; + + noParentOnlyCompress = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Do not use parent only compress list."; + }; + + neverCompress = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "List of file extensions to never compress."; + }; + + noParentNeverCompress = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Do not use parent never compress list."; + }; + + minSize = lib.mkOption { + type = lib.types.int; + default = 0; + description = "Minimum file size to compress."; + }; + + maxSize = lib.mkOption { + type = lib.types.int; + default = 0; + description = "Maximum file size to compress."; + }; + }; + + metadataCompression = { + compressorName = lib.mkOption { + type = compressionType; + default = "zstd-fastest"; + description = "Name of the compressor to use."; + }; + }; + + splitter = { + algorithm = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Name of the splitter algorithm to use."; + }; + }; + + # FIXME: add action definition afterward (maybe implement it during implement at zfs, btrfs snapshot) + + osSnapshots = { + volumeShadowCopy = { + enable = lib.mkOption { + type = lib.types.nullOr ( + lib.types.enum [ + "never" + "always" + "when-available" + "inherit" + ] + ); + default = null; + description = "Enable volume shadow copy"; + }; + }; + }; + + logging = { + directories = { + snapshotted = lib.mkOption { + type = lib.types.nullOr lib.types.int; + default = null; + description = "Log detail when a directory is snapshotted"; + }; + ignored = lib.mkOption { + type = lib.types.nullOr lib.types.int; + default = null; + description = "Log detail when a directory is ignored"; + }; + }; + + entries = { + snapshotted = lib.mkOption { + type = lib.types.nullOr lib.types.int; + default = null; + description = "Log detail when an entry is snapshotted"; + }; + ignored = lib.mkOption { + type = lib.types.nullOr lib.types.int; + default = null; + description = "Log detail when an entry is ignored"; + }; + }; + }; + + upload = { + # maxParallelSnapshots - GUI only + maxParallelFileReads = lib.mkOption { + type = lib.types.nullOr lib.types.int; + default = null; + description = "Maximum number of parallel file reads(GUI Only)"; + }; + parallelUploadAboveSize = lib.mkOption { + type = lib.types.nullOr lib.types.int; + default = null; + description = "Use parallel uploads above size(GUI Only)"; + }; + }; + }; + } + ); + default = null; + }; + + instanceType = lib.types.submodule { + options = { + policy = policyType; + }; + }; + + # application logic + jsonFormat = (pkgs.formats.json { }); + + # generate policy name for policy file generation + mkPolicyName = + user: hostname: path: + "${user}@${hostname}${if path != "" then ":${path}" else ""}"; + + mkPolicyFile = policy: (jsonFormat.generate "kopia-policy.json" policy); + + mkInstancePolicyService = + name: instance: + let + policyName = mkPolicyName instance.user config.networking.hostName instance.path; + policyFile = mkPolicyFile ( + { + "${policyName}" = instance.policy; + } + // lib.optionalAttrs (config.services.kopia.globalPolicy != null) { + "(global)" = config.services.kopia.globalPolicy; + } + ); + in + (lib.attrsets.nameValuePair "kopia-policy-${name}" { + description = "Kopia policy setup"; + wants = [ "kopia-repository-${name}.service" ]; + wantedBy = [ "kopia-snapshot-${name}.service" ]; + after = [ "kopia-repository-${name}.service" ]; + before = [ "kopia-snapshot-${name}.service" ]; + script = '' + ${pkgs.kopia}/bin/kopia policy import --from-file=${policyFile} + ''; + serviceConfig = { + Type = "oneshot"; + User = instance.user; + WorkingDirectory = "~"; + SetLoginEnvironment = true; + }; + }); +in +{ + options.services.kopia = { + globalPolicy = policyType; + + instances = lib.mkOption { + type = lib.types.attrsOf instanceType; + }; + }; + + config = lib.mkIf config.services.kopia.enable { + systemd.services = mkInstanceServices config.services.kopia.instances mkInstancePolicyService; + }; +} diff --git a/modules/backup/repositories.nix b/modules/backup/repositories.nix new file mode 100644 index 0000000..9ee2b3e --- /dev/null +++ b/modules/backup/repositories.nix @@ -0,0 +1,144 @@ +{ + pkgs, + lib, + config, + mkInstanceServices, + ... +}: +let + s3RepositoryType = lib.types.submodule { + options = { + bucket = lib.mkOption { + type = lib.types.str; + default = "default-bucket-value"; + description = "Bucket name for S3 repository."; + }; + region = lib.mkOption { + type = lib.types.str; + default = "eu-central-003"; + description = "Region for S3 repository."; + }; + endpoint = lib.mkOption { + type = lib.types.str; + default = "https://s3.eu-central-003.backblazeb2.com"; + description = "Endpoint for S3 repository."; + }; + disableTLS = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Disable TLS for S3 repository."; + }; + }; + }; + + b2RepositoryType = lib.types.submodule { + options = { + bucket = lib.mkOption { + type = lib.types.str; + default = "default-bucket-value"; + description = "Bucket name for S3 repository."; + }; + }; + }; + + instanceType = lib.types.submodule { + options = { + repository = lib.mkOption { + type = lib.types.attrTag { + s3 = lib.mkOption { + type = s3RepositoryType; + }; + b2 = lib.mkOption { + type = b2RepositoryType; + }; + }; + }; + }; + }; +in +{ + options.services.kopia.instances = lib.mkOption { + type = lib.types.attrsOf instanceType; + }; + + config = lib.mkIf config.services.kopia.enable { + # systemd service for repositories open + systemd.services = + let + mkRepositoryArgs = + name: instance: + ( + if lib.hasAttr "s3" instance.repository then + [ + "--bucket" + instance.repository.s3.bucket + "--endpoint" + instance.repository.s3.endpoint + "--region" + instance.repository.s3.region + ] + ++ (lib.optional (instance.repository.s3.disableTLS) "--disable-tls") + else if lib.hasAttr "b2" instance.repository then + [ "--bucket" instance.repository.s3.bucket ] + else + throw "Unsupported repository type for Kopia instance ${name}" + ); + + mkRepository = + let + mkTemplateRepository = + type: envs: name: instance: + lib.attrsets.nameValuePair "kopia-repository-${name}" { + description = "Kopia ${type} repository service"; + serviceConfig = + let + startScript = pkgs.writeShellScript "start-repository.sh" '' + # Check required environment variables + for var in KOPIA_PASSWORD ${toString envs}; do + if [[ -z "''${!var}" ]]; then + echo "''$var is not set, exiting." + exit 1 + fi + done + + if ! ${pkgs.kopia}/bin/kopia repository connect ${type} ${lib.concatStringsSep " " (mkRepositoryArgs name instance)}; then + ${pkgs.kopia}/bin/kopia repository create ${type} ${lib.concatStringsSep " " (mkRepositoryArgs name instance)}; + fi + ''; + + stopScript = pkgs.writeShellScript "stop-repository.sh" '' + ${pkgs.kopia}/bin/kopia repository disconnect + ''; + in + { + Type = "oneshot"; + User = "${instance.user}"; + WorkingDirectory = "~"; + SetLoginEnvironment = true; + EnvironmentFile = lib.mkIf (instance.environmentFile != null) instance.environmentFile; + RemainAfterExit = true; + ExecStart = "${startScript}"; + ExecStop = "${stopScript}"; + }; + }; + + mkS3Repository = mkTemplateRepository "S3" [ "AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY" ]; + mkB2Repository = mkTemplateRepository "B2" [ "B2_KEY_ID" "B2_KEY" ]; + + dispatch = { + s3 = mkS3Repository; + b2 = mkB2Repository; + }; + in + name: instance: + let + repoType = builtins.head (lib.attrNames instance.repository); + in + if lib.hasAttr repoType dispatch then + dispatch.${repoType} name instance + else + throw "Unsupported repository type for Kopia instance ${name}"; + in + mkInstanceServices config.services.kopia.instances mkRepository; + }; +} diff --git a/modules/backup/snapshot.nix b/modules/backup/snapshot.nix new file mode 100644 index 0000000..91076cf --- /dev/null +++ b/modules/backup/snapshot.nix @@ -0,0 +1,76 @@ +{ + pkgs, + lib, + config, + mkInstanceServices, + ... +}: +let + instanceType = lib.types.submodule { + options = { + path = lib.mkOption { + type = lib.types.str; + default = "/persistent"; + description = "snapshoted path for kopia instance."; + }; + schedule = lib.mkOption { + type = lib.types.str; + default = "daily"; + description = "Snapshot schedule for the Kopia instance."; + }; + }; + }; +in +{ + options.services.kopia.instances = lib.mkOption { + type = lib.types.attrsOf instanceType; + }; + + config = lib.mkIf config.services.kopia.enable ( + let + mkSnapshotService = + # refactor with mkRepositoryArgs + name: instance: + lib.attrsets.nameValuePair "kopia-snapshot-${name}" { + description = "Kopia S3 snapshot service"; + wants = [ + "kopia-repository-${name}.service" + ]; + after = [ "kopia-repository-${name}.service" ]; + script = '' + ${pkgs.kopia}/bin/kopia snapshot create ${instance.path} --description "Snapshot for ${name}" + ''; + serviceConfig = { + Type = "simple"; + User = "${instance.user}"; + WorkingDirectory = "~"; + SetLoginEnvironment = true; + # retry on failure + Restart = "on-failure"; + # wait 30 seconds before restarting + RestartSec = "30"; + # limit the number of restarts to 5 in 1 day + StartLimitInterval = "1d"; + StartLimitBurst = "5"; + # lower priority + Nice = "-19"; + IOSchedulingClass = "idle"; + }; + }; + mkSnapshotTimer = + name: instance: + lib.attrsets.nameValuePair "kopia-snapshot-${name}" { + description = "Kopia S3 snapshot timer"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = instance.schedule; + }; + }; + in + { + # systemd service for repositories open + systemd.services = mkInstanceServices config.services.kopia.instances mkSnapshotService; + systemd.timers = mkInstanceServices config.services.kopia.instances mkSnapshotTimer; + } + ); +} diff --git a/modules/backup/web.nix b/modules/backup/web.nix new file mode 100644 index 0000000..9937315 --- /dev/null +++ b/modules/backup/web.nix @@ -0,0 +1,75 @@ +{ + pkgs, + lib, + config, + mkInstanceServices, + ... +}: +let + instanceType = lib.types.submodule { + options = { + web = { + enable = lib.mkEnableOption "enable Kopia web interface"; + guiAddress = lib.mkOption { + type = lib.types.str; + default = "127.0.0.1:51515"; + }; + serverUsername = lib.mkOption { + type = lib.types.str; + default = "admin"; + description = "Username for the Kopia web server(basic auth)."; + }; + environmentFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = "File containing environment variables for kopia web server like password."; + }; + }; + }; + }; +in +{ + options.services.kopia.instances = lib.mkOption { + type = lib.types.attrsOf instanceType; + }; + + config = lib.mkIf config.services.kopia.enable { + # systemd service for repositories open + systemd.services = + let + mkWebService = + # refactor with mkRepositoryArgs + name: instance: + lib.attrsets.nameValuePair "kopia-web-${name}" { + description = "Kopia S3 web service"; + wants = [ + "kopia-repository-${name}.service" + ]; + after = [ "kopia-repository-${name}.service" ]; + environment = { + KOPIA_SERVER_USERNAME = instance.web.serverUsername; + }; + script = '' + export KOPIA_SERVER_USERNAME=${instance.web.serverUsername} + # Start Kopia web server + ${pkgs.kopia}/bin/kopia server start --insecure --address ${instance.web.guiAddress} + ''; + serviceConfig = { + Type = "simple"; + User = "${instance.user}"; + WorkingDirectory = "~"; + SetLoginEnvironment = true; + EnvironmentFile = lib.mkIf (instance.web.environmentFile != null) instance.web.environmentFile; + # retry on failure + Restart = "on-failure"; + # wait 30 seconds before restarting + RestartSec = "30"; + # limit the number of restarts to 5 in 1 day + StartLimitIntervalSec = "1d"; + StartLimitBurst = "5"; + }; + }; + in + mkInstanceServices config.services.kopia.instances mkWebService; + }; +} diff --git a/modules/default.nix b/modules/default.nix index afabc28..8deed27 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -1,5 +1,6 @@ { bgp = import ./bgp; + backup = import ./backup; common = import ./common; crypto = import ./crypto; git = import ./git; |