add kopia module

6 files changed, 647 insertions, 0 deletions

diff --git a/modules/backup/default.nix b/modules/backup/default.nix
new file mode 100644
index 0000000..ed319f2
--- /dev/null
+++ b/modules/backup/default.nix
@@ -0,0 +1,50 @@
+{
+  lib,
+  pkgs,
+  ...
+}:
+let
+  instanceType = lib.types.submodule {
+    options = {
+      enable = lib.mkEnableOption "Enable Kopia instance";
+      user = lib.mkOption {
+        type = lib.types.str;
+        default = "root";
+        description = "User under which the Kopia instance runs.";
+      };
+      environmentFile = lib.mkOption {
+        type = lib.types.nullOr lib.types.path;
+        default = null;
+        description = "File containing environment variables for kopia like password.";
+      };
+    };
+  };
+  mkInstanceServices =
+    instances:
+    serviceCreator:
+    lib.pipe instances [
+      (lib.attrsets.mapAttrs' serviceCreator)
+      (lib.recursiveUpdate { })
+    ];
+in
+{
+  imports = [
+    {
+      _module.args.mkInstanceServices = mkInstanceServices;
+      imports = [
+        ./repositories.nix
+        ./snapshot.nix
+        ./policy.nix
+        ./web.nix
+      ];
+    }
+  ];
+
+  options.services.kopia = {
+    enable = lib.mkEnableOption "Enable Kopia backup";
+    instances = lib.mkOption {
+      type = lib.types.attrsOf instanceType;
+      default = { };
+    };
+  };
+}
diff --git a/modules/backup/policy.nix b/modules/backup/policy.nix
new file mode 100644
index 0000000..5486cac
--- /dev/null
+++ b/modules/backup/policy.nix
@@ -0,0 +1,301 @@
+{
+  pkgs,
+  lib,
+  config,
+  mkInstanceServices,
+  ...
+}:
+let
+
+  # kopia policy json definition
+  compressionType = lib.types.enum [
+    "none"
+    "deflate-best-compression"
+    "deflate-best-speed"
+    "deflate-default"
+    "gzip"
+    "gzip-best-compression"
+    "gzip-best-speed"
+    "pgzip"
+    "pgzip-best-compression"
+    "pgzip-best-speed"
+    "s2-better"
+    "s2-default"
+    "s2-parallel-4"
+    "s2-parallel-8"
+    "zstd"
+    "zstd-better-compression"
+    "zstd-fastest"
+  ];
+
+  policyType = lib.mkOption {
+    type = lib.types.nullOr (
+      lib.types.submodule {
+        options = {
+          retention = {
+            keepLatest = lib.mkOption {
+              type = lib.types.int;
+              default = 5;
+              description = "Number of latest snapshots to keep.";
+            };
+            keepHourly = lib.mkOption {
+              type = lib.types.int;
+              default = 48;
+              description = "Number of hourly snapshots to keep.";
+            };
+            keepDaily = lib.mkOption {
+              type = lib.types.int;
+              default = 7;
+              description = "Number of daily snapshots to keep.";
+            };
+            keepWeekly = lib.mkOption {
+              type = lib.types.int;
+              default = 4;
+              description = "Number of weekly snapshots to keep.";
+            };
+            keepMonthly = lib.mkOption {
+              type = lib.types.int;
+              default = 3;
+              description = "Number of monthly snapshots to keep.";
+            };
+            keepAnnual = lib.mkOption {
+              type = lib.types.int;
+              default = 0;
+              description = "Number of yearly snapshots to keep.";
+            };
+          };
+
+          files = {
+            ignoreDotFiles = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              default = [
+                ".gitignore"
+                ".kopiaignore"
+              ];
+              description = "List of files to source ignore lists from.";
+            };
+            noParentDotFiles = lib.mkOption {
+              type = lib.types.nullOr lib.types.bool;
+              default = false;
+              description = "Do not use parent ignore dot files.";
+            };
+            ignoreCacheDirs = lib.mkOption {
+              type = lib.types.nullOr lib.types.bool;
+              default = false;
+              description = "Ignore cache directories.";
+            };
+            maxFileSize = lib.mkOption {
+              type = lib.types.nullOr lib.types.int;
+              default = null;
+              description = "Maximum file size to include in backup.";
+            };
+            oneFileSystem = lib.mkOption {
+              type = lib.types.nullOr lib.types.bool;
+              default = null;
+              description = "Stay in parent filesystem when finding files.";
+            };
+          };
+
+          errorHandling = {
+            ignoreFileErrors = lib.mkOption {
+              type = lib.types.nullOr lib.types.bool;
+              default = null;
+              description = "Ignore errors reading ignore files.";
+            };
+            ignoreDirectoryErrors = lib.mkOption {
+              type = lib.types.nullOr lib.types.bool;
+              default = null;
+              description = "Ignore errors reading directories.";
+            };
+            ignoreUnknownTypes = lib.mkOption {
+              type = lib.types.nullOr lib.types.bool;
+              default = null;
+              description = "Ignore unknown file types.";
+            };
+          };
+
+          compression = {
+            compressorName = lib.mkOption {
+              type = compressionType;
+              default = "none";
+              description = "Name of the compressor to use.";
+            };
+
+            onlyCompress = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              default = [ ];
+              description = "List of file extensions to compress.";
+            };
+
+            noParentOnlyCompress = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = "Do not use parent only compress list.";
+            };
+
+            neverCompress = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              default = [ ];
+              description = "List of file extensions to never compress.";
+            };
+
+            noParentNeverCompress = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = "Do not use parent never compress list.";
+            };
+
+            minSize = lib.mkOption {
+              type = lib.types.int;
+              default = 0;
+              description = "Minimum file size to compress.";
+            };
+
+            maxSize = lib.mkOption {
+              type = lib.types.int;
+              default = 0;
+              description = "Maximum file size to compress.";
+            };
+          };
+
+          metadataCompression = {
+            compressorName = lib.mkOption {
+              type = compressionType;
+              default = "zstd-fastest";
+              description = "Name of the compressor to use.";
+            };
+          };
+
+          splitter = {
+            algorithm = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              default = null;
+              description = "Name of the splitter algorithm to use.";
+            };
+          };
+
+          # FIXME: add action definition afterward (maybe implement it during implement at zfs, btrfs snapshot)
+
+          osSnapshots = {
+            volumeShadowCopy = {
+              enable = lib.mkOption {
+                type = lib.types.nullOr (
+                  lib.types.enum [
+                    "never"
+                    "always"
+                    "when-available"
+                    "inherit"
+                  ]
+                );
+                default = null;
+                description = "Enable volume shadow copy";
+              };
+            };
+          };
+
+          logging = {
+            directories = {
+              snapshotted = lib.mkOption {
+                type = lib.types.nullOr lib.types.int;
+                default = null;
+                description = "Log detail when a directory is snapshotted";
+              };
+              ignored = lib.mkOption {
+                type = lib.types.nullOr lib.types.int;
+                default = null;
+                description = "Log detail when a directory is ignored";
+              };
+            };
+
+            entries = {
+              snapshotted = lib.mkOption {
+                type = lib.types.nullOr lib.types.int;
+                default = null;
+                description = "Log detail when an entry is snapshotted";
+              };
+              ignored = lib.mkOption {
+                type = lib.types.nullOr lib.types.int;
+                default = null;
+                description = "Log detail when an entry is ignored";
+              };
+            };
+          };
+
+          upload = {
+            # maxParallelSnapshots - GUI only
+            maxParallelFileReads = lib.mkOption {
+              type = lib.types.nullOr lib.types.int;
+              default = null;
+              description = "Maximum number of parallel file reads(GUI Only)";
+            };
+            parallelUploadAboveSize = lib.mkOption {
+              type = lib.types.nullOr lib.types.int;
+              default = null;
+              description = "Use parallel uploads above size(GUI Only)";
+            };
+          };
+        };
+      }
+    );
+    default = null;
+  };
+
+  instanceType = lib.types.submodule {
+    options = {
+      policy = policyType;
+    };
+  };
+
+  # application logic
+  jsonFormat = (pkgs.formats.json { });
+
+  # generate policy name for policy file generation
+  mkPolicyName =
+    user: hostname: path:
+    "${user}@${hostname}${if path != "" then ":${path}" else ""}";
+
+  mkPolicyFile = policy: (jsonFormat.generate "kopia-policy.json" policy);
+
+  mkInstancePolicyService =
+    name: instance:
+    let
+      policyName = mkPolicyName instance.user config.networking.hostName instance.path;
+      policyFile = mkPolicyFile (
+        {
+          "${policyName}" = instance.policy;
+        }
+        // lib.optionalAttrs (config.services.kopia.globalPolicy != null) {
+          "(global)" = config.services.kopia.globalPolicy;
+        }
+      );
+    in
+    (lib.attrsets.nameValuePair "kopia-policy-${name}" {
+      description = "Kopia policy setup";
+      wants = [ "kopia-repository-${name}.service" ];
+      wantedBy = [ "kopia-snapshot-${name}.service" ];
+      after = [ "kopia-repository-${name}.service" ];
+      before = [ "kopia-snapshot-${name}.service" ];
+      script = ''
+        ${pkgs.kopia}/bin/kopia policy import --from-file=${policyFile}
+      '';
+      serviceConfig = {
+        Type = "oneshot";
+        User = instance.user;
+        WorkingDirectory = "~";
+        SetLoginEnvironment = true;
+      };
+    });
+in
+{
+  options.services.kopia = {
+    globalPolicy = policyType;
+
+    instances = lib.mkOption {
+      type = lib.types.attrsOf instanceType;
+    };
+  };
+
+  config = lib.mkIf config.services.kopia.enable {
+    systemd.services = mkInstanceServices config.services.kopia.instances mkInstancePolicyService;
+  };
+}
diff --git a/modules/backup/repositories.nix b/modules/backup/repositories.nix
new file mode 100644
index 0000000..9ee2b3e
--- /dev/null
+++ b/modules/backup/repositories.nix
@@ -0,0 +1,144 @@
+{
+  pkgs,
+  lib,
+  config,
+  mkInstanceServices,
+  ...
+}:
+let
+  s3RepositoryType = lib.types.submodule {
+    options = {
+      bucket = lib.mkOption {
+        type = lib.types.str;
+        default = "default-bucket-value";
+        description = "Bucket name for S3 repository.";
+      };
+      region = lib.mkOption {
+        type = lib.types.str;
+        default = "eu-central-003";
+        description = "Region for S3 repository.";
+      };
+      endpoint = lib.mkOption {
+        type = lib.types.str;
+        default = "https://s3.eu-central-003.backblazeb2.com";
+        description = "Endpoint for S3 repository.";
+      };
+      disableTLS = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Disable TLS for S3 repository.";
+      };
+    };
+  };
+
+  b2RepositoryType = lib.types.submodule {
+    options = {
+      bucket = lib.mkOption {
+        type = lib.types.str;
+        default = "default-bucket-value";
+        description = "Bucket name for S3 repository.";
+      };
+    };
+  };
+
+  instanceType = lib.types.submodule {
+    options = {
+      repository = lib.mkOption {
+        type = lib.types.attrTag {
+          s3 = lib.mkOption {
+            type = s3RepositoryType;
+          };
+          b2 = lib.mkOption {
+            type = b2RepositoryType;
+          };
+        };
+      };
+    };
+  };
+in
+{
+  options.services.kopia.instances = lib.mkOption {
+    type = lib.types.attrsOf instanceType;
+  };
+
+  config = lib.mkIf config.services.kopia.enable {
+    # systemd service for repositories open
+    systemd.services =
+      let
+        mkRepositoryArgs =
+          name: instance:
+          (
+            if lib.hasAttr "s3" instance.repository then
+              [
+                "--bucket"
+                instance.repository.s3.bucket
+                "--endpoint"
+                instance.repository.s3.endpoint
+                "--region"
+                instance.repository.s3.region
+              ]
+              ++ (lib.optional (instance.repository.s3.disableTLS) "--disable-tls")
+            else if lib.hasAttr "b2" instance.repository then
+              [ "--bucket" instance.repository.s3.bucket ]
+            else
+              throw "Unsupported repository type for Kopia instance ${name}"
+          );
+
+        mkRepository =
+          let
+            mkTemplateRepository = 
+              type: envs: name: instance:
+              lib.attrsets.nameValuePair "kopia-repository-${name}" {
+                description = "Kopia ${type} repository service";
+                serviceConfig =
+                  let
+                    startScript = pkgs.writeShellScript "start-repository.sh" ''
+                      # Check required environment variables
+                      for var in KOPIA_PASSWORD ${toString envs}; do
+                        if [[ -z "''${!var}" ]]; then
+                          echo "''$var is not set, exiting."
+                          exit 1
+                        fi
+                      done
+
+                      if ! ${pkgs.kopia}/bin/kopia repository connect ${type} ${lib.concatStringsSep " " (mkRepositoryArgs name instance)}; then
+                        ${pkgs.kopia}/bin/kopia repository create ${type} ${lib.concatStringsSep " " (mkRepositoryArgs name instance)};
+                      fi
+                    '';
+
+                    stopScript = pkgs.writeShellScript "stop-repository.sh" ''
+                      ${pkgs.kopia}/bin/kopia repository disconnect
+                    '';
+                  in
+                  {
+                    Type = "oneshot";
+                    User = "${instance.user}";
+                    WorkingDirectory = "~";
+                    SetLoginEnvironment = true;
+                    EnvironmentFile = lib.mkIf (instance.environmentFile != null) instance.environmentFile;
+                    RemainAfterExit = true;
+                    ExecStart = "${startScript}";
+                    ExecStop = "${stopScript}";
+                  };
+              };
+
+            mkS3Repository = mkTemplateRepository "S3" [ "AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY" ];
+            mkB2Repository = mkTemplateRepository "B2" [ "B2_KEY_ID" "B2_KEY" ];
+
+            dispatch = {
+              s3 = mkS3Repository;
+              b2 = mkB2Repository;
+            };
+          in
+          name: instance:
+          let
+            repoType = builtins.head (lib.attrNames instance.repository);
+          in
+          if lib.hasAttr repoType dispatch then
+            dispatch.${repoType} name instance
+          else
+            throw "Unsupported repository type for Kopia instance ${name}";
+      in
+      mkInstanceServices config.services.kopia.instances mkRepository;
+  };
+}
diff --git a/modules/backup/snapshot.nix b/modules/backup/snapshot.nix
new file mode 100644
index 0000000..91076cf
--- /dev/null
+++ b/modules/backup/snapshot.nix
@@ -0,0 +1,76 @@
+{
+  pkgs,
+  lib,
+  config,
+  mkInstanceServices,
+  ...
+}:
+let
+  instanceType = lib.types.submodule {
+    options = {
+      path = lib.mkOption {
+        type = lib.types.str;
+        default = "/persistent";
+        description = "snapshoted path for kopia instance.";
+      };
+      schedule = lib.mkOption {
+        type = lib.types.str;
+        default = "daily";
+        description = "Snapshot schedule for the Kopia instance.";
+      };
+    };
+  };
+in
+{
+  options.services.kopia.instances = lib.mkOption {
+    type = lib.types.attrsOf instanceType;
+  };
+
+  config = lib.mkIf config.services.kopia.enable (
+    let
+      mkSnapshotService =
+        # refactor with mkRepositoryArgs
+        name: instance:
+        lib.attrsets.nameValuePair "kopia-snapshot-${name}" {
+          description = "Kopia S3 snapshot service";
+          wants = [
+            "kopia-repository-${name}.service"
+          ];
+          after = [ "kopia-repository-${name}.service" ];
+          script = ''
+            ${pkgs.kopia}/bin/kopia snapshot create ${instance.path} --description "Snapshot for ${name}"
+          '';
+          serviceConfig = {
+            Type = "simple";
+            User = "${instance.user}";
+            WorkingDirectory = "~";
+            SetLoginEnvironment = true;
+            # retry on failure
+            Restart = "on-failure";
+            # wait 30 seconds before restarting
+            RestartSec = "30";
+            # limit the number of restarts to 5 in 1 day
+            StartLimitInterval = "1d";
+            StartLimitBurst = "5";
+            # lower priority
+            Nice = "-19";
+            IOSchedulingClass = "idle";
+          };
+        };
+      mkSnapshotTimer =
+        name: instance:
+        lib.attrsets.nameValuePair "kopia-snapshot-${name}" {
+          description = "Kopia S3 snapshot timer";
+          wantedBy = [ "timers.target" ];
+          timerConfig = {
+            OnCalendar = instance.schedule;
+          };
+        };
+    in
+    {
+      # systemd service for repositories open
+      systemd.services = mkInstanceServices config.services.kopia.instances mkSnapshotService;
+      systemd.timers = mkInstanceServices config.services.kopia.instances mkSnapshotTimer;
+    }
+  );
+}
diff --git a/modules/backup/web.nix b/modules/backup/web.nix
new file mode 100644
index 0000000..9937315
--- /dev/null
+++ b/modules/backup/web.nix
@@ -0,0 +1,75 @@
+{
+  pkgs,
+  lib,
+  config,
+  mkInstanceServices,
+  ...
+}:
+let
+  instanceType = lib.types.submodule {
+    options = {
+      web = {
+        enable = lib.mkEnableOption "enable Kopia web interface";
+        guiAddress = lib.mkOption {
+          type = lib.types.str;
+          default = "127.0.0.1:51515";
+        };
+        serverUsername = lib.mkOption {
+          type = lib.types.str;
+          default = "admin";
+          description = "Username for the Kopia web server(basic auth).";
+        };
+        environmentFile = lib.mkOption {
+          type = lib.types.nullOr lib.types.path;
+          default = null;
+          description = "File containing environment variables for kopia web server like password.";
+        };
+      };
+    };
+  };
+in
+{
+  options.services.kopia.instances = lib.mkOption {
+    type = lib.types.attrsOf instanceType;
+  };
+
+  config = lib.mkIf config.services.kopia.enable {
+    # systemd service for repositories open
+    systemd.services =
+      let
+        mkWebService =
+          # refactor with mkRepositoryArgs
+          name: instance:
+          lib.attrsets.nameValuePair "kopia-web-${name}" {
+            description = "Kopia S3 web service";
+            wants = [
+              "kopia-repository-${name}.service"
+            ];
+            after = [ "kopia-repository-${name}.service" ];
+            environment = {
+              KOPIA_SERVER_USERNAME = instance.web.serverUsername;
+            };
+            script = ''
+              export KOPIA_SERVER_USERNAME=${instance.web.serverUsername}
+              # Start Kopia web server
+              ${pkgs.kopia}/bin/kopia server start --insecure --address ${instance.web.guiAddress}
+            '';
+            serviceConfig = {
+              Type = "simple";
+              User = "${instance.user}";
+              WorkingDirectory = "~";
+              SetLoginEnvironment = true;
+              EnvironmentFile = lib.mkIf (instance.web.environmentFile != null) instance.web.environmentFile;
+              # retry on failure
+              Restart = "on-failure";
+              # wait 30 seconds before restarting
+              RestartSec = "30";
+              # limit the number of restarts to 5 in 1 day
+              StartLimitIntervalSec = "1d";
+              StartLimitBurst = "5";
+            };
+          };
+      in
+      mkInstanceServices config.services.kopia.instances mkWebService;
+  };
+}
diff --git a/modules/default.nix b/modules/default.nix
index afabc28..8deed27 100644
--- a/modules/default.nix
+++ b/modules/default.nix
@@ -1,5 +1,6 @@
 {
   bgp = import ./bgp;
+  backup = import ./backup;
   common = import ./common;
   crypto = import ./crypto;
   git = import ./git;