init

1 files changed, 160 insertions, 0 deletions

diff --git a/modules/kubernetes/default.nix b/modules/kubernetes/default.nix
new file mode 100644
index 0000000..0e6e522
--- /dev/null
+++ b/modules/kubernetes/default.nix
@@ -0,0 +1,160 @@
+{ config, lib, pkgs, nixpkgs, ... }:
+
+with lib;
+let
+  cfg = config.kubernetes;
+
+  clusterDomain = "kube.vapor.systems";
+
+  externalIP = {
+    v4 = if cfg.externalIP.v4 != "" then
+      cfg.externalIP.v4
+    else
+      (lib.elemAt config.networking.interfaces.eth0.ipv4.addresses 0).address;
+    v6 = if cfg.externalIP.v6 != "" then
+      cfg.externalIP.v6
+    else
+      (lib.elemAt config.networking.interfaces.eth0.ipv6.addresses 0).address;
+  };
+  internalIP = {
+    v4 = config.wireguard.v4.address;
+    v6 = "${config.wireguard.v6.ula}::${config.wireguard.v6.address}";
+  };
+in {
+  disabledModules =
+    [ "virtualisation/cri-o.nix" "services/cluster/k3s/default.nix" ];
+  imports = [ ../cri-o ../k3s ];
+
+  options = {
+    kubernetes = {
+      role = mkOption {
+        type = types.enum [ "server" "agent" ];
+        description = "Act as control plane or worker node";
+      };
+
+      labels = mkOption {
+        type = types.attrs;
+        description = "Address the k8s api is advertised on";
+        default = { };
+      };
+      taints = mkOption {
+        type = types.attrs;
+        description = "Address the k8s api is advertised on";
+        default = { };
+      };
+
+      advertiseAddress = mkOption {
+        type = types.str;
+        description = "Address the k8s api is advertised on";
+      };
+
+      externalIP = {
+        v4 = mkOption {
+          type = types.str;
+          description = "External Node IP Address";
+          default = "";
+        };
+        v6 = mkOption {
+          type = types.str;
+          description = "External Node IP Address";
+          default = "";
+        };
+      };
+    };
+  };
+
+  config = {
+    nixpkgs.overlays = [
+      (self: super: {
+        cri-o = super.callPackage ../../pkgs/cri-o { };
+        k3s = super.callPackage ../../pkgs/k3s { };
+      })
+    ];
+
+    networking.extraHosts = ''
+      10.10.0.1 ${clusterDomain}
+      fd15:3d8c:d429:beef::1 ${clusterDomain}
+    '';
+
+    environment.etc = {
+      "k3s/config.yaml" = {
+        text = generators.toJSON { } ({
+          # cluster-init = true;
+          token = "YPoyiPeBpQpB7oK8";
+
+          container-runtime-endpoint = "/run/crio/crio.sock";
+
+          node-ip = "${internalIP.v4},${internalIP.v6}";
+          node-external-ip = "${internalIP.v4},${internalIP.v6}";
+
+          node-label =
+            attrValues (mapAttrs (n: v: "${n}=${toString v}") cfg.labels);
+          node-taint =
+            attrValues (mapAttrs (n: v: "${n}=${toString v}") cfg.taints);
+
+          kubelet-arg = "cgroup-driver=systemd";
+
+          no-flannel = true;
+        } // (if cfg.role == "server" then {
+          advertise-address = "${internalIP.v4}";
+
+          kube-controller-manager-arg = "node-cidr-mask-size-ipv6=72";
+
+          cluster-cidr = "10.102.0.0/16,fd15:3d8c:d429:0102::/64";
+          service-cidr = "10.101.0.0/16,fd15:3d8c:d429:0101::/108";
+          cluster-dns = "10.101.0.10";
+          cluster-domain = clusterDomain;
+
+          disable = [ "servicelb" "traefik" "local-storage" ];
+          disable-kube-proxy = true;
+          disable-network-policy = true;
+
+          flannel-backend = "none";
+        } else
+          { }));
+      };
+    };
+
+    services.k3s = {
+      enable = true;
+      role = cfg.role;
+      serverAddr = "https://10.10.0.1:6443";
+      configPath = "/etc/k3s/config.yaml";
+    };
+
+    virtualisation.cri-o = {
+      enable = true;
+      settings = {
+        crio = {
+          network.plugin_dir = "/opt/cni/bin";
+          default_runtime = "crun";
+          runtime = {
+            allowed_devices = [ "/dev/fuse" ];
+            default_sysctls = [
+              "net.ipv4.ping_group_range=0 2147483647"
+            ];
+            workloads = {
+              gitlab = {
+                activation_annotation = "io.kubernetes.cri-o.workload/gitlab";
+                allowed_annotations = [
+                  "io.kubernetes.cri-o.userns-mode"
+                  "io.kubernetes.cri-o.Devices"
+                  "io.kubernetes.cri-o.ShmSize"
+                ];
+              };
+            };
+            runtimes.crun = {
+              runtime_type = "oci";
+              runtime_root = "/run/crun";
+              allowed_annotations = [
+                "io.kubernetes.cri-o.userns-mode"
+                "io.kubernetes.cri-o.Devices"
+                "io.kubernetes.cri-o.ShmSize"
+              ];
+            };
+          };
+        };
+      };
+    };
+  };
+}