{ config, lib, pkgs, nixpkgs, ... }: with lib; let cfg = config.kubernetes; clusterDomain = "kube.vapor.systems"; externalIP = { v4 = if cfg.externalIP.v4 != "" then cfg.externalIP.v4 else (lib.elemAt config.networking.interfaces.eth0.ipv4.addresses 0).address; v6 = if cfg.externalIP.v6 != "" then cfg.externalIP.v6 else (lib.elemAt config.networking.interfaces.eth0.ipv6.addresses 0).address; }; internalIP = { v4 = config.wireguard.v4.address; v6 = "${config.wireguard.v6.ula}::${config.wireguard.v6.address}"; }; in { disabledModules = [ "virtualisation/cri-o.nix" "services/cluster/k3s/default.nix" ]; imports = [ ../cri-o ../k3s ]; options = { kubernetes = { role = mkOption { type = types.enum [ "server" "agent" ]; description = "Act as control plane or worker node"; }; labels = mkOption { type = types.attrs; description = "Address the k8s api is advertised on"; default = { }; }; taints = mkOption { type = types.attrs; description = "Address the k8s api is advertised on"; default = { }; }; advertiseAddress = mkOption { type = types.str; description = "Address the k8s api is advertised on"; }; externalIP = { v4 = mkOption { type = types.str; description = "External Node IP Address"; default = ""; }; v6 = mkOption { type = types.str; description = "External Node IP Address"; default = ""; }; }; }; }; config = { nixpkgs.overlays = [ (self: super: { cri-o = super.callPackage ../../pkgs/cri-o { }; k3s = super.callPackage ../../pkgs/k3s { }; }) ]; networking.extraHosts = '' 10.10.0.1 ${clusterDomain} fd15:3d8c:d429:beef::1 ${clusterDomain} ''; environment.etc = { "k3s/config.yaml" = { text = generators.toJSON { } ({ # cluster-init = true; token = "YPoyiPeBpQpB7oK8"; container-runtime-endpoint = "/run/crio/crio.sock"; node-ip = "${internalIP.v4},${internalIP.v6}"; node-external-ip = "${internalIP.v4},${internalIP.v6}"; node-label = attrValues (mapAttrs (n: v: "${n}=${toString v}") cfg.labels); node-taint = attrValues (mapAttrs (n: v: "${n}=${toString v}") cfg.taints); kubelet-arg = "cgroup-driver=systemd"; no-flannel = true; } // (if cfg.role == "server" then { advertise-address = "${internalIP.v4}"; kube-controller-manager-arg = "node-cidr-mask-size-ipv6=72"; cluster-cidr = "10.102.0.0/16,fd15:3d8c:d429:0102::/64"; service-cidr = "10.101.0.0/16,fd15:3d8c:d429:0101::/108"; cluster-dns = "10.101.0.10"; cluster-domain = clusterDomain; disable = [ "servicelb" "traefik" "local-storage" ]; disable-kube-proxy = true; disable-network-policy = true; flannel-backend = "none"; } else { })); }; }; services.k3s = { enable = true; role = cfg.role; serverAddr = "https://10.10.0.1:6443"; configPath = "/etc/k3s/config.yaml"; }; virtualisation.cri-o = { enable = true; settings = { crio = { network.plugin_dir = "/opt/cni/bin"; default_runtime = "crun"; runtime = { allowed_devices = [ "/dev/fuse" ]; default_sysctls = [ "net.ipv4.ping_group_range=0 2147483647" ]; workloads = { gitlab = { activation_annotation = "io.kubernetes.cri-o.workload/gitlab"; allowed_annotations = [ "io.kubernetes.cri-o.userns-mode" "io.kubernetes.cri-o.Devices" "io.kubernetes.cri-o.ShmSize" ]; }; }; runtimes.crun = { runtime_type = "oci"; runtime_root = "/run/crun"; allowed_annotations = [ "io.kubernetes.cri-o.userns-mode" "io.kubernetes.cri-o.Devices" "io.kubernetes.cri-o.ShmSize" ]; }; }; }; }; }; }; }