add litellm proxy

3 files changed, 113 insertions, 5 deletions

diff --git a/modules/default.nix b/modules/default.nix
index ee4129a..751b9f1 100644
--- a/modules/default.nix
+++ b/modules/default.nix
@@ -23,6 +23,7 @@
   homeassistant = import ./homeassistant;
   garage = import ./garage;
   languagetool = import ./languagetool;
+  litellm = import ./litellm;
 
   games = import ./games;
   rtmp = import ./rtmp;
diff --git a/modules/litellm/default.nix b/modules/litellm/default.nix
new file mode 100644
index 0000000..7d1c737
--- /dev/null
+++ b/modules/litellm/default.nix
@@ -0,0 +1,98 @@
+{
+  config,
+  lib,
+  pkgs,
+  self,
+  ...
+}:
+
+let
+  llm_config = (pkgs.formats.yaml { }).generate "config.yaml" {
+    model_list = [
+      {
+        model_name = "gpt-4o-mini";
+        litellm_params = {
+          model = "azure/gpt-4o-mini";
+          api_base = "os.environ/AZURE_ENDPOINT";
+          api_key = "os.environ/AZURE_API_KEY";
+          api_version = "2024-08-01-preview";
+        };
+      }
+      {
+        model_name = "gpt-4o";
+        litellm_params = {
+          model = "azure/gpt-4o";
+          api_base = "os.environ/AZURE_ENDPOINT";
+          api_key = "os.environ/AZURE_API_KEY";
+          api_version = "2024-08-01-preview";
+        };
+      }
+      {
+        model_name = "mistral-nemo";
+        litellm_params = {
+          model = "azure/Mistral-Nemo";
+          api_base = "os.environ/AZURE_ENDPOINT";
+          api_key = "os.environ/AZURE_API_KEY";
+          api_version = "2024-05-01-preview";
+        };
+      }
+      {
+        model_name = "mistral-large";
+        litellm_params = {
+          model = "azure/Mistral-Large-2411";
+          api_base = "os.environ/AZURE_ENDPOINT";
+          api_key = "os.environ/AZURE_API_KEY";
+          api_version = "2024-05-01-preview";
+        };
+      }
+      {
+        model_name = "codestral";
+        litellm_params = {
+          model = "azure/Codestral-2501";
+          api_base = "os.environ/AZURE_ENDPOINT";
+          api_key = "os.environ/AZURE_API_KEY";
+          api_version = "2024-05-01-preview";
+        };
+      }
+    ];
+
+    general_settings = {
+      # [OPTIONAL] Only use this if you to require all calls to contain this key (Authorization: Bearer sk-1234)
+      master_key = "os.environ/MASTER_KEY";
+    };
+  };
+
+  litellm = pkgs.python311Packages.litellm.overridePythonAttrs (prev: {
+    dependencies =
+      prev.dependencies ++ prev.optional-dependencies.proxy ++ prev.optional-dependencies.extra_proxy;
+  });
+in
+{
+  systemd.services.litellm = {
+    description = "litellm ai service proxy";
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network.target" ];
+
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${litellm}/bin/litellm --host 10.10.0.1 --port 3289 --telemetry False --config ${llm_config}";
+      EnvironmentFile = config.secrets.ai.dest;
+    };
+  };
+
+  secrets = {
+    ai = {
+      source = ../../secrets/ai.env;
+      dest = "/etc/secrets/ai.env";
+    };
+  };
+
+  services.nginx.virtualHosts."ai.vapor.systems" =
+    self.lib.nginx.proxyDomain "vapor.systems" "http://10.10.0.1:3289";
+
+  security.acme.certs = {
+    "vapor.systems" = {
+      extraDomainNames = [ "*.vapor.systems" ];
+    };
+  };
+}
diff --git a/modules/powerdns/primary.nix b/modules/powerdns/primary.nix
index 60abfce..da19c50 100644
--- a/modules/powerdns/primary.nix
+++ b/modules/powerdns/primary.nix
@@ -1,12 +1,21 @@
-{ self, config, lib, pkgs, ... }:
+{
+  self,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 
 {
   config = lib.mkIf (config.services.powerdns.role == "primary") {
     services.powerdns-admin = {
-      enable = true;
+      enable = false;
       secretKeyFile = "/var/lib/pdns/secret.key";
       saltFile = "/var/lib/pdns/salt";
-      extraArgs = [ "-b" "10.10.0.1:8000" ];
+      extraArgs = [
+        "-b"
+        "10.10.0.1:8000"
+      ];
       config = ''
         SQLALCHEMY_DATABASE_URI = 'postgresql://powerdnsadmin@/pdns?host=/run/postgresql'
         # SAML_ENABLED = True
@@ -38,11 +47,11 @@
 
     services.nginx = {
       enable = true;
-      virtualHosts."ns.vapor.systems" = self.lib.nginx.proxyDomain "vapor.systems" "http://10.10.0.1:8000";
+      virtualHosts."ns.vapor.systems" =
+        self.lib.nginx.proxyDomain "vapor.systems" "http://10.10.0.1:8000";
     };
 
     systemd.services.powerdns-admin.serviceConfig = {
-      ExecStartPre = lib.mkForce "${pkgs.coreutils}/bin/env FLASK_APP=${pkgs.powerdns-admin}/share/powerdnsadmin/__init__.py SESSION_TYPE=sqlalchemy ${pkgs.python3Packages.flask}/bin/flask db upgrade -d ${pkgs.powerdns-admin}/share/migrations";
       BindPaths = [ "/run/postgresql" ];
     };