add litellm proxy

1 files changed, 98 insertions, 0 deletions

diff --git a/modules/litellm/default.nix b/modules/litellm/default.nix
new file mode 100644
index 0000000..7d1c737
--- /dev/null
+++ b/modules/litellm/default.nix
@@ -0,0 +1,98 @@
+{
+  config,
+  lib,
+  pkgs,
+  self,
+  ...
+}:
+
+let
+  llm_config = (pkgs.formats.yaml { }).generate "config.yaml" {
+    model_list = [
+      {
+        model_name = "gpt-4o-mini";
+        litellm_params = {
+          model = "azure/gpt-4o-mini";
+          api_base = "os.environ/AZURE_ENDPOINT";
+          api_key = "os.environ/AZURE_API_KEY";
+          api_version = "2024-08-01-preview";
+        };
+      }
+      {
+        model_name = "gpt-4o";
+        litellm_params = {
+          model = "azure/gpt-4o";
+          api_base = "os.environ/AZURE_ENDPOINT";
+          api_key = "os.environ/AZURE_API_KEY";
+          api_version = "2024-08-01-preview";
+        };
+      }
+      {
+        model_name = "mistral-nemo";
+        litellm_params = {
+          model = "azure/Mistral-Nemo";
+          api_base = "os.environ/AZURE_ENDPOINT";
+          api_key = "os.environ/AZURE_API_KEY";
+          api_version = "2024-05-01-preview";
+        };
+      }
+      {
+        model_name = "mistral-large";
+        litellm_params = {
+          model = "azure/Mistral-Large-2411";
+          api_base = "os.environ/AZURE_ENDPOINT";
+          api_key = "os.environ/AZURE_API_KEY";
+          api_version = "2024-05-01-preview";
+        };
+      }
+      {
+        model_name = "codestral";
+        litellm_params = {
+          model = "azure/Codestral-2501";
+          api_base = "os.environ/AZURE_ENDPOINT";
+          api_key = "os.environ/AZURE_API_KEY";
+          api_version = "2024-05-01-preview";
+        };
+      }
+    ];
+
+    general_settings = {
+      # [OPTIONAL] Only use this if you to require all calls to contain this key (Authorization: Bearer sk-1234)
+      master_key = "os.environ/MASTER_KEY";
+    };
+  };
+
+  litellm = pkgs.python311Packages.litellm.overridePythonAttrs (prev: {
+    dependencies =
+      prev.dependencies ++ prev.optional-dependencies.proxy ++ prev.optional-dependencies.extra_proxy;
+  });
+in
+{
+  systemd.services.litellm = {
+    description = "litellm ai service proxy";
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network.target" ];
+
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${litellm}/bin/litellm --host 10.10.0.1 --port 3289 --telemetry False --config ${llm_config}";
+      EnvironmentFile = config.secrets.ai.dest;
+    };
+  };
+
+  secrets = {
+    ai = {
+      source = ../../secrets/ai.env;
+      dest = "/etc/secrets/ai.env";
+    };
+  };
+
+  services.nginx.virtualHosts."ai.vapor.systems" =
+    self.lib.nginx.proxyDomain "vapor.systems" "http://10.10.0.1:3289";
+
+  security.acme.certs = {
+    "vapor.systems" = {
+      extraDomainNames = [ "*.vapor.systems" ];
+    };
+  };
+}