add litellm proxy

5 files changed, 205 insertions, 27 deletions

diff --git a/flake.nix b/flake.nix
index 1ba44f7..4145a52 100644
--- a/flake.nix
+++ b/flake.nix
@@ -15,9 +15,21 @@
     pastor.url = "gitlab:cocainefarm/pastor/feature/nix";
   };
 
-  outputs = inputs@{ self, nixpkgs, nixpkgs-unstable, nixpkgs-master, flake-parts, nixinate, ... }:
+  outputs =
+    inputs@{
+      self,
+      nixpkgs,
+      nixpkgs-unstable,
+      nixpkgs-master,
+      flake-parts,
+      nixinate,
+      ...
+    }:
     flake-parts.lib.mkFlake { inherit inputs; } {
-      imports = [ nixinate.flakeModule ./lib ];
+      imports = [
+        nixinate.flakeModule
+        ./lib
+      ];
       flake =
         let
           system = "x86_64-linux";
@@ -38,35 +50,93 @@
             };
           };
 
-          overlays = { config, pkgs, ... }: {
-            nixpkgs.overlays = [ overlay-unstable overlay-master ];
-          };
+          overlays =
+            { config, pkgs, ... }:
+            {
+              nixpkgs.overlays = [
+                overlay-unstable
+                overlay-master
+              ];
+            };
 
-          mkSystem = modules: nixpkgs.lib.nixosSystem {
-            inherit specialArgs system;
-            modules = modules ++ (with self.nixosModules; [
-              nixinate.nixosModules.default
-              overlays
+          mkSystem =
+            modules:
+            nixpkgs.lib.nixosSystem {
+              inherit specialArgs system;
+              modules =
+                modules
+                ++ (with self.nixosModules; [
+                  nixinate.nixosModules.default
+                  overlays
 
-              common
-              users
-              wireguard
-              crypto
-            ]);
-          };
+                  common
+                  users
+                  wireguard
+                  crypto
+                ]);
+            };
         in
         {
           nixosModules = import ./modules;
           nixosConfigurations = with self.nixosModules; {
             # vultr-image = mkSystem [ common users image vultr ];
 
-            ettves = mkSystem [ (import ./machines/ettves) zfs teamspeak postgresql quassel powerdns acme authentik hydra homeassistant games languagetool git laplace catinator pastor ];
-            phaenn = mkSystem [ (import ./machines/phaenn) zfs acme tlmp ];
-            fra01 = mkSystem [ (import ./machines/fra01) vultr bgp powerdns acme garage ];
-            nyc01 = mkSystem [ (import ./machines/nyc01) vultr bgp powerdns acme garage ];
-            sin01 = mkSystem [ (import ./machines/sin01) vultr bgp powerdns acme garage ];
+            ettves = mkSystem [
+              (import ./machines/ettves)
+              zfs
+              teamspeak
+              postgresql
+              quassel
+              powerdns
+              acme
+              authentik
+              hydra
+              homeassistant
+              games
+              languagetool
+              git
+              laplace
+              catinator
+              pastor
+              litellm
+            ];
+            phaenn = mkSystem [
+              (import ./machines/phaenn)
+              zfs
+              acme
+              tlmp
+            ];
+            fra01 = mkSystem [
+              (import ./machines/fra01)
+              vultr
+              bgp
+              powerdns
+              acme
+              garage
+            ];
+            nyc01 = mkSystem [
+              (import ./machines/nyc01)
+              vultr
+              bgp
+              powerdns
+              acme
+              garage
+            ];
+            sin01 = mkSystem [
+              (import ./machines/sin01)
+              vultr
+              bgp
+              powerdns
+              acme
+              garage
+            ];
           };
         };
-      systems = [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ];
+      systems = [
+        "x86_64-linux"
+        "x86_64-darwin"
+        "aarch64-linux"
+        "aarch64-darwin"
+      ];
     };
 }
diff --git a/modules/default.nix b/modules/default.nix
index ee4129a..751b9f1 100644
--- a/modules/default.nix
+++ b/modules/default.nix
@@ -23,6 +23,7 @@
   homeassistant = import ./homeassistant;
   garage = import ./garage;
   languagetool = import ./languagetool;
+  litellm = import ./litellm;
 
   games = import ./games;
   rtmp = import ./rtmp;
diff --git a/modules/litellm/default.nix b/modules/litellm/default.nix
new file mode 100644
index 0000000..7d1c737
--- /dev/null
+++ b/modules/litellm/default.nix
@@ -0,0 +1,98 @@
+{
+  config,
+  lib,
+  pkgs,
+  self,
+  ...
+}:
+
+let
+  llm_config = (pkgs.formats.yaml { }).generate "config.yaml" {
+    model_list = [
+      {
+        model_name = "gpt-4o-mini";
+        litellm_params = {
+          model = "azure/gpt-4o-mini";
+          api_base = "os.environ/AZURE_ENDPOINT";
+          api_key = "os.environ/AZURE_API_KEY";
+          api_version = "2024-08-01-preview";
+        };
+      }
+      {
+        model_name = "gpt-4o";
+        litellm_params = {
+          model = "azure/gpt-4o";
+          api_base = "os.environ/AZURE_ENDPOINT";
+          api_key = "os.environ/AZURE_API_KEY";
+          api_version = "2024-08-01-preview";
+        };
+      }
+      {
+        model_name = "mistral-nemo";
+        litellm_params = {
+          model = "azure/Mistral-Nemo";
+          api_base = "os.environ/AZURE_ENDPOINT";
+          api_key = "os.environ/AZURE_API_KEY";
+          api_version = "2024-05-01-preview";
+        };
+      }
+      {
+        model_name = "mistral-large";
+        litellm_params = {
+          model = "azure/Mistral-Large-2411";
+          api_base = "os.environ/AZURE_ENDPOINT";
+          api_key = "os.environ/AZURE_API_KEY";
+          api_version = "2024-05-01-preview";
+        };
+      }
+      {
+        model_name = "codestral";
+        litellm_params = {
+          model = "azure/Codestral-2501";
+          api_base = "os.environ/AZURE_ENDPOINT";
+          api_key = "os.environ/AZURE_API_KEY";
+          api_version = "2024-05-01-preview";
+        };
+      }
+    ];
+
+    general_settings = {
+      # [OPTIONAL] Only use this if you to require all calls to contain this key (Authorization: Bearer sk-1234)
+      master_key = "os.environ/MASTER_KEY";
+    };
+  };
+
+  litellm = pkgs.python311Packages.litellm.overridePythonAttrs (prev: {
+    dependencies =
+      prev.dependencies ++ prev.optional-dependencies.proxy ++ prev.optional-dependencies.extra_proxy;
+  });
+in
+{
+  systemd.services.litellm = {
+    description = "litellm ai service proxy";
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network.target" ];
+
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${litellm}/bin/litellm --host 10.10.0.1 --port 3289 --telemetry False --config ${llm_config}";
+      EnvironmentFile = config.secrets.ai.dest;
+    };
+  };
+
+  secrets = {
+    ai = {
+      source = ../../secrets/ai.env;
+      dest = "/etc/secrets/ai.env";
+    };
+  };
+
+  services.nginx.virtualHosts."ai.vapor.systems" =
+    self.lib.nginx.proxyDomain "vapor.systems" "http://10.10.0.1:3289";
+
+  security.acme.certs = {
+    "vapor.systems" = {
+      extraDomainNames = [ "*.vapor.systems" ];
+    };
+  };
+}
diff --git a/modules/powerdns/primary.nix b/modules/powerdns/primary.nix
index 60abfce..da19c50 100644
--- a/modules/powerdns/primary.nix
+++ b/modules/powerdns/primary.nix
@@ -1,12 +1,21 @@
-{ self, config, lib, pkgs, ... }:
+{
+  self,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 
 {
   config = lib.mkIf (config.services.powerdns.role == "primary") {
     services.powerdns-admin = {
-      enable = true;
+      enable = false;
       secretKeyFile = "/var/lib/pdns/secret.key";
       saltFile = "/var/lib/pdns/salt";
-      extraArgs = [ "-b" "10.10.0.1:8000" ];
+      extraArgs = [
+        "-b"
+        "10.10.0.1:8000"
+      ];
       config = ''
         SQLALCHEMY_DATABASE_URI = 'postgresql://powerdnsadmin@/pdns?host=/run/postgresql'
         # SAML_ENABLED = True
@@ -38,11 +47,11 @@
 
     services.nginx = {
       enable = true;
-      virtualHosts."ns.vapor.systems" = self.lib.nginx.proxyDomain "vapor.systems" "http://10.10.0.1:8000";
+      virtualHosts."ns.vapor.systems" =
+        self.lib.nginx.proxyDomain "vapor.systems" "http://10.10.0.1:8000";
     };
 
     systemd.services.powerdns-admin.serviceConfig = {
-      ExecStartPre = lib.mkForce "${pkgs.coreutils}/bin/env FLASK_APP=${pkgs.powerdns-admin}/share/powerdnsadmin/__init__.py SESSION_TYPE=sqlalchemy ${pkgs.python3Packages.flask}/bin/flask db upgrade -d ${pkgs.powerdns-admin}/share/migrations";
       BindPaths = [ "/run/postgresql" ];
     };
 
diff --git a/secrets b/secrets
-Subproject 21f3a5c51dd80a434f43c2e1c960a9f2c88e6a8
+Subproject 2b6f8b113760dfb2f96590edec0fc7eaaac01e4