From 4f7207090ffe66debb82ab67315629c6063322fd Mon Sep 17 00:00:00 2001 From: Max Audron Date: Sat, 22 Feb 2025 14:30:35 +0100 Subject: add litellm proxy --- modules/litellm/default.nix | 98 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 modules/litellm/default.nix (limited to 'modules/litellm') diff --git a/modules/litellm/default.nix b/modules/litellm/default.nix new file mode 100644 index 0000000..7d1c737 --- /dev/null +++ b/modules/litellm/default.nix @@ -0,0 +1,98 @@ +{ + config, + lib, + pkgs, + self, + ... +}: + +let + llm_config = (pkgs.formats.yaml { }).generate "config.yaml" { + model_list = [ + { + model_name = "gpt-4o-mini"; + litellm_params = { + model = "azure/gpt-4o-mini"; + api_base = "os.environ/AZURE_ENDPOINT"; + api_key = "os.environ/AZURE_API_KEY"; + api_version = "2024-08-01-preview"; + }; + } + { + model_name = "gpt-4o"; + litellm_params = { + model = "azure/gpt-4o"; + api_base = "os.environ/AZURE_ENDPOINT"; + api_key = "os.environ/AZURE_API_KEY"; + api_version = "2024-08-01-preview"; + }; + } + { + model_name = "mistral-nemo"; + litellm_params = { + model = "azure/Mistral-Nemo"; + api_base = "os.environ/AZURE_ENDPOINT"; + api_key = "os.environ/AZURE_API_KEY"; + api_version = "2024-05-01-preview"; + }; + } + { + model_name = "mistral-large"; + litellm_params = { + model = "azure/Mistral-Large-2411"; + api_base = "os.environ/AZURE_ENDPOINT"; + api_key = "os.environ/AZURE_API_KEY"; + api_version = "2024-05-01-preview"; + }; + } + { + model_name = "codestral"; + litellm_params = { + model = "azure/Codestral-2501"; + api_base = "os.environ/AZURE_ENDPOINT"; + api_key = "os.environ/AZURE_API_KEY"; + api_version = "2024-05-01-preview"; + }; + } + ]; + + general_settings = { + # [OPTIONAL] Only use this if you to require all calls to contain this key (Authorization: Bearer sk-1234) + master_key = "os.environ/MASTER_KEY"; + }; + }; + + litellm = pkgs.python311Packages.litellm.overridePythonAttrs (prev: { + dependencies = + prev.dependencies ++ prev.optional-dependencies.proxy ++ prev.optional-dependencies.extra_proxy; + }); +in +{ + systemd.services.litellm = { + description = "litellm ai service proxy"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + serviceConfig = { + Type = "simple"; + ExecStart = "${litellm}/bin/litellm --host 10.10.0.1 --port 3289 --telemetry False --config ${llm_config}"; + EnvironmentFile = config.secrets.ai.dest; + }; + }; + + secrets = { + ai = { + source = ../../secrets/ai.env; + dest = "/etc/secrets/ai.env"; + }; + }; + + services.nginx.virtualHosts."ai.vapor.systems" = + self.lib.nginx.proxyDomain "vapor.systems" "http://10.10.0.1:3289"; + + security.acme.certs = { + "vapor.systems" = { + extraDomainNames = [ "*.vapor.systems" ]; + }; + }; +} -- cgit v1.2.3 th class='left'>AuthorLines 2025-04-08only checkout repo if cleanMax Audron-174/+343 2025-03-26delete files removed from index during checkoutMax Audron-3/+29 2025-03-26fix wrong ref being updated on checkout if branch contained slashMax Audron-2/+4 2025-03-26update dependencies and rust versionMax Audron-903/+1501 2024-09-04i do not know what i did anymoreMax Audron-14/+45 2024-03-27refactor testsMax Audron-23/+29 2024-03-27reorganize repo git implsMax Audron-209/+235 2024-03-27implement basic cloning and updating with gixMax Audron-771/+2229 2023-08-02release 1.0.4Max Audron-1/+1 2023-08-02fix static compile on darwinMax Audron-9/+16 2023-08-02add direnv to gitignoreMax Audron-0/+1 2023-08-02document issue with ignored ssh configMax Audron-0/+14 2023-08-02add error message if no projects foundMax Audron-2/+6 2023-08-02setup new nix flake using craneMax Audron-115/+225 2023-06-22update dependenciesMax Audron-469/+457 2022-07-06release 1.0.1Max Audron-1/+57 2022-07-06paginate results from gitlab apiMax Audron-11/+36 2022-06-20add gitlab upload commandMax Audron-0/+4 2022-06-20prepare release 1.0.0Max Audron-3/+17