blob: 7d1c7370d22c751dfbf73dc8155a5186a517ad7c (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
|
{
config,
lib,
pkgs,
self,
...
}:
let
llm_config = (pkgs.formats.yaml { }).generate "config.yaml" {
model_list = [
{
model_name = "gpt-4o-mini";
litellm_params = {
model = "azure/gpt-4o-mini";
api_base = "os.environ/AZURE_ENDPOINT";
api_key = "os.environ/AZURE_API_KEY";
api_version = "2024-08-01-preview";
};
}
{
model_name = "gpt-4o";
litellm_params = {
model = "azure/gpt-4o";
api_base = "os.environ/AZURE_ENDPOINT";
api_key = "os.environ/AZURE_API_KEY";
api_version = "2024-08-01-preview";
};
}
{
model_name = "mistral-nemo";
litellm_params = {
model = "azure/Mistral-Nemo";
api_base = "os.environ/AZURE_ENDPOINT";
api_key = "os.environ/AZURE_API_KEY";
api_version = "2024-05-01-preview";
};
}
{
model_name = "mistral-large";
litellm_params = {
model = "azure/Mistral-Large-2411";
api_base = "os.environ/AZURE_ENDPOINT";
api_key = "os.environ/AZURE_API_KEY";
api_version = "2024-05-01-preview";
};
}
{
model_name = "codestral";
litellm_params = {
model = "azure/Codestral-2501";
api_base = "os.environ/AZURE_ENDPOINT";
api_key = "os.environ/AZURE_API_KEY";
api_version = "2024-05-01-preview";
};
}
];
general_settings = {
# [OPTIONAL] Only use this if you to require all calls to contain this key (Authorization: Bearer sk-1234)
master_key = "os.environ/MASTER_KEY";
};
};
litellm = pkgs.python311Packages.litellm.overridePythonAttrs (prev: {
dependencies =
prev.dependencies ++ prev.optional-dependencies.proxy ++ prev.optional-dependencies.extra_proxy;
});
in
{
systemd.services.litellm = {
description = "litellm ai service proxy";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
serviceConfig = {
Type = "simple";
ExecStart = "${litellm}/bin/litellm --host 10.10.0.1 --port 3289 --telemetry False --config ${llm_config}";
EnvironmentFile = config.secrets.ai.dest;
};
};
secrets = {
ai = {
source = ../../secrets/ai.env;
dest = "/etc/secrets/ai.env";
};
};
services.nginx.virtualHosts."ai.vapor.systems" =
self.lib.nginx.proxyDomain "vapor.systems" "http://10.10.0.1:3289";
security.acme.certs = {
"vapor.systems" = {
extraDomainNames = [ "*.vapor.systems" ];
};
};
}
|
