blob: 094049ca5d2d30fc04b3579058aac6732a38f031 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
|
(define-module (epistemia services machine-learning)
#:use-module (gnu packages admin)
#:use-module (gnu services)
#:use-module (gnu services shepherd)
#:use-module (gnu system shadow)
#:use-module (guix gexp)
#:use-module (guix records)
#:export (llama-swap-configuration
llama-swap-configuration?
llama-swap-service-type))
(define-record-type* <llm-router-configuration>
llm-router-configuration make-llm-router-configuration
llm-router-configuration?
(package llm-router-configuration-package
(default llm-router))
(llama-cpp llm-router-configuration-llama-cpp
(default llama-cpp))
(config llm-router-configuration-config)
(listen llm-router-configuration-listen
(default ":8080"))
(watch-config? llm-router-configuration-watch-config?
(default #f)))
(define llm-router-account-service
(list (user-account
(name "llm-router")
(group "nogroup")
(system? #t)
(comment "llm-router daemon user")
(home-directory "/var/empty")
(shell (file-append shadow "/sbin/nologin")))))
(define (llm-router-shepherd-service config)
(let ((pkg (llm-router-configuration-package config))
(lcpp (llm-router-configuration-llama-cpp config))
(cfg (llm-router-configuration-config config)))
(list
(shepherd-service
(documentation "llm-router OpenAI-compatible model proxy")
(provision '(llm-router))
(requirement '(networking))
(start #~(make-forkexec-constructor
(append
(list #$(file-append pkg "/bin/llm-router")))
#:user "llm-router"
#:environment-variables
(cons (string-append "PATH=" #$(file-append lcpp "/bin"))
(default-environment-variables))))
(stop #~(make-kill-destructor))))))
(define llm-router-service-type
(service-type (name 'llm-router)
(extensions
(list (service-extension account-service-type
(const llm-router-account-service))
(service-extension shepherd-root-service-type
llm-router-shepherd-service)))
(description
"llm-router is a proxy to OpenAI-compatible and SDAPI-compatible API
servers.")))
|
