blob: 022630033b7e02145dba1c5b285a90953903f93a (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
|
(define-module (epistemia services machine-learning)
#:use-module (gnu services)
#:use-module (gnu services shepherd)
#:use-module (guix gexp)
#:use-module (guix records)
#:export (llama-swap-configuration
llama-swap-configuration?
llama-swap-service-type))
(define-record-type* <llama-swap-configuration>
llama-swap-configuration make-llama-swap-configuration
llama-swap-configuration?
(package llama-swap-configuration-package
(default llama-swap))
(llama-cpp llama-swap-configuration-llama-cpp
(default llama-cpp))
(config llama-swap-configuration-config)
(listen llama-swap-configuration-listen
(default ":8080"))
(watch-config? llama-swap-configuration-watch-config?
(default #f)))
(define (llama-swap-shepherd-service config)
(let ((pkg (llama-swap-configuration-package config))
(lcpp (llama-swap-configuration-llama-cpp config))
(cfg (llama-swap-configuration-config config))
(listen (llama-swap-configuration-listen config))
(watch? (llama-swap-configuration-watch-config? config)))
(list
(shepherd-service
(documentation "llama-swap OpenAI-compatible model proxy")
(provision '(llama-swap))
(requirement '(networking))
(start #~(make-forkexec-constructor
(append
(list #$(file-append pkg "/bin/llama-swap")
"-config" #$cfg
"-listen" #$listen)
(if #$watch? '("-watch-config") '()))
#:environment-variables
(cons (string-append "PATH=" #$(file-append lcpp "/bin"))
(default-environment-variables))))
(stop #~(make-kill-destructor))))))
(define llama-swap-service-type
(service-type
(name 'llama-swap)
(extensions
(list (service-extension shepherd-root-service-type
llama-swap-shepherd-service)))
(description
"llama-swap is an @code{OpenAI} API compatible server that gives you complete
control over how you use your hardware. It automatically swaps to the
configuration of your choice for serving a model.")))
|
