services/machine-learning: add service for llama-swap

1 files changed, 54 insertions, 0 deletions

diff --git a/epistemia/services/machine-learning.scm b/epistemia/services/machine-learning.scm
new file mode 100644
index 0000000..0226300
--- /dev/null
+++ b/epistemia/services/machine-learning.scm
@@ -0,0 +1,54 @@
+(define-module (epistemia services machine-learning)
+  #:use-module (gnu services)
+  #:use-module (gnu services shepherd)
+  #:use-module (guix gexp)
+  #:use-module (guix records)
+  #:export (llama-swap-configuration
+            llama-swap-configuration?
+            llama-swap-service-type))
+
+(define-record-type* <llama-swap-configuration>
+  llama-swap-configuration make-llama-swap-configuration
+  llama-swap-configuration?
+  (package       llama-swap-configuration-package
+                 (default llama-swap))
+  (llama-cpp     llama-swap-configuration-llama-cpp
+                 (default llama-cpp))
+  (config        llama-swap-configuration-config)
+  (listen        llama-swap-configuration-listen
+                 (default ":8080"))
+  (watch-config? llama-swap-configuration-watch-config?
+                 (default #f)))
+
+(define (llama-swap-shepherd-service config)
+  (let ((pkg       (llama-swap-configuration-package config))
+        (lcpp      (llama-swap-configuration-llama-cpp config))
+        (cfg       (llama-swap-configuration-config config))
+        (listen    (llama-swap-configuration-listen config))
+        (watch?    (llama-swap-configuration-watch-config? config)))
+    (list
+     (shepherd-service
+      (documentation "llama-swap OpenAI-compatible model proxy")
+      (provision '(llama-swap))
+      (requirement '(networking))
+      (start #~(make-forkexec-constructor
+                (append
+                 (list #$(file-append pkg "/bin/llama-swap")
+                       "-config" #$cfg
+                       "-listen" #$listen)
+                 (if #$watch? '("-watch-config") '()))
+                #:environment-variables
+                (cons (string-append "PATH=" #$(file-append lcpp "/bin"))
+                      (default-environment-variables))))
+      (stop #~(make-kill-destructor))))))
+
+(define llama-swap-service-type
+  (service-type
+   (name 'llama-swap)
+   (extensions
+    (list (service-extension shepherd-root-service-type
+                             llama-swap-shepherd-service)))
+    (description
+     "llama-swap is an @code{OpenAI} API compatible server that gives you complete
+control over how you use your hardware. It automatically swaps to the
+configuration of your choice for serving a model.")))