guix-epistemia

Personal Guix channel
Log | Files | Refs

machine-learning.scm (2276B)

      1 (define-module (epistemia services machine-learning)
      2   #:use-module (gnu packages admin)
      3   #:use-module (gnu services)
      4   #:use-module (gnu services shepherd)
      5   #:use-module (gnu system shadow)
      6   #:use-module (guix gexp)
      7   #:use-module (guix records)
      8   #:export (llama-swap-configuration
      9             llama-swap-configuration?
     10             llama-swap-service-type))
     11 
     12 (define-record-type* <llm-router-configuration>
     13   llm-router-configuration make-llm-router-configuration
     14   llm-router-configuration?
     15   (package       llm-router-configuration-package
     16                  (default llm-router))
     17   (llama-cpp     llm-router-configuration-llama-cpp
     18                  (default llama-cpp))
     19   (config        llm-router-configuration-config)
     20   (listen        llm-router-configuration-listen
     21                  (default ":8080"))
     22   (watch-config? llm-router-configuration-watch-config?
     23                  (default #f)))
     24 
     25 (define llm-router-account-service
     26   (list (user-account
     27          (name "llm-router")
     28          (group "nogroup")
     29          (system? #t)
     30          (comment "llm-router daemon user")
     31          (home-directory "/var/empty")
     32          (shell (file-append shadow "/sbin/nologin")))))
     33 
     34 (define (llm-router-shepherd-service config)
     35   (let ((pkg       (llm-router-configuration-package config))
     36         (lcpp      (llm-router-configuration-llama-cpp config))
     37         (cfg       (llm-router-configuration-config config)))
     38     (list
     39      (shepherd-service
     40       (documentation "llm-router OpenAI-compatible model proxy")
     41       (provision '(llm-router))
     42       (requirement '(networking))
     43       (start #~(make-forkexec-constructor
     44                 (append
     45                  (list #$(file-append pkg "/bin/llm-router")))
     46 		#:user "llm-router"
     47                 #:environment-variables
     48 		(cons (string-append "PATH=" #$(file-append lcpp "/bin"))
     49 		      (default-environment-variables))))
     50       (stop #~(make-kill-destructor))))))
     51 
     52 (define llm-router-service-type
     53   (service-type (name 'llm-router)
     54 		(extensions
     55 		 (list (service-extension account-service-type
     56 					  (const llm-router-account-service))
     57 		       (service-extension shepherd-root-service-type
     58 					  llm-router-shepherd-service)))
     59 		(description
     60 		 "llm-router is a proxy to OpenAI-compatible and SDAPI-compatible API
     61 servers.")))