Initial commit: LlamaLink Go rewrite
Complete rewrite from Python/FastAPI to Go/Gin: - Go backend: auth (API keys + bcrypt), llama.cpp subprocess manager, hot-swap multi-model, rate limiting, quota system, webhooks - Vue 3 SPA admin panel (src/) with Tailwind CSS - Deployment: Docker multi-stage, docker-compose, nginx, systemd - GORM/SQLite models: ApiKey, Model, UsageLog, Quota, Webhook - REST API: /api/v1/admin/* (keys, models, chat, usage, health) - Embedded frontend via go:embed (build output at web/dist/) Removed legacy Python artifacts (app/, tests/, pyproject.toml, etc.)
This commit is contained in:
@@ -0,0 +1,407 @@
|
||||
package llama
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/llamalink/llamalink/internal/config"
|
||||
"github.com/llamalink/llamalink/internal/db"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrModelNotFound = errors.New("model not found in registry")
|
||||
ErrModelDisabled = errors.New("model is disabled")
|
||||
ErrSwapInProgress = errors.New("model swap already in progress")
|
||||
ErrAlreadyLoaded = errors.New("model already loaded")
|
||||
ErrServerNotRunning = errors.New("llama-server not running")
|
||||
)
|
||||
|
||||
type Manager struct {
|
||||
cfg *config.Config
|
||||
db *gorm.DB
|
||||
mu sync.RWMutex
|
||||
state State
|
||||
proc *exec.Cmd
|
||||
done chan struct{}
|
||||
url string
|
||||
}
|
||||
|
||||
func NewManager(cfg *config.Config, db *gorm.DB) *Manager {
|
||||
return &Manager{
|
||||
cfg: cfg,
|
||||
db: db,
|
||||
done: make(chan struct{}),
|
||||
url: cfg.LlamaServerURL(),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) GetDB() *gorm.DB {
|
||||
return m.db
|
||||
}
|
||||
|
||||
func (m *Manager) Start() error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
slog.Info("llama manager starting", "url", m.url)
|
||||
|
||||
// Load default model on startup
|
||||
var model db.Model
|
||||
if err := m.db.Where("is_default = ? AND is_enabled = ?", true, true).First(&model).Error; err == nil {
|
||||
slog.Info("loading default model", "name", model.Name)
|
||||
if err := m.loadModelInternal(&model); err != nil {
|
||||
slog.Warn("failed to load default model", "error", err)
|
||||
m.state.Status = StatusFailed
|
||||
m.state.LastError = err.Error()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Start health check loop
|
||||
go m.healthCheckLoop()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) Stop() {
|
||||
slog.Info("llama manager stopping")
|
||||
close(m.done)
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if m.proc != nil && m.proc.Process != nil {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), m.cfg.LlamaServerStopTimeoutDuration())
|
||||
defer cancel()
|
||||
|
||||
m.proc.SysProcAttr = &syscall.SysProcAttr{
|
||||
Setpgid: true,
|
||||
}
|
||||
|
||||
pgid, err := syscall.Getpgid(m.proc.Process.Pid)
|
||||
if err == nil {
|
||||
syscall.Kill(-pgid, syscall.SIGTERM)
|
||||
} else {
|
||||
m.proc.Process.Signal(syscall.SIGTERM)
|
||||
}
|
||||
|
||||
<-ctx.Done()
|
||||
if m.proc.ProcessState == nil {
|
||||
syscall.Kill(-pgid, syscall.SIGKILL)
|
||||
}
|
||||
}
|
||||
|
||||
m.state = State{Status: StatusStopped}
|
||||
slog.Info("llama manager stopped")
|
||||
}
|
||||
|
||||
func (m *Manager) IsReady() bool {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.state.Status == StatusReady && m.proc != nil && m.proc.ProcessState != nil && !m.proc.ProcessState.Exited()
|
||||
}
|
||||
|
||||
func (m *Manager) Status() Status {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.state.Status
|
||||
}
|
||||
|
||||
func (m *Manager) CurrentModel() string {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.state.CurrentModel
|
||||
}
|
||||
|
||||
func (m *Manager) GetStatus() *State {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return &m.state
|
||||
}
|
||||
|
||||
func (m *Manager) LoadModel(name string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
// Find model in DB
|
||||
var model db.Model
|
||||
if err := m.db.Where("name = ? AND is_enabled = ?", name, true).First(&model).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return ErrModelNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Check current state
|
||||
if m.state.Status == StatusLoading || m.state.Status == StatusSwapping {
|
||||
if m.state.CurrentModel == name {
|
||||
return nil // Already loading this model
|
||||
}
|
||||
return fmt.Errorf("swap in progress for %s, try again later", m.state.TargetModel)
|
||||
}
|
||||
|
||||
if m.state.CurrentModel == name && m.state.Status == StatusReady {
|
||||
return nil // Already loaded
|
||||
}
|
||||
|
||||
return m.loadModelInternal(&model)
|
||||
}
|
||||
|
||||
func (m *Manager) loadModelInternal(model *db.Model) error {
|
||||
isSwap := m.state.Status == StatusReady && m.state.CurrentModel != ""
|
||||
m.state.Status = StatusSwapping
|
||||
if !isSwap {
|
||||
m.state.Status = StatusLoading
|
||||
}
|
||||
m.state.TargetModel = model.Name
|
||||
m.state.LastError = ""
|
||||
now := time.Now()
|
||||
m.state.SwapStartedAt = &now
|
||||
|
||||
slog.Info("loading model", "name", model.Name, "is_swap", isSwap)
|
||||
|
||||
// Kill existing process
|
||||
if m.proc != nil && m.proc.Process != nil {
|
||||
m.terminateProcess()
|
||||
}
|
||||
|
||||
// Build command
|
||||
cmd := m.buildCommand(model)
|
||||
m.proc = cmd
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
m.state.Status = StatusFailed
|
||||
m.state.LastError = err.Error()
|
||||
return fmt.Errorf("failed to start llama-server: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("llama-server started", "pid", cmd.Process.Pid)
|
||||
m.state.PID = cmd.Process.Pid
|
||||
|
||||
// Wait for server to be ready
|
||||
if err := m.waitUntilReady(); err != nil {
|
||||
m.state.Status = StatusFailed
|
||||
m.state.LastError = err.Error()
|
||||
return fmt.Errorf("model failed to start: %w", err)
|
||||
}
|
||||
|
||||
m.state.CurrentModel = model.Name
|
||||
m.state.Status = StatusReady
|
||||
m.state.TargetModel = ""
|
||||
m.state.LoadedAt = &now
|
||||
|
||||
// Update DB
|
||||
m.db.Model(model).Updates(map[string]interface{}{
|
||||
"is_active": true,
|
||||
"loaded_at": now,
|
||||
})
|
||||
|
||||
slog.Info("model loaded successfully", "name", model.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) buildCommand(model *db.Model) *exec.Cmd {
|
||||
args := []string{
|
||||
"--model", model.ModelPath,
|
||||
"--alias", model.Alias,
|
||||
"--host", m.cfg.LlamaServerHost,
|
||||
"--port", fmt.Sprintf("%d", m.cfg.LlamaServerPort),
|
||||
"--ctx-size", fmt.Sprintf("%d", model.CtxSize),
|
||||
"--n-gpu-layers", fmt.Sprintf("%d", model.NGPULayers),
|
||||
}
|
||||
|
||||
// Add extra args from JSON
|
||||
extraArgs := ParseModelExtraArgs(model.ExtraArgs)
|
||||
for k, v := range extraArgs {
|
||||
if bv, ok := v.(bool); ok && bv {
|
||||
args = append(args, "--"+k)
|
||||
} else if v != nil {
|
||||
args = append(args, "--"+k, fmt.Sprintf("%v", v))
|
||||
}
|
||||
}
|
||||
|
||||
cmd := exec.Command(m.cfg.LlamaServerBin, args...)
|
||||
cmd.Stdout = io.Discard
|
||||
cmd.Stderr = io.Discard
|
||||
|
||||
// Set process group for clean kill
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{
|
||||
Setpgid: true,
|
||||
}
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func (m *Manager) terminateProcess() {
|
||||
if m.proc == nil || m.proc.Process == nil {
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("terminating llama-server", "pid", m.proc.Process.Pid)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), m.cfg.LlamaServerStopTimeoutDuration())
|
||||
defer cancel()
|
||||
|
||||
pgid, err := syscall.Getpgid(m.proc.Process.Pid)
|
||||
if err == nil {
|
||||
syscall.Kill(-pgid, syscall.SIGTERM)
|
||||
} else {
|
||||
m.proc.Process.Signal(syscall.SIGTERM)
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- m.proc.Wait()
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if pgid, err := syscall.Getpgid(m.proc.Process.Pid); err == nil {
|
||||
syscall.Kill(-pgid, syscall.SIGKILL)
|
||||
}
|
||||
case <-done:
|
||||
}
|
||||
|
||||
m.proc = nil
|
||||
}
|
||||
|
||||
func (m *Manager) waitUntilReady() error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), m.cfg.LlamaServerStartupTimeoutDuration())
|
||||
defer cancel()
|
||||
|
||||
ticker := time.NewTicker(500 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
if m.checkHealth() {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) checkHealth() bool {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", m.url+"/health", nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
return resp.StatusCode == http.StatusOK
|
||||
}
|
||||
|
||||
func (m *Manager) healthCheckLoop() {
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-m.done:
|
||||
return
|
||||
case <-ticker.C:
|
||||
m.healthCheck()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) healthCheck() {
|
||||
m.mu.RLock()
|
||||
running := m.proc != nil && m.proc.Process != nil && m.proc.ProcessState != nil && !m.proc.ProcessState.Exited()
|
||||
m.mu.RUnlock()
|
||||
|
||||
if !running && m.state.Status == StatusReady {
|
||||
m.mu.Lock()
|
||||
m.state.Status = StatusFailed
|
||||
m.state.LastError = "llama-server process died unexpectedly"
|
||||
m.mu.Unlock()
|
||||
slog.Error("llama-server process died", "current_model", m.state.CurrentModel)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) GetUsageStats() (totalRequests, totalTokens int64, avgLatencyMs float64) {
|
||||
now := time.Now()
|
||||
monthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
var result struct {
|
||||
TotalRequests int64
|
||||
TotalTokens int64
|
||||
AvgLatency float64
|
||||
}
|
||||
|
||||
m.db.Model(&db.UsageLog{}).
|
||||
Where("created_at >= ?", monthStart).
|
||||
Select("COUNT(*) as total_requests, COALESCE(SUM(total_tokens), 0) as total_tokens, COALESCE(AVG(latency_ms), 0) as avg_latency").
|
||||
Scan(&result)
|
||||
|
||||
return result.TotalRequests, result.TotalTokens, result.AvgLatency
|
||||
}
|
||||
|
||||
// ProxyRequest sends a request to the llama-server proxy
|
||||
func (m *Manager) ProxyRequest(ctx context.Context, method, path string, body io.Reader, headers map[string]string) (*http.Response, error) {
|
||||
if !m.IsReady() {
|
||||
return nil, ErrServerNotRunning
|
||||
}
|
||||
|
||||
url := m.url + path
|
||||
req, err := http.NewRequestWithContext(ctx, method, url, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
|
||||
return http.DefaultClient.Do(req)
|
||||
}
|
||||
|
||||
func ParseModelExtraArgs(extraArgs db.StringArray) map[string]interface{} {
|
||||
if len(extraArgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// If it's a JSON string, parse it
|
||||
if len(extraArgs) == 1 {
|
||||
var result map[string]interface{}
|
||||
if json.Unmarshal([]byte(extraArgs[0]), &result) == nil {
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
// Otherwise assume key=value pairs
|
||||
result := make(map[string]interface{})
|
||||
for _, arg := range extraArgs {
|
||||
parts := strings.SplitN(arg, "=", 2)
|
||||
if len(parts) == 2 {
|
||||
result[parts[0]] = parts[1]
|
||||
} else {
|
||||
result[arg] = true
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
package llama
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Proxy struct {
|
||||
manager *Manager
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewProxy(manager *Manager) *Proxy {
|
||||
return &Proxy{
|
||||
manager: manager,
|
||||
client: &http.Client{
|
||||
Timeout: 60 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Proxy) Manager() *Manager {
|
||||
return p.manager
|
||||
}
|
||||
|
||||
type ChatMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
type ChatCompletionRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []ChatMessage `json:"messages"`
|
||||
Stream bool `json:"stream,omitempty"`
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
Temperature float64 `json:"temperature,omitempty"`
|
||||
TopP float64 `json:"top_p,omitempty"`
|
||||
}
|
||||
|
||||
type ChatCompletionResponse struct {
|
||||
ID string `json:"id"`
|
||||
Object string `json:"object"`
|
||||
Created int64 `json:"created"`
|
||||
Model string `json:"model"`
|
||||
Choices []Choice `json:"choices"`
|
||||
Usage Usage `json:"usage"`
|
||||
}
|
||||
|
||||
type Choice struct {
|
||||
Index int `json:"index"`
|
||||
Message ChatMessage `json:"message"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
}
|
||||
|
||||
type Usage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
}
|
||||
|
||||
type StreamChoice struct {
|
||||
Index int `json:"index"`
|
||||
Delta ChatMessage `json:"delta"`
|
||||
FinishReason string `json:"finish_reason,omitempty"`
|
||||
}
|
||||
|
||||
type StreamResponse struct {
|
||||
ID string `json:"id"`
|
||||
Object string `json:"object"`
|
||||
Created int64 `json:"created"`
|
||||
Model string `json:"model"`
|
||||
Choices []StreamChoice `json:"choices"`
|
||||
}
|
||||
|
||||
// ChatCompletion calls llama-server and returns the response
|
||||
func (p *Proxy) ChatCompletion(ctx context.Context, req ChatCompletionRequest) (*ChatCompletionResponse, error) {
|
||||
if !p.manager.IsReady() {
|
||||
return nil, ErrServerNotRunning
|
||||
}
|
||||
|
||||
// Convert to llama-server format
|
||||
llamaReq := map[string]interface{}{
|
||||
"model": req.Model,
|
||||
"messages": req.Messages,
|
||||
"stream": false,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(llamaReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "POST", p.manager.url+"/v1/chat/completions", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := p.client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("llama-server request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
bodyBytes, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("llama-server returned %d: %s", resp.StatusCode, string(bodyBytes))
|
||||
}
|
||||
|
||||
var result ChatCompletionResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode response: %w", err)
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// ChatCompletionStream returns a channel of streaming responses
|
||||
func (p *Proxy) ChatCompletionStream(ctx context.Context, req ChatCompletionRequest) (<-chan *StreamResponse, <-chan error) {
|
||||
stream := make(chan *StreamResponse, 100)
|
||||
errCh := make(chan error, 1)
|
||||
|
||||
if !p.manager.IsReady() {
|
||||
errCh <- ErrServerNotRunning
|
||||
close(stream)
|
||||
return stream, errCh
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(stream)
|
||||
defer close(errCh)
|
||||
|
||||
llamaReq := map[string]interface{}{
|
||||
"model": req.Model,
|
||||
"messages": req.Messages,
|
||||
"stream": true,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(llamaReq)
|
||||
if err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "POST", p.manager.url+"/v1/chat/completions", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := p.client.Do(httpReq)
|
||||
if err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
bodyBytes, _ := io.ReadAll(resp.Body)
|
||||
errCh <- fmt.Errorf("llama-server returned %d: %s", resp.StatusCode, string(bodyBytes))
|
||||
return
|
||||
}
|
||||
|
||||
reader := bufio.NewReader(resp.Body)
|
||||
for {
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
errCh <- err
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || !strings.HasPrefix(line, "data: ") {
|
||||
continue
|
||||
}
|
||||
|
||||
if line == "data: [DONE]" {
|
||||
break
|
||||
}
|
||||
|
||||
data := strings.TrimPrefix(line, "data: ")
|
||||
var streamResp StreamResponse
|
||||
if err := json.Unmarshal([]byte(data), &streamResp); err != nil {
|
||||
slog.Debug("failed to parse stream chunk", "error", err, "data", data)
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case stream <- &streamResp:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return stream, errCh
|
||||
}
|
||||
|
||||
// ModelsList returns available models from llama-server
|
||||
func (p *Proxy) ModelsList(ctx context.Context) ([]string, error) {
|
||||
if !p.manager.IsReady() {
|
||||
return nil, ErrServerNotRunning
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", p.manager.url+"/v1/models", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("llama-server returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Data []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
models := make([]string, len(result.Data))
|
||||
for i, m := range result.Data {
|
||||
models[i] = m.ID
|
||||
}
|
||||
return models, nil
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package llama
|
||||
|
||||
import "time"
|
||||
|
||||
type Status string
|
||||
|
||||
const (
|
||||
StatusStopped Status = "stopped"
|
||||
StatusLoading Status = "loading"
|
||||
StatusReady Status = "ready"
|
||||
StatusSwapping Status = "swapping"
|
||||
StatusFailed Status = "failed"
|
||||
)
|
||||
|
||||
type State struct {
|
||||
CurrentModel string `json:"current_model"`
|
||||
TargetModel string `json:"target_model,omitempty"`
|
||||
Status Status `json:"status"`
|
||||
PID int `json:"pid,omitempty"`
|
||||
LoadedAt *time.Time `json:"loaded_at,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
SwapStartedAt *time.Time `json:"swap_started_at,omitempty"`
|
||||
SwapInProgress bool `json:"swap_in_progress"`
|
||||
}
|
||||
|
||||
type ModelInfo struct {
|
||||
Name string
|
||||
ModelPath string
|
||||
Alias string
|
||||
CtxSize int
|
||||
NGPULayers int
|
||||
ExtraArgs map[string]interface{}
|
||||
IsDefault bool
|
||||
}
|
||||
Reference in New Issue
Block a user