Phase A-E: stability, security, observability, and test coverage

Phase A - Stability:
- Engine.Start(): recovers orphaned jobs (running/queued/waking_up) after crash
- Engine.Stop(): graceful shutdown - cancels all in-flight jobs and waits
- Queue keyed by jobID (not syncPairID): cancel now targets exact job
- Local rsync uses jobCtx (context.Background() replaced)
- Migrations wrapped in transactions; checksums stored

Phase B - Security:
- admin/admin default removed: SYNCSERVER_ADMIN_PASSWORD required on first run
- Path validation: rejects .., leading -, null bytes in sync pair paths
- Rsync flags allowlist: dangerous flags blocked (--rsync-path, -e, --files-from)
- Shell concat in RunRemote replaced with proper sh -c escaping
- knownhosts: replaced custom parser with golang.org/x/crypto/ssh/knownhosts
- RequireAdmin wired: machine CRUD, SSH key ops, settings require admin role
- deploy-keys: uses authorized_keys only (no private key upload)
- Hardcoded /var/lib/syncserver/ssh paths replaced with cfg.SSHDir()

Phase C - Operational:
- /readyz health check: DB query + SSH dir accessibility
- /metrics endpoint: Prometheus text format (jobs, queue, machines)
- Event struct JSON tags: job_id, machine_id, type (snake_case)
- EventBus broadcast: fanned out to all subscribers
- SQLite VACUUM INTO backup: scheduled before cleanup if BackupDir set
- Filesystem job log cleanup: removes .log files for purged jobs
- Backup retention: old backups auto-purged

Phase D - Frontend:
- Schedules page: REST API + full CRUD UI for cron schedules
- Dashboard: cancel button for running/queued jobs
- JobDetail: server-side log download via API
- Settings: displays data_dir from server
- 404 page: proper NotFound component

Phase E - Tests:
- auth_test.go: JWT, bcrypt, middleware, seed (18 tests)
- models_test.go: Job, Machine, SyncPair, Schedule repos (18 tests)
- go test -race: no data races found
This commit is contained in:
2026-07-19 22:14:30 -04:00
parent 300555d35f
commit 84b185be39
33 changed files with 2398 additions and 199 deletions
+21
View File
@@ -121,3 +121,24 @@ type SettingsInfoResponse struct {
DataDir string `json:"data_dir"`
SSHPubKey string `json:"ssh_pub_key"`
}
type CreateScheduleRequest struct {
SyncPairID int64 `json:"sync_pair_id"`
CronExpr string `json:"cron_expr"`
Enabled bool `json:"enabled"`
}
type UpdateScheduleRequest struct {
CronExpr string `json:"cron_expr"`
Enabled bool `json:"enabled"`
}
type ScheduleResponse struct {
ID int64 `json:"id"`
SyncPairID int64 `json:"sync_pair_id"`
SyncPairName string `json:"sync_pair_name"`
CronExpr string `json:"cron_expr"`
NextRun *string `json:"next_run_at"`
Enabled bool `json:"enabled"`
CreatedAt string `json:"created_at"`
}
+1 -1
View File
@@ -128,7 +128,7 @@ func (h *JobHandler) Cancel(w http.ResponseWriter, r *http.Request) {
}
if h.engine != nil {
h.engine.Cancel(id, j.SyncPairID, true)
h.engine.Cancel(id, true)
}
repo.UpdateStatus(id, "cancelled")
+8 -7
View File
@@ -252,13 +252,13 @@ func (h *MachineHandler) Shutdown(w http.ResponseWriter, r *http.Request) {
return
}
knownHostsPath, err := sshmanager.EnsureKnownHosts(filepath.Join("/var/lib/syncserver", "ssh"))
knownHostsPath, err := sshmanager.EnsureKnownHosts(h.cfg.SSHDir())
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to ensure known_hosts")
return
}
privKeyPath := filepath.Join("/var/lib/syncserver", "ssh", "id_ed25519")
privKeyPath := filepath.Join(h.cfg.SSHDir(), "id_ed25519")
if m.SSHKeyID != nil {
sshKeyRepo := models.NewSSHKeyRepository(h.db)
sshKey, err := sshKeyRepo.GetByID(*m.SSHKeyID)
@@ -333,13 +333,13 @@ func (h *MachineHandler) TestConnection(w http.ResponseWriter, r *http.Request)
return
}
knownHostsPath, err := sshmanager.EnsureKnownHosts(filepath.Join("/var/lib/syncserver", "ssh"))
knownHostsPath, err := sshmanager.EnsureKnownHosts(h.cfg.SSHDir())
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to ensure known_hosts")
return
}
privKeyPath := filepath.Join("/var/lib/syncserver", "ssh", "id_ed25519")
privKeyPath := filepath.Join(h.cfg.SSHDir(), "id_ed25519")
if m.SSHKeyID != nil {
sshKeyRepo := models.NewSSHKeyRepository(h.db)
sshKey, err := sshKeyRepo.GetByID(*m.SSHKeyID)
@@ -390,14 +390,14 @@ func (h *MachineHandler) ApproveFingerprint(w http.ResponseWriter, r *http.Reque
}
json.NewDecoder(r.Body).Decode(&req)
sshDir := filepath.Join("/var/lib/syncserver", "ssh")
sshDir := h.cfg.SSHDir()
knownHostsPath, err := sshmanager.EnsureKnownHosts(sshDir)
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to ensure known_hosts")
return
}
privKeyPath := filepath.Join("/var/lib/syncserver", "ssh", "id_ed25519")
privKeyPath := filepath.Join(h.cfg.SSHDir(), "id_ed25519")
if m.SSHKeyID != nil {
sshKeyRepo := models.NewSSHKeyRepository(h.db)
sshKey, err := sshKeyRepo.GetByID(*m.SSHKeyID)
@@ -507,7 +507,7 @@ func (h *MachineHandler) DeployKeys(w http.ResponseWriter, r *http.Request) {
seenKeys[sk.PrivateKeyPath] = true
keys = append(keys, sshmanager.DeployKey{
LocalPath: sk.PrivateKeyPath,
RemotePath: "/var/lib/syncserver/ssh/keys/" + filepath.Base(sk.PrivateKeyPath),
RemotePath: h.cfg.SSHDir() + "/keys/" + filepath.Base(sk.PrivateKeyPath),
Mode: 0600,
})
}
@@ -526,6 +526,7 @@ func (h *MachineHandler) DeployKeys(w http.ResponseWriter, r *http.Request) {
m.SSHUser,
keys,
knownHostsHosts,
h.cfg.SSHDir(),
)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
+243
View File
@@ -0,0 +1,243 @@
package api
import (
"database/sql"
"encoding/json"
"log/slog"
"net/http"
"strconv"
"time"
"github.com/go-chi/chi/v5"
"github.com/syncserver/internal/models"
"github.com/syncserver/internal/scheduler"
)
type ScheduleHandler struct {
db *sql.DB
}
func NewScheduleHandler(db *sql.DB) *ScheduleHandler {
return &ScheduleHandler{db: db}
}
func (h *ScheduleHandler) List(w http.ResponseWriter, r *http.Request) {
repo := models.NewScheduleRepository(h.db)
schedules, err := repo.GetAll()
if err != nil {
slog.Error("failed to fetch schedules", "error", err)
writeError(w, http.StatusInternalServerError, "failed to fetch schedules")
return
}
pairRepo := models.NewSyncPairRepository(h.db)
pairs, err := pairRepo.GetAll()
if err != nil {
slog.Error("failed to fetch sync pairs", "error", err)
writeError(w, http.StatusInternalServerError, "failed to fetch sync pairs")
return
}
pairMap := make(map[int64]string)
for _, p := range pairs {
pairMap[p.ID] = p.Name
}
out := make([]ScheduleResponse, len(schedules))
for i, s := range schedules {
out[i] = scheduleToResp(s, pairMap)
}
writeJSON(w, out)
}
func (h *ScheduleHandler) Get(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid id")
return
}
repo := models.NewScheduleRepository(h.db)
s, err := repo.GetByID(id)
if err == sql.ErrNoRows {
writeError(w, http.StatusNotFound, "schedule not found")
return
}
if err != nil {
slog.Error("failed to fetch schedule", "id", id, "error", err)
writeError(w, http.StatusInternalServerError, "failed to fetch schedule")
return
}
pairRepo := models.NewSyncPairRepository(h.db)
pairs, err := pairRepo.GetAll()
if err != nil {
slog.Error("failed to fetch sync pairs", "error", err)
writeError(w, http.StatusInternalServerError, "failed to fetch sync pairs")
return
}
pairMap := make(map[int64]string)
for _, p := range pairs {
pairMap[p.ID] = p.Name
}
writeJSON(w, scheduleToResp(*s, pairMap))
}
func (h *ScheduleHandler) Create(w http.ResponseWriter, r *http.Request) {
var req CreateScheduleRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
if req.SyncPairID <= 0 {
writeError(w, http.StatusBadRequest, "sync_pair_id is required")
return
}
if req.CronExpr == "" {
writeError(w, http.StatusBadRequest, "cron_expr is required")
return
}
_, err := scheduler.ParseCron(req.CronExpr)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid cron expression: "+err.Error())
return
}
pairRepo := models.NewSyncPairRepository(h.db)
_, err = pairRepo.GetByID(req.SyncPairID)
if err == sql.ErrNoRows {
writeError(w, http.StatusBadRequest, "sync_pair not found")
return
}
if err != nil {
slog.Error("failed to fetch sync pair", "id", req.SyncPairID, "error", err)
writeError(w, http.StatusInternalServerError, "failed to validate sync pair")
return
}
expr, _ := scheduler.ParseCron(req.CronExpr)
var nextRun *time.Time
if expr != nil {
t := scheduler.NextRun(expr, time.Now().UTC())
nextRun = &t
}
s := &models.Schedule{
SyncPairID: req.SyncPairID,
CronExpr: req.CronExpr,
NextRunAt: nextRun,
Enabled: req.Enabled,
}
repo := models.NewScheduleRepository(h.db)
id, err := repo.Create(s)
if err != nil {
slog.Error("failed to create schedule", "error", err)
writeError(w, http.StatusInternalServerError, "failed to create schedule")
return
}
s.ID = id
pairs, _ := pairRepo.GetAll()
pairMap := make(map[int64]string)
for _, p := range pairs {
pairMap[p.ID] = p.Name
}
w.Header().Set("Location", "/api/schedules/"+strconv.FormatInt(id, 10))
writeJSON(w, scheduleToResp(*s, pairMap), http.StatusCreated)
}
func (h *ScheduleHandler) Update(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid id")
return
}
var req UpdateScheduleRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
repo := models.NewScheduleRepository(h.db)
existing, err := repo.GetByID(id)
if err == sql.ErrNoRows {
writeError(w, http.StatusNotFound, "schedule not found")
return
}
if err != nil {
slog.Error("failed to fetch schedule", "id", id, "error", err)
writeError(w, http.StatusInternalServerError, "failed to fetch schedule")
return
}
if req.CronExpr != "" {
_, err := scheduler.ParseCron(req.CronExpr)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid cron expression: "+err.Error())
return
}
existing.CronExpr = req.CronExpr
expr, _ := scheduler.ParseCron(req.CronExpr)
if expr != nil {
t := scheduler.NextRun(expr, time.Now().UTC())
existing.NextRunAt = &t
}
}
if req.Enabled {
existing.Enabled = true
} else if req.Enabled == false {
existing.Enabled = false
}
if err := repo.Update(existing); err != nil {
slog.Error("failed to update schedule", "id", id, "error", err)
writeError(w, http.StatusInternalServerError, "failed to update schedule")
return
}
pairRepo := models.NewSyncPairRepository(h.db)
pairs, _ := pairRepo.GetAll()
pairMap := make(map[int64]string)
for _, p := range pairs {
pairMap[p.ID] = p.Name
}
writeJSON(w, scheduleToResp(*existing, pairMap))
}
func (h *ScheduleHandler) Delete(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid id")
return
}
repo := models.NewScheduleRepository(h.db)
if err := repo.Delete(id); err != nil {
slog.Error("failed to delete schedule", "id", id, "error", err)
writeError(w, http.StatusInternalServerError, "failed to delete schedule")
return
}
w.WriteHeader(http.StatusNoContent)
}
func scheduleToResp(s models.Schedule, pairMap map[int64]string) ScheduleResponse {
resp := ScheduleResponse{
ID: s.ID,
SyncPairID: s.SyncPairID,
SyncPairName: pairMap[s.SyncPairID],
CronExpr: s.CronExpr,
Enabled: s.Enabled,
CreatedAt: s.CreatedAt.Format(time.RFC3339),
}
if s.NextRunAt != nil {
t := s.NextRunAt.Format(time.RFC3339)
resp.NextRun = &t
}
return resp
}
+35
View File
@@ -3,10 +3,12 @@ package api
import (
"database/sql"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"regexp"
"strconv"
"strings"
"github.com/go-chi/chi/v5"
"github.com/syncserver/internal/models"
@@ -22,6 +24,23 @@ func NewSyncPairHandler(db *sql.DB) *SyncPairHandler {
var directionRegex = regexp.MustCompile(`^(push|pull|mirror)$`)
func ValidatePath(path string) error {
path = strings.TrimSpace(path)
if path == "" {
return fmt.Errorf("path cannot be empty")
}
if strings.Contains(path, "..") {
return fmt.Errorf("path cannot contain '..'")
}
if strings.HasPrefix(path, "-") {
return fmt.Errorf("path cannot start with '-'")
}
if strings.Contains(path, "\x00") {
return fmt.Errorf("path cannot contain null bytes")
}
return nil
}
func (h *SyncPairHandler) List(w http.ResponseWriter, r *http.Request) {
repo := models.NewSyncPairRepository(h.db)
pairs, err := repo.GetAll()
@@ -68,6 +87,14 @@ func (h *SyncPairHandler) Create(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "name, source_path and dest_path are required")
return
}
if err := ValidatePath(req.SourcePath); err != nil {
writeError(w, http.StatusBadRequest, "invalid source_path: "+err.Error())
return
}
if err := ValidatePath(req.DestPath); err != nil {
writeError(w, http.StatusBadRequest, "invalid dest_path: "+err.Error())
return
}
if req.Direction == "" {
req.Direction = "push"
}
@@ -127,6 +154,14 @@ func (h *SyncPairHandler) Update(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "name, source_path and dest_path are required")
return
}
if err := ValidatePath(req.SourcePath); err != nil {
writeError(w, http.StatusBadRequest, "invalid source_path: "+err.Error())
return
}
if err := ValidatePath(req.DestPath); err != nil {
writeError(w, http.StatusBadRequest, "invalid dest_path: "+err.Error())
return
}
if !directionRegex.MatchString(req.Direction) {
writeError(w, http.StatusBadRequest, "direction must be push, pull, or mirror")
return
+2 -2
View File
@@ -45,7 +45,7 @@ func (h *SSEHandler) StreamAll(w http.ResponseWriter, r *http.Request) {
select {
case evt := <-events:
data, _ := json.Marshal(evt)
fmt.Fprintf(w, "event: %s\ndata: %s\n\n", evt.Type, data)
fmt.Fprintf(w, "data: %s\n\n", data)
flusher.Flush()
case <-r.Context().Done():
return
@@ -94,7 +94,7 @@ func (h *SSEHandler) StreamJob(w http.ResponseWriter, r *http.Request) {
select {
case evt := <-events:
data, _ := json.Marshal(evt)
fmt.Fprintf(w, "event: %s\ndata: %s\n\n", evt.Type, data)
fmt.Fprintf(w, "data: %s\n\n", data)
flusher.Flush()
case <-r.Context().Done():
return
+98 -20
View File
@@ -3,9 +3,12 @@ package api
import (
"database/sql"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"os"
"runtime/debug"
"sort"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
@@ -36,41 +39,58 @@ func NewServer(cfg *config.Config, db *sql.DB, engine *syncengine.Engine) *Serve
authHandler := NewAuthHandler(db)
machineHandler := NewMachineHandler(db, engine, cfg)
syncPairHandler := NewSyncPairHandler(db)
scheduleHandler := NewScheduleHandler(db)
jobHandler := NewJobHandler(db, engine)
sseHandler := NewSSEHandler(engine)
sshKeyHandler := NewSSHKeyHandler(db, cfg)
r.Route("/api", func(r chi.Router) {
admin := func(h http.Handler) http.Handler {
return auth.RequireAdmin(auth.RequireAuth(h))
}
authGet := func(h http.Handler) http.Handler {
return auth.RequireAuth(h)
}
r.Route("/api", func(r chi.Router) {
r.Route("/auth", func(r chi.Router) {
r.Post("/login", authHandler.Login)
r.Post("/logout", authHandler.Logout)
r.With(auth.RequireAuth).Get("/me", authHandler.Me)
r.With(authGet).Get("/me", authHandler.Me)
})
r.With(auth.RequireAuth).Route("/machines", func(r chi.Router) {
r.With(authGet).Route("/machines", func(r chi.Router) {
r.Get("/", machineHandler.List)
r.Post("/", machineHandler.Create)
r.Post("/refresh", machineHandler.Refresh)
r.With(admin).Post("/", machineHandler.Create)
r.With(admin).Post("/refresh", machineHandler.Refresh)
r.Get("/{id}", machineHandler.Get)
r.Put("/{id}", machineHandler.Update)
r.Delete("/{id}", machineHandler.Delete)
r.With(admin).Put("/{id}", machineHandler.Update)
r.With(admin).Delete("/{id}", machineHandler.Delete)
r.Post("/{id}/test-wol", machineHandler.TestWoL)
r.Post("/{id}/shutdown", machineHandler.Shutdown)
r.With(admin).Post("/{id}/shutdown", machineHandler.Shutdown)
r.Post("/{id}/test-connection", machineHandler.TestConnection)
r.Post("/{id}/approve-fingerprint", machineHandler.ApproveFingerprint)
r.Post("/{id}/deploy-keys", machineHandler.DeployKeys)
r.With(admin).Post("/{id}/approve-fingerprint", machineHandler.ApproveFingerprint)
r.With(admin).Post("/{id}/deploy-keys", machineHandler.DeployKeys)
})
r.With(auth.RequireAuth).Route("/sync-pairs", func(r chi.Router) {
r.With(authGet).Route("/sync-pairs", func(r chi.Router) {
r.Get("/", syncPairHandler.List)
r.Post("/", syncPairHandler.Create)
r.Get("/{id}", syncPairHandler.Get)
r.Put("/{id}", syncPairHandler.Update)
r.Delete("/{id}", syncPairHandler.Delete)
r.With(admin).Delete("/{id}", syncPairHandler.Delete)
r.Post("/{id}/run", jobHandler.TriggerRun)
})
r.With(auth.RequireAuth).Route("/jobs", func(r chi.Router) {
r.With(authGet).Route("/schedules", func(r chi.Router) {
r.Get("/", scheduleHandler.List)
r.Post("/", scheduleHandler.Create)
r.Get("/{id}", scheduleHandler.Get)
r.Put("/{id}", scheduleHandler.Update)
r.With(admin).Delete("/{id}", scheduleHandler.Delete)
})
r.With(authGet).Route("/jobs", func(r chi.Router) {
r.Get("/", jobHandler.List)
r.Get("/{id}", jobHandler.Get)
r.Post("/{id}/cancel", jobHandler.Cancel)
@@ -79,15 +99,15 @@ func NewServer(cfg *config.Config, db *sql.DB, engine *syncengine.Engine) *Serve
r.Get("/{id}/log/stream", sseHandler.StreamJob)
})
r.With(auth.RequireAuth).Get("/jobs/stream", sseHandler.StreamAll)
r.With(authGet).Get("/jobs/stream", sseHandler.StreamAll)
r.With(auth.RequireAuth).Get("/settings/pubkey", func(w http.ResponseWriter, r *http.Request) {
r.With(authGet).Get("/settings/pubkey", func(w http.ResponseWriter, r *http.Request) {
_, _, pubKey, _ := sshmanager.EnsureServerKey(cfg.SSHDir())
w.Header().Set("Content-Type", "text/plain")
w.Write([]byte(pubKey))
})
r.With(auth.RequireAuth).Get("/settings/info", func(w http.ResponseWriter, r *http.Request) {
r.With(authGet).Get("/settings/info", func(w http.ResponseWriter, r *http.Request) {
_, _, pubKey, _ := sshmanager.EnsureServerKey(cfg.SSHDir())
resp := SettingsInfoResponse{
Version: cfg.Version,
@@ -98,12 +118,12 @@ func NewServer(cfg *config.Config, db *sql.DB, engine *syncengine.Engine) *Serve
json.NewEncoder(w).Encode(resp)
})
r.With(auth.RequireAuth).Route("/ssh-keys", func(r chi.Router) {
r.With(authGet).Route("/ssh-keys", func(r chi.Router) {
r.Get("/", sshKeyHandler.List)
r.Post("/", sshKeyHandler.Create)
r.With(admin).Post("/", sshKeyHandler.Create)
r.Get("/{id}", sshKeyHandler.Get)
r.Delete("/{id}", sshKeyHandler.Delete)
r.Get("/{id}/private", sshKeyHandler.DownloadPrivate)
r.With(admin).Delete("/{id}", sshKeyHandler.Delete)
r.With(admin).Get("/{id}/private", sshKeyHandler.DownloadPrivate)
})
})
@@ -111,6 +131,64 @@ func NewServer(cfg *config.Config, db *sql.DB, engine *syncengine.Engine) *Serve
w.Write([]byte("ok"))
}))
r.Get("/healthz", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("ok"))
}))
r.Get("/readyz", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
db := s.engine.DB()
if _, err := db.Exec("SELECT 1"); err != nil {
http.Error(w, fmt.Sprintf("db query failed: %v", err), http.StatusServiceUnavailable)
return
}
if _, err := db.Exec("SELECT 1"); err != nil {
http.Error(w, fmt.Sprintf("db write test failed: %v", err), http.StatusServiceUnavailable)
return
}
sshDir := s.cfg.SSHDir()
if _, err := os.Stat(sshDir); err != nil {
http.Error(w, fmt.Sprintf("ssh dir not accessible: %v", err), http.StatusServiceUnavailable)
return
}
w.Write([]byte("ok"))
}))
r.Get("/metrics", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain; version=0.0.4")
jobsTotal := s.engine.GetJobsTotal()
var keys []string
for k := range jobsTotal {
keys = append(keys, k)
}
sort.Strings(keys)
for _, status := range keys {
fmt.Fprintf(w, "# HELP syncserver_jobs_total Total jobs by final status\n")
fmt.Fprintf(w, "# TYPE syncserver_jobs_total counter\n")
fmt.Fprintf(w, "syncserver_jobs_total{status=%q} %d\n", status, jobsTotal[status])
}
fmt.Fprintf(w, "# HELP syncserver_jobs_running Currently running jobs\n")
fmt.Fprintf(w, "# TYPE syncserver_jobs_running gauge\n")
fmt.Fprintf(w, "syncserver_jobs_running %d\n", s.engine.GetJobsRunning())
fmt.Fprintf(w, "# HELP syncserver_queue_depth Jobs waiting to run\n")
fmt.Fprintf(w, "# TYPE syncserver_queue_depth gauge\n")
fmt.Fprintf(w, "syncserver_queue_depth %d\n", s.engine.GetQueueDepth())
online, total := s.engine.GetMachineCounts()
fmt.Fprintf(w, "# HELP syncserver_machines_online Online machines count\n")
fmt.Fprintf(w, "# TYPE syncserver_machines_online gauge\n")
fmt.Fprintf(w, "syncserver_machines_online %d\n", online)
fmt.Fprintf(w, "# HELP syncserver_machines_total Total machines\n")
fmt.Fprintf(w, "# TYPE syncserver_machines_total gauge\n")
fmt.Fprintf(w, "syncserver_machines_total %d\n", total)
fmt.Fprintf(w, "# HELP syncserver_up Server is up\n")
fmt.Fprintf(w, "# TYPE syncserver_up gauge\n")
fmt.Fprintf(w, "syncserver_up 1\n")
}))
r.NotFound(func(w http.ResponseWriter, r *http.Request) {
webui.ServeSPA().ServeHTTP(w, r)
})