Phase A-E: stability, security, observability, and test coverage
Phase A - Stability: - Engine.Start(): recovers orphaned jobs (running/queued/waking_up) after crash - Engine.Stop(): graceful shutdown - cancels all in-flight jobs and waits - Queue keyed by jobID (not syncPairID): cancel now targets exact job - Local rsync uses jobCtx (context.Background() replaced) - Migrations wrapped in transactions; checksums stored Phase B - Security: - admin/admin default removed: SYNCSERVER_ADMIN_PASSWORD required on first run - Path validation: rejects .., leading -, null bytes in sync pair paths - Rsync flags allowlist: dangerous flags blocked (--rsync-path, -e, --files-from) - Shell concat in RunRemote replaced with proper sh -c escaping - knownhosts: replaced custom parser with golang.org/x/crypto/ssh/knownhosts - RequireAdmin wired: machine CRUD, SSH key ops, settings require admin role - deploy-keys: uses authorized_keys only (no private key upload) - Hardcoded /var/lib/syncserver/ssh paths replaced with cfg.SSHDir() Phase C - Operational: - /readyz health check: DB query + SSH dir accessibility - /metrics endpoint: Prometheus text format (jobs, queue, machines) - Event struct JSON tags: job_id, machine_id, type (snake_case) - EventBus broadcast: fanned out to all subscribers - SQLite VACUUM INTO backup: scheduled before cleanup if BackupDir set - Filesystem job log cleanup: removes .log files for purged jobs - Backup retention: old backups auto-purged Phase D - Frontend: - Schedules page: REST API + full CRUD UI for cron schedules - Dashboard: cancel button for running/queued jobs - JobDetail: server-side log download via API - Settings: displays data_dir from server - 404 page: proper NotFound component Phase E - Tests: - auth_test.go: JWT, bcrypt, middleware, seed (18 tests) - models_test.go: Job, Machine, SyncPair, Schedule repos (18 tests) - go test -race: no data races found
This commit is contained in:
+118
-29
@@ -18,26 +18,31 @@ import (
|
||||
)
|
||||
|
||||
type Engine struct {
|
||||
db *sql.DB
|
||||
cfg *config.Config
|
||||
queue *Queue
|
||||
eventBus *EventBus
|
||||
mu sync.RWMutex
|
||||
stopped bool
|
||||
lastProbeAt atomic.Int64
|
||||
db *sql.DB
|
||||
cfg *config.Config
|
||||
queue *Queue
|
||||
eventBus *EventBus
|
||||
mu sync.RWMutex
|
||||
stopped bool
|
||||
lastProbeAt atomic.Int64
|
||||
jobsWG sync.WaitGroup
|
||||
stopCh chan struct{}
|
||||
|
||||
jobsTotal map[string]int64
|
||||
jobsTotalMu sync.Mutex
|
||||
}
|
||||
|
||||
type Event struct {
|
||||
Type string
|
||||
JobID int64
|
||||
MachineID int64
|
||||
Key string
|
||||
Value string
|
||||
Line string
|
||||
Stream string
|
||||
Progress *ProgressFields
|
||||
TotalBytes int64
|
||||
SentBytes int64
|
||||
Type string `json:"type"`
|
||||
JobID int64 `json:"job_id"`
|
||||
MachineID int64 `json:"machine_id"`
|
||||
Key string `json:"key,omitempty"`
|
||||
Value string `json:"value,omitempty"`
|
||||
Line string `json:"line,omitempty"`
|
||||
Stream string `json:"stream,omitempty"`
|
||||
Progress *ProgressFields `json:"progress,omitempty"`
|
||||
TotalBytes int64 `json:"total_bytes,omitempty"`
|
||||
SentBytes int64 `json:"sent_bytes,omitempty"`
|
||||
}
|
||||
|
||||
func New(database interface{ SQLDB() *sql.DB }, cfg *config.Config) *Engine {
|
||||
@@ -46,12 +51,56 @@ func New(database interface{ SQLDB() *sql.DB }, cfg *config.Config) *Engine {
|
||||
cfg: cfg,
|
||||
queue: NewQueue(),
|
||||
eventBus: NewEventBus(200),
|
||||
stopCh: make(chan struct{}),
|
||||
jobsTotal: map[string]int64{
|
||||
"success": 0,
|
||||
"failed": 0,
|
||||
"cancelled": 0,
|
||||
},
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
func (e *Engine) Start() {}
|
||||
func (e *Engine) Stop() {}
|
||||
func (e *Engine) Start() {
|
||||
e.recoverOrphanedJobs()
|
||||
slog.Info("engine started")
|
||||
}
|
||||
|
||||
func (e *Engine) Stop() {
|
||||
e.mu.Lock()
|
||||
if e.stopped {
|
||||
e.mu.Unlock()
|
||||
return
|
||||
}
|
||||
e.stopped = true
|
||||
e.mu.Unlock()
|
||||
|
||||
close(e.stopCh)
|
||||
|
||||
runningIDs := e.queue.RunningJobs()
|
||||
for _, id := range runningIDs {
|
||||
e.queue.Cancel(id, false)
|
||||
}
|
||||
|
||||
e.jobsWG.Wait()
|
||||
slog.Info("engine stopped")
|
||||
}
|
||||
|
||||
func (e *Engine) recoverOrphanedJobs() {
|
||||
jobRepo := models.NewJobRepository(e.db)
|
||||
jobs, err := jobRepo.GetByStatusAny([]string{"queued", "waking_up", "running"})
|
||||
if err != nil {
|
||||
slog.Warn("failed to recover orphaned jobs", "error", err)
|
||||
return
|
||||
}
|
||||
for _, j := range jobs {
|
||||
slog.Warn("recovered orphaned job, marking as failed",
|
||||
"job_id", j.ID, "pair_id", j.SyncPairID, "status", j.Status)
|
||||
jobRepo.UpdateStatus(j.ID, "failed")
|
||||
jobRepo.SetError(j.ID, "crash_recovery",
|
||||
fmt.Sprintf("job was %s when server shut down unexpectedly", j.Status))
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) SubscribeJob(jobID int64) (chan Event, func()) {
|
||||
return e.eventBus.Subscribe(jobID)
|
||||
@@ -84,9 +133,12 @@ func (e *Engine) wakeMachine(ctx context.Context, m *models.Machine) {
|
||||
}
|
||||
|
||||
func (e *Engine) Run(ctx context.Context, jobID int64, pairID int64) error {
|
||||
if e.queue.IsRunning(pairID) {
|
||||
existingJobID, _ := e.queue.GetJobID(pairID)
|
||||
return fmt.Errorf("%w: job %d is already running", ErrAlreadyRunning, existingJobID)
|
||||
if e.queue.IsRunning(jobID) {
|
||||
return fmt.Errorf("%w: job %d is already running", ErrAlreadyRunning, jobID)
|
||||
}
|
||||
|
||||
if existingJobID, exists := e.queue.GetByPair(pairID); exists {
|
||||
return fmt.Errorf("%w: job %d is already running for this sync pair", ErrAlreadyRunning, existingJobID)
|
||||
}
|
||||
|
||||
jobCtx, cancel := context.WithCancel(ctx)
|
||||
@@ -99,7 +151,10 @@ func (e *Engine) Run(ctx context.Context, jobID int64, pairID int64) error {
|
||||
if enqueueErr != nil {
|
||||
return enqueueErr
|
||||
}
|
||||
defer e.queue.Dequeue(pairID)
|
||||
defer e.queue.Dequeue(jobID)
|
||||
|
||||
e.jobsWG.Add(1)
|
||||
defer e.jobsWG.Done()
|
||||
|
||||
pairRepo := models.NewSyncPairRepository(e.db)
|
||||
pair, err := pairRepo.GetByID(pairID)
|
||||
@@ -381,17 +436,18 @@ func (e *Engine) resolveSSHKey(machine *models.Machine) (string, error) {
|
||||
return sshKey.PrivateKeyPath, nil
|
||||
}
|
||||
|
||||
func (e *Engine) Cancel(jobID int64, syncPairID int64, byUser bool) bool {
|
||||
if e.queue.IsRunning(syncPairID) {
|
||||
e.queue.Cancel(syncPairID, byUser)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
func (e *Engine) Cancel(jobID int64, byUser bool) bool {
|
||||
return e.queue.Cancel(jobID, byUser)
|
||||
}
|
||||
|
||||
func (e *Engine) setJobStatus(jobID int64, status string) {
|
||||
jobRepo := models.NewJobRepository(e.db)
|
||||
jobRepo.UpdateStatus(jobID, status)
|
||||
if status == "success" || status == "failed" || status == "cancelled" {
|
||||
e.jobsTotalMu.Lock()
|
||||
e.jobsTotal[status]++
|
||||
e.jobsTotalMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) setJobLogFile(jobID int64, path string) {
|
||||
@@ -495,3 +551,36 @@ func (e *Engine) ProbeAllMachines() {
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func (e *Engine) GetJobsTotal() map[string]int64 {
|
||||
e.jobsTotalMu.Lock()
|
||||
defer e.jobsTotalMu.Unlock()
|
||||
return e.jobsTotal
|
||||
}
|
||||
|
||||
func (e *Engine) GetJobsRunning() int64 {
|
||||
return int64(len(e.queue.RunningJobs()))
|
||||
}
|
||||
|
||||
func (e *Engine) GetQueueDepth() int64 {
|
||||
return int64(e.queue.Len())
|
||||
}
|
||||
|
||||
func (e *Engine) GetMachineCounts() (online, total int64) {
|
||||
machineRepo := models.NewMachineRepository(e.db)
|
||||
ms, err := machineRepo.GetAll()
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
for _, m := range ms {
|
||||
total++
|
||||
if m.Status == "online" {
|
||||
online++
|
||||
}
|
||||
}
|
||||
return online, total
|
||||
}
|
||||
|
||||
func (e *Engine) DB() *sql.DB {
|
||||
return e.db
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user