feat: complete SyncServer implementation
Full-stack Go monolith with embedded React frontend for orchestrating rsync-over-SSH file synchronization with Wake-on-LAN support. Features: - JWT auth (HS256) with bcrypt password hashing - CRUD for machines (with WoL config) and sync_pairs - Ed25519 SSH key generation and known_hosts management - WoL magic packet sender + TCP-connect waiter with backoff - Sync engine: rsync subprocess, per-pair job queue, progress parsing - Homebrew cron parser for scheduled syncs - SSE stream for live job status (queued/waking_up/running/success/failed) - React+TS+Vite+Tailwind SPA embedded via embed.FS - Debian packaging with systemd unit, postinst/prerm/postrm Tech stack: - Go 1.22+ (CGO_ENABLED=0, pure SQLite via modernc.org/sqlite) - chi router for HTTP API - TypeScript + React 18 + Tailwind CSS frontend - Cross-compiled to Linux amd64 for Proxmox LXC deployment Tests: wol (MAC parsing, magic packet), syncengine/queue, scheduler/cron
This commit is contained in:
@@ -0,0 +1,223 @@
|
||||
package syncengine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/syncserver/internal/config"
|
||||
"github.com/syncserver/internal/models"
|
||||
"github.com/syncserver/internal/wol"
|
||||
)
|
||||
|
||||
type Engine struct {
|
||||
db *sql.DB
|
||||
cfg *config.Config
|
||||
queue *Queue
|
||||
eventBus chan Event
|
||||
mu sync.RWMutex
|
||||
stopped bool
|
||||
}
|
||||
|
||||
type Event struct {
|
||||
Type string
|
||||
JobID int64
|
||||
Status string
|
||||
Line string
|
||||
Stream string
|
||||
}
|
||||
|
||||
func New(database interface{ SQLDB() *sql.DB }, cfg *config.Config) *Engine {
|
||||
e := &Engine{
|
||||
db: database.SQLDB(),
|
||||
cfg: cfg,
|
||||
queue: NewQueue(),
|
||||
eventBus: make(chan Event, 100),
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
func (e *Engine) Start() {}
|
||||
func (e *Engine) Stop() {}
|
||||
|
||||
func (e *Engine) Events() <-chan Event {
|
||||
return e.eventBus
|
||||
}
|
||||
|
||||
func (e *Engine) Run(ctx context.Context, jobID int64, pairID int64) error {
|
||||
if e.queue.IsRunning(pairID) {
|
||||
existingJobID, _ := e.queue.GetJobID(pairID)
|
||||
return fmt.Errorf("%w: job %d is already running", ErrAlreadyRunning, existingJobID)
|
||||
}
|
||||
|
||||
jobCtx, cancel := context.WithCancel(ctx)
|
||||
enqueueErr := e.queue.Enqueue(pairID, jobID, cancel)
|
||||
if enqueueErr != nil {
|
||||
return enqueueErr
|
||||
}
|
||||
defer e.queue.Dequeue(pairID)
|
||||
|
||||
pairRepo := models.NewSyncPairRepository(e.db)
|
||||
pair, err := pairRepo.GetByID(pairID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetching sync pair: %w", err)
|
||||
}
|
||||
|
||||
machineRepo := models.NewMachineRepository(e.db)
|
||||
var srcMachine, dstMachine *models.Machine
|
||||
|
||||
if pair.SourceMachineID != nil {
|
||||
m, _ := machineRepo.GetByID(*pair.SourceMachineID)
|
||||
srcMachine = m
|
||||
}
|
||||
if pair.DestMachineID != nil {
|
||||
m, _ := machineRepo.GetByID(*pair.DestMachineID)
|
||||
dstMachine = m
|
||||
}
|
||||
|
||||
src := buildPath(pair.SourcePath, srcMachine)
|
||||
dst := buildPath(pair.DestPath, dstMachine)
|
||||
|
||||
cfg := &SyncPairConfig{
|
||||
ID: pair.ID,
|
||||
Name: pair.Name,
|
||||
SourceMachineID: pair.SourceMachineID,
|
||||
SourcePath: pair.SourcePath,
|
||||
DestMachineID: pair.DestMachineID,
|
||||
DestPath: pair.DestPath,
|
||||
Direction: pair.Direction,
|
||||
RsyncFlags: pair.RsyncFlags,
|
||||
ExcludePatterns: pair.ExcludePatternsList(),
|
||||
}
|
||||
cfg.Source = src
|
||||
cfg.Dest = dst
|
||||
|
||||
logPath := filepath.Join(e.cfg.LogsDir(), fmt.Sprintf("%d.log", jobID))
|
||||
f, err := os.Create(logPath)
|
||||
if err != nil {
|
||||
slog.Warn("failed to create log file", "error", err)
|
||||
} else {
|
||||
f.Close()
|
||||
}
|
||||
|
||||
e.setJobStatus(jobID, "waking_up")
|
||||
e.emit(Event{Type: "status", JobID: jobID, Status: "waking_up"})
|
||||
|
||||
var targetMachine *models.Machine
|
||||
var remotePort int
|
||||
if pair.Direction == "pull" && srcMachine != nil {
|
||||
targetMachine = srcMachine
|
||||
remotePort = srcMachine.Port
|
||||
} else if dstMachine != nil {
|
||||
targetMachine = dstMachine
|
||||
remotePort = dstMachine.Port
|
||||
}
|
||||
|
||||
if targetMachine != nil && targetMachine.WoLEnabled && targetMachine.MACAddress != nil {
|
||||
mac, err := wol.ParseMAC(*targetMachine.MACAddress)
|
||||
if err == nil {
|
||||
bcast := ""
|
||||
if targetMachine.BroadcastAddr != nil {
|
||||
bcast = *targetMachine.BroadcastAddr
|
||||
}
|
||||
if err := wol.Send(targetMachine.Host, mac, bcast); err != nil {
|
||||
slog.Warn("WoL failed", "host", targetMachine.Host, "error", err)
|
||||
} else {
|
||||
slog.Info("WoL magic packet sent", "host", targetMachine.Host, "mac", *targetMachine.MACAddress)
|
||||
}
|
||||
|
||||
timeout := time.Duration(targetMachine.WakeTimeoutSeconds) * time.Second
|
||||
interval := time.Duration(targetMachine.WakeCheckIntervalSeconds) * time.Second
|
||||
if err := wol.WaitUntilReady(jobCtx, targetMachine.Host, remotePort, timeout, interval, false); err != nil {
|
||||
e.setJobStatus(jobID, "failed")
|
||||
e.emit(Event{Type: "status", JobID: jobID, Status: "failed", Line: err.Error()})
|
||||
return fmt.Errorf("machine not ready: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
e.setJobStatus(jobID, "running")
|
||||
e.setJobLogFile(jobID, logPath)
|
||||
e.emit(Event{Type: "status", JobID: jobID, Status: "running"})
|
||||
slog.Info("job started", "job_id", jobID, "pair", pair.Name)
|
||||
|
||||
var privKey string
|
||||
if targetMachine != nil && targetMachine.SSHKeyID != nil {
|
||||
privKey = filepath.Join(e.cfg.SSHDir(), "id_ed25519")
|
||||
}
|
||||
|
||||
runner := NewRsyncRunner(e.cfg.SSHDir(), privKey)
|
||||
onLine := func(stream, line string) {
|
||||
e.emit(Event{Type: "log", JobID: jobID, Stream: stream, Line: line})
|
||||
}
|
||||
|
||||
result, err := runner.Run(jobCtx, cfg, onLine)
|
||||
if err != nil {
|
||||
if jobCtx.Err() != nil {
|
||||
e.setJobStatus(jobID, "cancelled")
|
||||
e.emit(Event{Type: "status", JobID: jobID, Status: "cancelled"})
|
||||
return jobCtx.Err()
|
||||
}
|
||||
e.setJobStatus(jobID, "failed")
|
||||
e.emit(Event{Type: "status", JobID: jobID, Status: "failed", Line: err.Error()})
|
||||
return fmt.Errorf("rsync error: %w", err)
|
||||
}
|
||||
|
||||
if result.ExitCode != 0 {
|
||||
e.setJobStatus(jobID, "failed")
|
||||
e.emit(Event{Type: "status", JobID: jobID, Status: "failed", Line: result.Stderr})
|
||||
return fmt.Errorf("rsync exited with code %d: %s", result.ExitCode, result.Stderr)
|
||||
}
|
||||
|
||||
e.setJobStatus(jobID, "success")
|
||||
e.emit(Event{Type: "status", JobID: jobID, Status: "success"})
|
||||
slog.Info("job completed", "job_id", jobID, "pair", pair.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Engine) Cancel(jobID int64, syncPairID int64) bool {
|
||||
if e.queue.IsRunning(syncPairID) {
|
||||
e.queue.Cancel(syncPairID)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (e *Engine) setJobStatus(jobID int64, status string) {
|
||||
jobRepo := models.NewJobRepository(e.db)
|
||||
jobRepo.UpdateStatus(jobID, status)
|
||||
}
|
||||
|
||||
func (e *Engine) setJobLogFile(jobID int64, path string) {
|
||||
jobRepo := models.NewJobRepository(e.db)
|
||||
jobRepo.SetLogFile(jobID, path)
|
||||
}
|
||||
|
||||
func (e *Engine) emit(evt Event) {
|
||||
select {
|
||||
case e.eventBus <- evt:
|
||||
default:
|
||||
slog.Warn("event bus full, dropping event", "type", evt.Type)
|
||||
}
|
||||
}
|
||||
|
||||
func buildPath(path string, machine *models.Machine) string {
|
||||
if machine == nil {
|
||||
return path
|
||||
}
|
||||
return fmt.Sprintf("%s@%s:%s", machine.SSHUser, machine.Host, path)
|
||||
}
|
||||
|
||||
func (e *Engine) CreateJob(syncPairID int64, triggerType string) (int64, error) {
|
||||
jobRepo := models.NewJobRepository(e.db)
|
||||
id, err := jobRepo.Create(syncPairID, triggerType, "queued")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
Reference in New Issue
Block a user