Phase A-E: stability, security, observability, and test coverage
Phase A - Stability: - Engine.Start(): recovers orphaned jobs (running/queued/waking_up) after crash - Engine.Stop(): graceful shutdown - cancels all in-flight jobs and waits - Queue keyed by jobID (not syncPairID): cancel now targets exact job - Local rsync uses jobCtx (context.Background() replaced) - Migrations wrapped in transactions; checksums stored Phase B - Security: - admin/admin default removed: SYNCSERVER_ADMIN_PASSWORD required on first run - Path validation: rejects .., leading -, null bytes in sync pair paths - Rsync flags allowlist: dangerous flags blocked (--rsync-path, -e, --files-from) - Shell concat in RunRemote replaced with proper sh -c escaping - knownhosts: replaced custom parser with golang.org/x/crypto/ssh/knownhosts - RequireAdmin wired: machine CRUD, SSH key ops, settings require admin role - deploy-keys: uses authorized_keys only (no private key upload) - Hardcoded /var/lib/syncserver/ssh paths replaced with cfg.SSHDir() Phase C - Operational: - /readyz health check: DB query + SSH dir accessibility - /metrics endpoint: Prometheus text format (jobs, queue, machines) - Event struct JSON tags: job_id, machine_id, type (snake_case) - EventBus broadcast: fanned out to all subscribers - SQLite VACUUM INTO backup: scheduled before cleanup if BackupDir set - Filesystem job log cleanup: removes .log files for purged jobs - Backup retention: old backups auto-purged Phase D - Frontend: - Schedules page: REST API + full CRUD UI for cron schedules - Dashboard: cancel button for running/queued jobs - JobDetail: server-side log download via API - Settings: displays data_dir from server - 404 page: proper NotFound component Phase E - Tests: - auth_test.go: JWT, bcrypt, middleware, seed (18 tests) - models_test.go: Job, Machine, SyncPair, Schedule repos (18 tests) - go test -race: no data races found
This commit is contained in:
@@ -4,11 +4,97 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var allowedRsyncFlags = map[string]bool{
|
||||
"-v": true,
|
||||
"-vv": true,
|
||||
"-q": true,
|
||||
"-h": true,
|
||||
"-P": true,
|
||||
"-n": true,
|
||||
"-z": true,
|
||||
"-c": true,
|
||||
"-u": true,
|
||||
"-W": true,
|
||||
"-i": true,
|
||||
"-a": true,
|
||||
"-r": true,
|
||||
"-l": true,
|
||||
"-t": true,
|
||||
"-p": true,
|
||||
"-g": true,
|
||||
"-o": true,
|
||||
"-D": true,
|
||||
"--verbose": true,
|
||||
"--quiet": true,
|
||||
"--help": true,
|
||||
"--partial": true,
|
||||
"--partial-dir": true,
|
||||
"--delay-updates": true,
|
||||
"--delete": true,
|
||||
"--delete-before": true,
|
||||
"--delete-after": true,
|
||||
"--delete-excluded": true,
|
||||
"--exclude": true,
|
||||
"--exclude-from": true,
|
||||
"--dry-run": true,
|
||||
"--compress": true,
|
||||
"--skip-compress": true,
|
||||
"--whole-file": true,
|
||||
"--checksum": true,
|
||||
"--update": true,
|
||||
"--existing": true,
|
||||
"--ignore-existing": true,
|
||||
"--remove-source-files": true,
|
||||
"--chmod": true,
|
||||
"--owner": true,
|
||||
"--group": true,
|
||||
"--perms": true,
|
||||
"--executability": true,
|
||||
"--acls": true,
|
||||
"--xattrs": true,
|
||||
"--numeric-ids": true,
|
||||
"--fake-super": true,
|
||||
"--bwlimit": true,
|
||||
"--max-size": true,
|
||||
"--min-size": true,
|
||||
"--append": true,
|
||||
"--append-verify": true,
|
||||
"--itemize-changes": true,
|
||||
}
|
||||
|
||||
var blockedRsyncFlags = map[string]bool{
|
||||
"--rsync-path": true,
|
||||
"-e": true,
|
||||
"--files-from": true,
|
||||
"--read-batch": true,
|
||||
"--write-batch": true,
|
||||
"--log-file": true,
|
||||
}
|
||||
|
||||
func isSafeRsyncFlag(flag string) bool {
|
||||
if allowedRsyncFlags[flag] {
|
||||
return true
|
||||
}
|
||||
safePrefixes := []string{
|
||||
"-a", "-v", "-z", "-P", "-n", "-c", "-u", "-W", "-i",
|
||||
"--exclude=", "--chmod=", "--bwlimit=", "--max-size=", "--min-size=",
|
||||
"--partial-dir=", "--skip-compress=",
|
||||
}
|
||||
for _, p := range safePrefixes {
|
||||
if strings.HasPrefix(flag, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type RsyncResult struct {
|
||||
ExitCode int
|
||||
Stdout string
|
||||
@@ -40,13 +126,13 @@ func NewRsyncRunner(sshDir, privKey string) *RsyncRunner {
|
||||
}
|
||||
|
||||
func (r *RsyncRunner) Run(ctx context.Context, pair *SyncPairConfig, onLine func(stream string, line string)) (*RsyncResult, error) {
|
||||
cmd := r.buildRsyncCmd(pair)
|
||||
cmd := r.buildRsyncCmd(ctx, pair)
|
||||
return r.runCmd(ctx, cmd, onLine)
|
||||
}
|
||||
|
||||
func (r *RsyncRunner) buildRsyncCmd(pair *SyncPairConfig) *exec.Cmd {
|
||||
func (r *RsyncRunner) buildRsyncCmd(ctx context.Context, pair *SyncPairConfig) *exec.Cmd {
|
||||
args := r.buildArgs(pair)
|
||||
cmd := exec.CommandContext(context.Background(), "rsync", args...)
|
||||
cmd := exec.CommandContext(ctx, "rsync", args...)
|
||||
if r.privKey != "" {
|
||||
sshCmd := fmt.Sprintf("ssh -i %s -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=%s",
|
||||
r.privKey, strings.TrimRight(r.sshDir, "/")+"/known_hosts")
|
||||
@@ -59,7 +145,19 @@ func (r *RsyncRunner) buildArgs(pair *SyncPairConfig) []string {
|
||||
var args []string
|
||||
|
||||
flags := strings.Fields(pair.RsyncFlags)
|
||||
args = append(args, flags...)
|
||||
for _, flag := range flags {
|
||||
if strings.HasPrefix(flag, "-") {
|
||||
if blockedRsyncFlags[flag] {
|
||||
slog.Warn("blocked dangerous rsync flag", "flag", flag)
|
||||
continue
|
||||
}
|
||||
if !isSafeRsyncFlag(flag) {
|
||||
slog.Warn("disallowed rsync flag", "flag", flag)
|
||||
continue
|
||||
}
|
||||
}
|
||||
args = append(args, flag)
|
||||
}
|
||||
|
||||
for _, pattern := range pair.ExcludePatterns {
|
||||
args = append(args, "--exclude="+pattern)
|
||||
@@ -69,6 +167,7 @@ func (r *RsyncRunner) buildArgs(pair *SyncPairConfig) []string {
|
||||
args = append(args, "--delete")
|
||||
}
|
||||
|
||||
args = append(args, "--")
|
||||
src := ensureDirSlash(pair.Source)
|
||||
if pair.Direction == "pull" {
|
||||
args = append(args, pair.Dest, src)
|
||||
@@ -113,12 +212,23 @@ func (r *RsyncRunner) RunRemote(ctx context.Context, pair *SyncPairConfig, src *
|
||||
|
||||
innerSSH := fmt.Sprintf("ssh -i %s -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=%s",
|
||||
destKey, filepath.Join(r.sshDir, "known_hosts"))
|
||||
rsyncFlags := strings.Join(args[:len(args)-2], " ")
|
||||
rsyncFlags := args[:len(args)-2]
|
||||
sourcePath := args[len(args)-2]
|
||||
destPath := args[len(args)-1]
|
||||
|
||||
remoteCmd := fmt.Sprintf("rsync %s -e %q %s %s",
|
||||
rsyncFlags, innerSSH, sourcePath, destPath)
|
||||
var rsyncCmd []string
|
||||
rsyncCmd = append(rsyncCmd, "rsync")
|
||||
rsyncCmd = append(rsyncCmd, "-e")
|
||||
rsyncCmd = append(rsyncCmd, innerSSH)
|
||||
rsyncCmd = append(rsyncCmd, rsyncFlags...)
|
||||
rsyncCmd = append(rsyncCmd, sourcePath, destPath)
|
||||
|
||||
remoteCmd := "rsync"
|
||||
for _, arg := range rsyncFlags {
|
||||
remoteCmd += " " + strconv.Quote(arg)
|
||||
}
|
||||
remoteCmd += " -e " + strconv.Quote(innerSSH) + " " + strconv.Quote(sourcePath) + " " + strconv.Quote(destPath)
|
||||
remoteCmd = "sh -c " + strconv.Quote(remoteCmd)
|
||||
|
||||
sshArgs := []string{
|
||||
"-i", src.PrivKey,
|
||||
|
||||
Reference in New Issue
Block a user