proyectos/move-data-nas
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add SSH key management, job history persistence, and live streaming

Browse Source 
- SSH key management: generate ed25519 keypairs or import public keys
  from UI (/ssh-keys), per-machine key selection in Machines form,
  one-time private key download with hash verification
- Fix engine to use machine-specific SSH key (was hardcoded to server key)
- Job log persistence: write to job_logs table (DB) with batched inserts,
  buffer of 50 lines; GetAllFiltered with status/pair/date range filters
- EventBus refactor: per-job subscriber channels, global channel, non-blocking
- SSE endpoints: /jobs/stream (all), /jobs/:id/log/stream (per-job live)
- JobDetail page: live log streaming, auto-scroll, cancel, duration
- JobHistory: filters (pair, status, date range), pagination, link to detail
- Cleanup scheduler: daily purge of job_logs and finished jobs older than
  SYNCSERVER_RETENTION_DAYS (default 30)
- Migration 0002: indexes on job_logs(job_id), jobs(status,created_at),
  jobs(sync_pair_id)
This commit is contained in:
Daniel Arroyo
2026-07-07 20:36:11 -04:00
parent 5374ab81cc
commit bfa006f4ab
22 changed files with 1424 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/dto.go
+33 -7
View File
@@ -55,13 +55,39 @@ type SyncPairResponse struct {
}
type JobResponse struct {
ID int64 `json:"id"`
SyncPairID int64 `json:"sync_pair_id"`
TriggerType string `json:"trigger_type"`
Status string `json:"status"`
StartedAt *string `json:"started_at"`
FinishedAt *string `json:"finished_at"`
LogFile *string `json:"log_file"`
ID int64 `json:"id"`
SyncPairID int64 `json:"sync_pair_id"`
TriggerType string `json:"trigger_type"`
Status string `json:"status"`
StartedAt *string `json:"started_at"`
FinishedAt *string `json:"finished_at"`
LogFile *string `json:"log_file"`
DurationSeconds *int64 `json:"duration_seconds,omitempty"`
LogLineCount *int64 `json:"log_line_count,omitempty"`
}
type LogLineResponse struct {
ID int64 `json:"id"`
JobID int64 `json:"job_id"`
Stream string `json:"stream"`
Content string `json:"content"`
Timestamp string `json:"timestamp"`
}
type SSHKeyRequest struct {
Label string `json:"label"`
Generate bool `json:"generate"`
PublicKey string `json:"public_key"`
}
type SSHKeyResponse struct {
ID int64 `json:"id"`
Label string `json:"label"`
PublicKey string `json:"public_key"`
Fingerprint string `json:"fingerprint"`
InUse bool `json:"in_use"`
HasPrivateKey bool `json:"has_private_key"`
CreatedAt string `json:"created_at"`
}
type ErrorResponse struct {
internal/api/handlers_jobs.go
+75 -15
View File
@@ -2,6 +2,7 @@ package api
import (
"database/sql"
"fmt"
"net/http"
"os"
"strconv"
@@ -28,8 +29,30 @@ func (h *JobHandler) List(w http.ResponseWriter, r *http.Request) {
limit = 50
}
repo := models.NewJobRepository(h.db)
jobs, err := repo.GetAll(limit, offset)
var syncPairID *int64
if spidStr := r.URL.Query().Get("sync_pair_id"); spidStr != "" {
if spid, err := strconv.ParseInt(spidStr, 10, 64); err == nil {
syncPairID = &spid
}
}
status := r.URL.Query().Get("status")
triggerType := r.URL.Query().Get("trigger_type")
var from, to *time.Time
if fromStr := r.URL.Query().Get("from"); fromStr != "" {
if t, err := time.Parse(time.RFC3339, fromStr); err == nil {
from = &t
}
}
if toStr := r.URL.Query().Get("to"); toStr != "" {
if t, err := time.Parse(time.RFC3339, toStr); err == nil {
to = &t
}
}
repo := models.NewJobLogRepository(h.db)
jobs, total, err := repo.GetAllFiltered(limit, offset, syncPairID, status, triggerType, from, to)
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to fetch jobs")
return
@@ -37,8 +60,9 @@ func (h *JobHandler) List(w http.ResponseWriter, r *http.Request) {
out := make([]JobResponse, len(jobs))
for i, j := range jobs {
out[i] = jobToResp(j)
out[i] = jobWithStatsToResp(j)
}
w.Header().Set("X-Total-Count", fmt.Sprintf("%d", total))
writeJSON(w, out)
}
@@ -120,30 +144,59 @@ func (h *JobHandler) TriggerRun(w http.ResponseWriter, r *http.Request) {
writeJSON(w, jobToResp(*j), http.StatusCreated)
}
func (h *JobHandler) StreamLog(w http.ResponseWriter, r *http.Request) {
func (h *JobHandler) GetLog(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid id")
return
}
flusher, ok := w.(http.Flusher)
if !ok {
writeError(w, http.StatusInternalServerError, "streaming not supported")
offset, _ := strconv.Atoi(r.URL.Query().Get("offset"))
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
if limit <= 0 {
limit = 1000
}
logRepo := models.NewJobLogRepository(h.db)
logs, err := logRepo.GetByJobID(id, limit, offset)
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to fetch logs")
return
}
w.Header().Set("Content-Type", "text/plain")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
flusher.Flush()
count, _ := logRepo.CountByJobID(id)
w.Header().Set("X-Total-Count", fmt.Sprintf("%d", count))
writeJSON(w, logs)
}
func (h *JobHandler) DownloadLog(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid id")
return
}
jobRepo := models.NewJobRepository(h.db)
j, err := jobRepo.GetByID(id)
if err == nil && j.LogFile != nil {
data, _ := os.ReadFile(*j.LogFile)
w.Write(data)
flusher.Flush()
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to fetch job")
return
}
if j.LogFile != nil {
data, err := os.ReadFile(*j.LogFile)
if err == nil {
w.Header().Set("Content-Type", "text/plain")
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="job-%d.log"`, id))
w.Write(data)
return
}
}
logRepo := models.NewJobLogRepository(h.db)
logs, _ := logRepo.GetByJobID(id, 100000, 0)
for _, l := range logs {
fmt.Fprintf(w, "[%s] %s\n", l.Timestamp.Format(time.RFC3339), l.Content)
}
}
@@ -165,3 +218,10 @@ func jobToResp(j models.Job) JobResponse {
}
return resp
}
func jobWithStatsToResp(j models.JobWithStats) JobResponse {
resp := jobToResp(j.Job)
resp.DurationSeconds = j.DurationSeconds
resp.LogLineCount = &j.LogLineCount
return resp
}
internal/api/handlers_sshkeys.go
+223
View File
@@ -0,0 +1,223 @@
package api
import (
"crypto/sha256"
"database/sql"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/go-chi/chi/v5"
"github.com/syncserver/internal/config"
"github.com/syncserver/internal/models"
"github.com/syncserver/internal/sshmanager"
)
type SSHKeyHandler struct {
db *sql.DB
cfg *config.Config
}
func NewSSHKeyHandler(db *sql.DB, cfg *config.Config) *SSHKeyHandler {
return &SSHKeyHandler{db: db, cfg: cfg}
}
func (h *SSHKeyHandler) List(w http.ResponseWriter, r *http.Request) {
repo := models.NewSSHKeyRepository(h.db)
machineRepo := models.NewMachineRepository(h.db)
keys, err := repo.GetAll()
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to fetch ssh keys")
return
}
machines, _ := machineRepo.GetAll()
out := make([]SSHKeyResponse, len(keys))
for i, k := range keys {
inUse := false
for _, m := range machines {
if m.SSHKeyID != nil && *m.SSHKeyID == k.ID {
inUse = true
break
}
}
hasPriv := false
if _, err := os.Stat(k.PrivateKeyPath); err == nil {
hasPriv = true
}
fp, _ := sshmanager.Fingerprint(k.PublicKey)
out[i] = SSHKeyResponse{
ID: k.ID,
Label: k.Label,
PublicKey: k.PublicKey,
Fingerprint: fp,
InUse: inUse,
HasPrivateKey: hasPriv,
CreatedAt: k.CreatedAt.Format("2006-01-02T15:04:05Z07:00"),
}
}
writeJSON(w, out)
}
func (h *SSHKeyHandler) Create(w http.ResponseWriter, r *http.Request) {
var req SSHKeyRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
if req.Label == "" {
writeError(w, http.StatusBadRequest, "label is required")
return
}
if !req.Generate && req.PublicKey == "" {
writeError(w, http.StatusBadRequest, "either generate=true or public_key is required")
return
}
keysDir := filepath.Join(h.cfg.SSHDir(), "keys")
if err := os.MkdirAll(keysDir, 0700); err != nil {
writeError(w, http.StatusInternalServerError, "failed to create keys directory")
return
}
var pubKey, privPath, fp string
var err error
if req.Generate {
privPath, _, pubKey, fp, err = sshmanager.GenerateKeyPair(req.Label, keysDir)
if err != nil {
writeError(w, http.StatusInternalServerError, fmt.Sprintf("generating key: %v", err))
return
}
} else {
pubKey = strings.TrimSpace(req.PublicKey)
fp, err = sshmanager.Fingerprint(pubKey)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid public key format")
return
}
privPath = ""
}
repo := models.NewSSHKeyRepository(h.db)
id, err := repo.Create(req.Label, privPath, pubKey)
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to store ssh key")
return
}
w.Header().Set("Location", "/api/ssh-keys/"+strconv.FormatInt(id, 10))
writeJSON(w, SSHKeyResponse{
ID: id,
Label: req.Label,
PublicKey: pubKey,
Fingerprint: fp,
InUse: false,
HasPrivateKey: privPath != "",
}, http.StatusCreated)
}
func (h *SSHKeyHandler) Get(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid id")
return
}
repo := models.NewSSHKeyRepository(h.db)
k, err := repo.GetByID(id)
if err == sql.ErrNoRows {
writeError(w, http.StatusNotFound, "ssh key not found")
return
}
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to fetch ssh key")
return
}
machineRepo := models.NewMachineRepository(h.db)
machines, _ := machineRepo.GetAll()
inUse := false
for _, m := range machines {
if m.SSHKeyID != nil && *m.SSHKeyID == k.ID {
inUse = true
break
}
}
hasPriv := false
if _, err := os.Stat(k.PrivateKeyPath); err == nil {
hasPriv = true
}
fp, _ := sshmanager.Fingerprint(k.PublicKey)
writeJSON(w, SSHKeyResponse{
ID: k.ID,
Label: k.Label,
PublicKey: k.PublicKey,
Fingerprint: fp,
InUse: inUse,
HasPrivateKey: hasPriv,
CreatedAt: k.CreatedAt.Format("2006-01-02T15:04:05Z07:00"),
})
}
func (h *SSHKeyHandler) Delete(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid id")
return
}
repo := models.NewSSHKeyRepository(h.db)
machineRepo := models.NewMachineRepository(h.db)
machines, _ := machineRepo.GetAll()
for _, m := range machines {
if m.SSHKeyID != nil && *m.SSHKeyID == id {
writeError(w, http.StatusConflict, "ssh key is in use by machines")
return
}
}
k, err := repo.GetByID(id)
if err == nil && k.PrivateKeyPath != "" {
os.Remove(k.PrivateKeyPath)
os.Remove(k.PrivateKeyPath + ".pub")
}
if err := repo.Delete(id); err != nil {
writeError(w, http.StatusInternalServerError, "failed to delete ssh key")
return
}
w.WriteHeader(http.StatusNoContent)
}
func (h *SSHKeyHandler) DownloadPrivate(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid id")
return
}
repo := models.NewSSHKeyRepository(h.db)
k, err := repo.GetByID(id)
if err == sql.ErrNoRows {
writeError(w, http.StatusNotFound, "ssh key not found")
return
}
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to fetch ssh key")
return
}
if k.PrivateKeyPath == "" {
writeError(w, http.StatusNotFound, "no private key available for this entry")
return
}
data, err := os.ReadFile(k.PrivateKeyPath)
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to read private key")
return
}
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s.key"`, k.Label))
w.Header().Set("X-Private-Key-Hash", fmt.Sprintf("sha256:%x", sha256.Sum256(data)))
io.WriteString(w, string(data))
}
internal/api/handlers_ws.go
+47 -11
View File
@@ -18,11 +18,51 @@ func NewSSEHandler(engine *syncengine.Engine) *SSEHandler {
return &SSEHandler{engine: engine}
}
func (h *SSEHandler) Stream(w http.ResponseWriter, r *http.Request) {
func (h *SSEHandler) StreamAll(w http.ResponseWriter, r *http.Request) {
flusher, ok := w.(http.Flusher)
if !ok {
http.Error(w, "SSE not supported", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("X-Accel-Buffering", "no")
flusher.Flush()
if h.engine == nil {
return
}
events, unsub := h.engine.SubscribeGlobal()
defer unsub()
for {
select {
case evt := <-events:
data, _ := json.Marshal(evt)
fmt.Fprintf(w, "event: %s\ndata: %s\n\n", evt.Type, data)
flusher.Flush()
case <-r.Context().Done():
return
case <-time.After(30 * time.Second):
fmt.Fprintf(w, ": keepalive\n\n")
flusher.Flush()
}
}
}
func (h *SSEHandler) StreamJob(w http.ResponseWriter, r *http.Request) {
jobIDStr := r.URL.Query().Get("job_id")
var filterJobID int64
if jobIDStr != "" {
filterJobID, _ = strconv.ParseInt(jobIDStr, 10, 64)
if jobIDStr == "" {
http.Error(w, "job_id required", http.StatusBadRequest)
return
}
jobID, err := strconv.ParseInt(jobIDStr, 10, 64)
if err != nil {
http.Error(w, "invalid job_id", http.StatusBadRequest)
return
}
flusher, ok := w.(http.Flusher)
@@ -35,27 +75,23 @@ func (h *SSEHandler) Stream(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("X-Accel-Buffering", "no")
flusher.Flush()
if h.engine == nil {
return
}
events := h.engine.Events()
events, unsub := h.engine.SubscribeJob(jobID)
defer unsub()
for {
select {
case evt := <-events:
if filterJobID != 0 && evt.JobID != filterJobID {
continue
}
data, _ := json.Marshal(evt)
fmt.Fprintf(w, "event: %s\ndata: %s\n\n", evt.Type, data)
flusher.Flush()
case <-r.Context().Done():
return
case <-time.After(30 * time.Second):
fmt.Fprintf(w, ": keepalive\n\n")
flusher.Flush()
internal/api/router.go
+13 -2
View File
@@ -35,6 +35,7 @@ func NewServer(cfg *config.Config, db *sql.DB, engine *syncengine.Engine) *Serve
syncPairHandler := NewSyncPairHandler(db)
jobHandler := NewJobHandler(db, engine)
sseHandler := NewSSEHandler(engine)
sshKeyHandler := NewSSHKeyHandler(db, cfg)
r.Route("/api", func(r chi.Router) {
r.Route("/auth", func(r chi.Router) {
@@ -64,16 +65,26 @@ func NewServer(cfg *config.Config, db *sql.DB, engine *syncengine.Engine) *Serve
r.Get("/", jobHandler.List)
r.Get("/{id}", jobHandler.Get)
r.Post("/{id}/cancel", jobHandler.Cancel)
r.Get("/{id}/log", jobHandler.StreamLog)
r.Get("/{id}/log", jobHandler.GetLog)
r.Get("/{id}/log/download", jobHandler.DownloadLog)
r.Get("/{id}/log/stream", sseHandler.StreamJob)
})
r.With(auth.RequireAuth).Get("/jobs/stream", sseHandler.Stream)
r.With(auth.RequireAuth).Get("/jobs/stream", sseHandler.StreamAll)
r.With(auth.RequireAuth).Get("/settings/pubkey", func(w http.ResponseWriter, r *http.Request) {
_, _, pubKey, _ := sshmanager.EnsureServerKey(cfg.SSHDir())
w.Header().Set("Content-Type", "text/plain")
w.Write([]byte(pubKey))
})
r.With(auth.RequireAuth).Route("/ssh-keys", func(r chi.Router) {
r.Get("/", sshKeyHandler.List)
r.Post("/", sshKeyHandler.Create)
r.Get("/{id}", sshKeyHandler.Get)
r.Delete("/{id}", sshKeyHandler.Delete)
r.Get("/{id}/private", sshKeyHandler.DownloadPrivate)
})
})
r.Get("/health", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {