Add job error persistence and friendly error UI

Backend: - Migration 0003_job_error: adds error_message and error_code columns to jobs table - models/job.go: add ErrorMessage, ErrorCode fields to Job struct; SetError method; update all SELECT queries - models/job_log.go: GetAllFiltered also reads error_message and error_code (via Job embed) - syncengine/engine.go: setJobError() helper; capture errors at Wol timeout (wol_timeout), rsync error (rsync_error), and exit_code failure points - api/dto.go: add ErrorMessage and ErrorCode to JobResponse - api/handlers_jobs.go: jobToResp propagates error fields Frontend: - api/client.ts: add error_message? and error_code? to Job interface - lib/status.ts: add ERROR_CODES map with friendly titles/hints; getErrorCodeInfo() - components/ErrorDetailsModal.tsx: new modal showing error title, hint, full message, job metadata, and stderr log; copy-all and download-log buttons - pages/JobDetail.tsx: error banner for failed jobs with title/hint; View error button opens ErrorDetailsModal; SSE updates error_message in real-time
2026-07-08 09:04:47 -04:00
parent e0e94bd518
commit 9d32ef7fd6
10 changed files with 356 additions and 34 deletions
@@ -62,6 +62,8 @@ type JobResponse struct {
 	StartedAt        *string `json:"started_at"`
 	FinishedAt       *string `json:"finished_at"`
 	LogFile          *string `json:"log_file"`
+	ErrorMessage     *string `json:"error_message,omitempty"`
+	ErrorCode        *string `json:"error_code,omitempty"`
 	DurationSeconds  *int64  `json:"duration_seconds,omitempty"`
 	LogLineCount     *int64  `json:"log_line_count,omitempty"`
 }
@@ -206,11 +206,13 @@ func (h *JobHandler) DownloadLog(w http.ResponseWriter, r *http.Request) {

 func jobToResp(j models.Job) JobResponse {
 	resp := JobResponse{
-		ID:          j.ID,
-		SyncPairID:  j.SyncPairID,
-		TriggerType: j.TriggerType,
-		Status:      j.Status,
-		LogFile:     j.LogFile,
+		ID:           j.ID,
+		SyncPairID:   j.SyncPairID,
+		TriggerType:  j.TriggerType,
+		Status:       j.Status,
+		LogFile:      j.LogFile,
+		ErrorMessage: j.ErrorMessage,
+		ErrorCode:    j.ErrorCode,
 	}
 	if j.StartedAt != nil {
 		s := j.StartedAt.Format(time.RFC3339)
@@ -0,0 +1,4 @@
+-- 0003_job_error.sql
+
+ALTER TABLE jobs ADD COLUMN error_message TEXT;
+ALTER TABLE jobs ADD COLUMN error_code TEXT;
@@ -6,14 +6,16 @@ import (
 )

 type Job struct {
-	ID          int64      `db:"id" json:"id"`
-	SyncPairID  int64      `db:"sync_pair_id" json:"sync_pair_id"`
-	TriggerType string     `db:"trigger_type" json:"trigger_type"`
-	Status      string     `db:"status" json:"status"`
-	StartedAt   *time.Time `db:"started_at" json:"started_at"`
-	FinishedAt  *time.Time `db:"finished_at" json:"finished_at"`
-	LogFile     *string    `db:"log_file" json:"log_file"`
-	CreatedAt   time.Time  `db:"created_at" json:"created_at"`
+	ID           int64      `db:"id" json:"id"`
+	SyncPairID   int64      `db:"sync_pair_id" json:"sync_pair_id"`
+	TriggerType  string     `db:"trigger_type" json:"trigger_type"`
+	Status       string     `db:"status" json:"status"`
+	StartedAt    *time.Time `db:"started_at" json:"started_at"`
+	FinishedAt   *time.Time `db:"finished_at" json:"finished_at"`
+	LogFile      *string    `db:"log_file" json:"log_file"`
+	ErrorMessage *string    `db:"error_message" json:"error_message,omitempty"`
+	ErrorCode    *string    `db:"error_code" json:"error_code,omitempty"`
+	CreatedAt    time.Time  `db:"created_at" json:"created_at"`
 }

 type JobRepository struct {
@@ -38,12 +40,12 @@ func (r *JobRepository) Create(syncPairID int64, triggerType, status string) (in
 func (r *JobRepository) GetByID(id int64) (*Job, error) {
 	var j Job
 	var started, finished sql.NullTime
-	var logFile sql.NullString
+	var logFile, errMsg, errCode sql.NullString
 	err := r.db.QueryRow(`
 		SELECT id, sync_pair_id, trigger_type, status, started_at, finished_at,
-			log_file, created_at FROM jobs WHERE id = ?`, id).Scan(
+			log_file, error_message, error_code, created_at FROM jobs WHERE id = ?`, id).Scan(
 		&j.ID, &j.SyncPairID, &j.TriggerType, &j.Status, &started, &finished,
-		&logFile, &j.CreatedAt)
+		&logFile, &errMsg, &errCode, &j.CreatedAt)
 	if err != nil {
 		return nil, err
 	}
@@ -56,13 +58,20 @@ func (r *JobRepository) GetByID(id int64) (*Job, error) {
 	if logFile.Valid {
 		j.LogFile = &logFile.String
 	}
+	if errMsg.Valid {
+		j.ErrorMessage = &errMsg.String
+	}
+	if errCode.Valid {
+		j.ErrorCode = &errCode.String
+	}
 	return &j, nil
 }

 func (r *JobRepository) GetAll(limit, offset int) ([]Job, error) {
 	rows, err := r.db.Query(`
 		SELECT id, sync_pair_id, trigger_type, status, started_at, finished_at,
-			log_file, created_at FROM jobs ORDER BY created_at DESC LIMIT ? OFFSET ?`,
+			log_file, error_message, error_code, created_at
+		FROM jobs ORDER BY created_at DESC LIMIT ? OFFSET ?`,
 		limit, offset)
 	if err != nil {
 		return nil, err
@@ -73,9 +82,9 @@ func (r *JobRepository) GetAll(limit, offset int) ([]Job, error) {
 	for rows.Next() {
 		var j Job
 		var started, finished sql.NullTime
-		var logFile sql.NullString
+		var logFile, errMsg, errCode sql.NullString
 		if err := rows.Scan(&j.ID, &j.SyncPairID, &j.TriggerType, &j.Status,
-			&started, &finished, &logFile, &j.CreatedAt); err != nil {
+			&started, &finished, &logFile, &errMsg, &errCode, &j.CreatedAt); err != nil {
 			return nil, err
 		}
 		if started.Valid {
@@ -87,6 +96,12 @@ func (r *JobRepository) GetAll(limit, offset int) ([]Job, error) {
 		if logFile.Valid {
 			j.LogFile = &logFile.String
 		}
+		if errMsg.Valid {
+			j.ErrorMessage = &errMsg.String
+		}
+		if errCode.Valid {
+			j.ErrorCode = &errCode.String
+		}
 		jobs = append(jobs, j)
 	}
 	return jobs, rows.Err()
@@ -115,17 +130,25 @@ func (r *JobRepository) SetLogFile(id int64, path string) error {
 	return err
 }

+func (r *JobRepository) SetError(id int64, code, message string) error {
+	_, err := r.db.Exec(
+		"UPDATE jobs SET error_code = ?, error_message = ? WHERE id = ?",
+		code, message, id,
+	)
+	return err
+}
+
 func (r *JobRepository) GetRunningBySyncPair(syncPairID int64) (*Job, error) {
 	var j Job
 	var started sql.NullTime
-	var logFile sql.NullString
+	var logFile, errMsg, errCode sql.NullString
 	err := r.db.QueryRow(`
 		SELECT id, sync_pair_id, trigger_type, status, started_at, finished_at,
-			log_file, created_at FROM jobs
+			log_file, error_message, error_code, created_at FROM jobs
 		WHERE sync_pair_id = ? AND status IN ('queued','waking_up','running')
 		ORDER BY created_at DESC LIMIT 1`, syncPairID).Scan(
 		&j.ID, &j.SyncPairID, &j.TriggerType, &j.Status, &started,
-		&j.FinishedAt, &logFile, &j.CreatedAt)
+		&j.FinishedAt, &logFile, &errMsg, &errCode, &j.CreatedAt)
 	if err != nil {
 		return nil, err
 	}
@@ -135,6 +158,12 @@ func (r *JobRepository) GetRunningBySyncPair(syncPairID int64) (*Job, error) {
 	if logFile.Valid {
 		j.LogFile = &logFile.String
 	}
+	if errMsg.Valid {
+		j.ErrorMessage = &errMsg.String
+	}
+	if errCode.Valid {
+		j.ErrorCode = &errCode.String
+	}
 	return &j, nil
 }

@@ -124,7 +124,8 @@ func (r *JobLogRepository) GetAllFiltered(limit, offset int, syncPairID *int64,
 	query := `
 		SELECT
 			j.id, j.sync_pair_id, j.trigger_type, j.status,
-			j.started_at, j.finished_at, j.log_file, j.created_at,
+			j.started_at, j.finished_at, j.log_file,
+			j.error_message, j.error_code, j.created_at,
 			CASE WHEN j.finished_at IS NOT NULL AND j.started_at IS NOT NULL
 				THEN (j.finished_at - j.started_at) ELSE NULL END as duration_seconds,
 			(SELECT COUNT(*) FROM job_logs WHERE job_id = j.id) as log_line_count
@@ -143,10 +144,11 @@ func (r *JobLogRepository) GetAllFiltered(limit, offset int, syncPairID *int64,
 	for rows.Next() {
 		var j JobWithStats
 		var started, finished sql.NullTime
-		var logFile sql.NullString
+		var logFile, errMsg, errCode sql.NullString
 		var durationSeconds sql.NullInt64
 		if err := rows.Scan(&j.ID, &j.SyncPairID, &j.TriggerType, &j.Status,
-			&started, &finished, &logFile, &j.CreatedAt,
+			&started, &finished, &logFile,
+			&errMsg, &errCode, &j.CreatedAt,
 			&durationSeconds, &j.LogLineCount); err != nil {
 			return nil, 0, err
 		}
@@ -159,6 +161,12 @@ func (r *JobLogRepository) GetAllFiltered(limit, offset int, syncPairID *int64,
 		if logFile.Valid {
 			j.LogFile = &logFile.String
 		}
+		if errMsg.Valid {
+			j.ErrorMessage = &errMsg.String
+		}
+		if errCode.Valid {
+			j.ErrorCode = &errCode.String
+		}
 		if durationSeconds.Valid {
 			j.DurationSeconds = &durationSeconds.Int64
 		}
@@ -138,11 +138,12 @@ func (e *Engine) Run(ctx context.Context, jobID int64, pairID int64) error {

 			timeout := time.Duration(targetMachine.WakeTimeoutSeconds) * time.Second
 			interval := time.Duration(targetMachine.WakeCheckIntervalSeconds) * time.Second
-			if err := wol.WaitUntilReady(jobCtx, targetMachine.Host, remotePort, timeout, interval, false); err != nil {
-		e.setJobStatus(jobID, "failed")
-		e.emit(Event{Type: "status", JobID: jobID, Key: "status", Value: "failed", Line: err.Error()})
-		return fmt.Errorf("machine not ready: %w", err)
-			}
+		if err := wol.WaitUntilReady(jobCtx, targetMachine.Host, remotePort, timeout, interval, false); err != nil {
+			e.setJobStatus(jobID, "failed")
+			e.setJobError(jobID, "wol_timeout", err.Error())
+			e.emit(Event{Type: "status", JobID: jobID, Key: "status", Value: "failed", Line: err.Error()})
+			return fmt.Errorf("machine not ready: %w", err)
+		}
 		}
 	}

@@ -198,17 +199,19 @@ func (e *Engine) Run(ctx context.Context, jobID int64, pairID int64) error {

 	if err != nil {
 		if jobCtx.Err() != nil {
-		e.setJobStatus(jobID, "cancelled")
+			e.setJobStatus(jobID, "cancelled")
 			e.emit(Event{Type: "status", JobID: jobID, Key: "status", Value: "cancelled"})
 			return jobCtx.Err()
 		}
 		e.setJobStatus(jobID, "failed")
+		e.setJobError(jobID, "rsync_error", err.Error())
 		e.emit(Event{Type: "status", JobID: jobID, Key: "status", Value: "failed", Line: err.Error()})
 		return fmt.Errorf("rsync error: %w", err)
 	}

 	if result.ExitCode != 0 {
-	e.setJobStatus(jobID, "failed")
+		e.setJobStatus(jobID, "failed")
+		e.setJobError(jobID, "exit_code", result.Stderr)
 		e.emit(Event{Type: "status", JobID: jobID, Key: "status", Value: "failed", Line: result.Stderr})
 		return fmt.Errorf("rsync exited with code %d: %s", result.ExitCode, result.Stderr)
 	}
@@ -254,6 +257,11 @@ func (e *Engine) setJobLogFile(jobID int64, path string) {
 	jobRepo.SetLogFile(jobID, path)
 }

+func (e *Engine) setJobError(jobID int64, code, message string) {
+	jobRepo := models.NewJobRepository(e.db)
+	jobRepo.SetError(jobID, code, message)
+}
+
 func (e *Engine) emit(evt Event) {
 	e.eventBus.Publish(evt)
 }
@@ -75,6 +75,8 @@ export interface Job {
  started_at: string | null;
  finished_at: string | null;
  log_file: string | null;
+  error_message?: string | null;
+  error_code?: string | null;
  duration_seconds?: number | null;
  log_line_count?: number | null;
 }
@@ -0,0 +1,175 @@
+import * as React from 'react'
+import { AlertCircle, Download, Clock, Zap, Copy } from 'lucide-react'
+import {
+  Modal,
+  ModalContent,
+  ModalHeader,
+  ModalTitle,
+  ModalDescription,
+  ModalBody,
+  ModalFooter,
+} from '@/components/ui/Modal'
+import { Button } from '@/components/ui/Button'
+import { Badge } from '@/components/ui/Badge'
+import { statusVariant } from '@/lib/status'
+import { getErrorCodeInfo } from '@/lib/status'
+import type { Job } from '@/api/client'
+
+interface ErrorDetailsModalProps {
+  open: boolean
+  onOpenChange: (open: boolean) => void
+  job: Job
+  fullLog?: string
+}
+
+export function ErrorDetailsModal({
+  open,
+  onOpenChange,
+  job,
+  fullLog,
+}: ErrorDetailsModalProps) {
+  const [copied, setCopied] = React.useState(false)
+  const errorInfo = getErrorCodeInfo(job.error_code ?? null)
+
+  const handleCopy = async () => {
+    const text = [
+      `Job #${job.id}`,
+      `Sync Pair ID: ${job.sync_pair_id}`,
+      `Trigger: ${job.trigger_type}`,
+      `Status: ${job.status}`,
+      `Error Code: ${job.error_code ?? 'unknown'}`,
+      `Error: ${job.error_message ?? 'none'}`,
+      '',
+      '--- Full Log ---',
+      fullLog ?? '(no log available)',
+    ].join('\n')
+    await navigator.clipboard.writeText(text)
+    setCopied(true)
+    setTimeout(() => setCopied(false), 2000)
+  }
+
+  return (
+    <Modal open={open} onOpenChange={onOpenChange}>
+      <ModalContent size="lg">
+        <ModalHeader>
+          <div className="flex items-center gap-2">
+            <AlertCircle className="h-5 w-5 text-rose-400" />
+            <ModalTitle>Job Failed</ModalTitle>
+          </div>
+          <ModalDescription>
+            Job #{job.id} finished with status{' '}
+            <Badge variant={statusVariant(job.status)} label={job.status} />
+          </ModalDescription>
+        </ModalHeader>
+        <ModalBody className="space-y-4">
+          {errorInfo && (
+            <div className="rounded-card border border-rose-500/30 bg-rose-500/10 p-4 space-y-1">
+              <div className="flex items-center gap-2">
+                <AlertCircle className="h-4 w-4 text-rose-400 shrink-0" />
+                <span className="text-sm font-semibold text-rose-300">
+                  {errorInfo.title}
+                </span>
+              </div>
+              <p className="text-xs text-rose-400/70 pl-6">{errorInfo.hint}</p>
+            </div>
+          )}
+
+          {job.error_message && (
+            <div className="space-y-1.5">
+              <p className="text-xs font-medium text-fg-muted uppercase tracking-wider">
+                Error Message
+              </p>
+              <div className="rounded-card border border-border bg-canvas-raised p-3 font-mono text-xs text-rose-300/80 whitespace-pre-wrap break-all max-h-40 overflow-y-auto">
+                {job.error_message}
+              </div>
+            </div>
+          )}
+
+          <div className="grid grid-cols-2 gap-3">
+            <div className="space-y-1">
+              <p className="text-xs font-medium text-fg-muted uppercase tracking-wider">
+                Error Code
+              </p>
+              <p className="text-sm font-mono text-fg">
+                {job.error_code ?? '—'}
+              </p>
+            </div>
+            <div className="space-y-1">
+              <p className="text-xs font-medium text-fg-muted uppercase tracking-wider">
+                Trigger
+              </p>
+              <p className="text-sm text-fg capitalize">
+                {job.trigger_type}
+              </p>
+            </div>
+            {job.started_at && (
+              <div className="space-y-1">
+                <p className="text-xs font-medium text-fg-muted uppercase tracking-wider flex items-center gap-1">
+                  <Clock className="h-3 w-3" />
+                  Started
+                </p>
+                <p className="text-sm text-fg">
+                  {new Date(job.started_at).toLocaleString()}
+                </p>
+              </div>
+            )}
+            {job.finished_at && (
+              <div className="space-y-1">
+                <p className="text-xs font-medium text-fg-muted uppercase tracking-wider flex items-center gap-1">
+                  <Zap className="h-3 w-3" />
+                  Finished
+                </p>
+                <p className="text-sm text-fg">
+                  {new Date(job.finished_at).toLocaleString()}
+                </p>
+              </div>
+            )}
+          </div>
+
+          {fullLog && (
+            <div className="space-y-1.5">
+              <p className="text-xs font-medium text-fg-muted uppercase tracking-wider">
+                Stderr / Error Log
+              </p>
+              <div className="rounded-card border border-border bg-canvas-raised p-3 font-mono text-xs text-rose-400/70 whitespace-pre-wrap break-all max-h-48 overflow-y-auto">
+                {fullLog}
+              </div>
+            </div>
+          )}
+        </ModalBody>
+        <ModalFooter>
+          <Button variant="secondary" onClick={() => onOpenChange(false)}>
+            Close
+          </Button>
+          <Button
+            variant="secondary"
+            size="sm"
+            onClick={() => {
+              const a = document.createElement('a')
+              a.href = `/api/jobs/${job.id}/log/download`
+              a.download = `job-${job.id}.log`
+              document.body.appendChild(a)
+              a.click()
+              document.body.removeChild(a)
+            }}
+          >
+            <Download className="h-4 w-4" />
+            Full Log
+          </Button>
+          <Button onClick={handleCopy}>
+            {copied ? (
+              <>
+                <span className="text-emerald-400">Copied</span>
+              </>
+            ) : (
+              <>
+                <Copy className="h-4 w-4" />
+                Copy All
+              </>
+            )}
+          </Button>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  )
+}
@@ -54,3 +54,36 @@ export function statusLabel(status: string): string {
  }
  return labels[status.toLowerCase()] ?? status
 }
+
+export interface ErrorCodeInfo {
+  title: string;
+  hint: string;
+}
+
+const ERROR_CODES: Record<string, ErrorCodeInfo> = {
+  wol_timeout: {
+    title: "Machine didn't wake up",
+    hint: 'Check Wake-on-LAN settings, MAC address, and network connectivity',
+  },
+  rsync_error: {
+    title: 'rsync failed',
+    hint: 'See the full log for rsync error details',
+  },
+  exit_code: {
+    title: 'rsync exited with errors',
+    hint: 'Check stderr output for details',
+  },
+  ssh_key_fallback: {
+    title: 'SSH key not found',
+    hint: 'Server fell back to its own key. Verify the machine SSH key configuration',
+  },
+  cancelled: {
+    title: 'Cancelled by user',
+    hint: 'The job was manually cancelled',
+  },
+}
+
+export function getErrorCodeInfo(code: string | null | undefined): ErrorCodeInfo | null {
+  if (!code) return null
+  return ERROR_CODES[code] ?? { title: code, hint: 'See full error details below' }
+}
@@ -16,15 +16,17 @@ import {
  ModalDescription,
  ModalFooter,
 } from '@/components/ui/Modal';
+import { ErrorDetailsModal } from '@/components/ErrorDetailsModal';
 import {
  ArrowLeft,
  Download,
  XCircle,
  ScrollText,
  Terminal,
+  AlertCircle,
 } from 'lucide-react';
 import { toast } from 'sonner';
-import { statusVariant, statusLabel } from '@/lib/status';
+import { statusVariant, statusLabel, getErrorCodeInfo } from '@/lib/status';
 import { formatDuration } from '@/lib/utils';
 import { cn } from '@/lib/utils';

@@ -48,6 +50,7 @@ export default function JobDetail() {
  const jobId = Number(id);
  const [loading, setLoading] = useState(true);
  const [cancelModal, setCancelModal] = useState(false);
+  const [errorModal, setErrorModal] = useState(false);

  useEffect(() => {
    loadJob();
@@ -63,7 +66,12 @@ export default function JobDetail() {
          setLiveLines(prev => [...prev, { stream: evt.stream!, text: evt.line! }]);
        }
        if (evt.type === 'status') {
-          setJob(prev => prev ? { ...prev, status: evt.status! } : prev);
+          setJob(prev => {
+            if (!prev) return prev;
+            const updated = { ...prev, status: evt.status! };
+            if (evt.line) updated.error_message = evt.line;
+            return updated;
+          });
        }
      };
    }
@@ -162,6 +170,13 @@ export default function JobDetail() {

  const totalLines = logs.length + liveLines.length;

+  const fullErrorLog = [
+    ...logs.filter(l => l.stream === 'stderr').map(l => l.content),
+    ...liveLines.filter(l => l.stream === 'stderr').map(l => l.text),
+  ].join('\n');
+
+  const errorInfo = getErrorCodeInfo(job.error_code ?? null);
+
  return (
    <div className="space-y-4">
      <div className="flex items-center gap-3">
@@ -212,6 +227,43 @@ export default function JobDetail() {
        ))}
      </div>

+      {job.status === 'failed' && job.error_message && (
+        <div className="rounded-card border border-rose-500/30 bg-rose-500/10 p-4 space-y-2">
+          <div className="flex items-start gap-2">
+            <AlertCircle className="h-4 w-4 text-rose-400 shrink-0 mt-0.5" />
+            <div className="flex-1 min-w-0">
+              <div className="flex items-center gap-2 flex-wrap">
+                <span className="text-sm font-semibold text-rose-300">
+                  {errorInfo?.title ?? 'Job failed'}
+                </span>
+                {job.error_code && (
+                  <span className="text-xs font-mono text-rose-400/60">
+                    [{job.error_code}]
+                  </span>
+                )}
+              </div>
+              {errorInfo?.hint && (
+                <p className="text-xs text-rose-400/70 mt-0.5">
+                  {errorInfo.hint}
+                </p>
+              )}
+              <p className="text-xs font-mono text-rose-300/80 mt-1 truncate max-w-2xl">
+                {job.error_message}
+              </p>
+            </div>
+            <Button
+              variant="secondary"
+              size="sm"
+              onClick={() => setErrorModal(true)}
+              className="shrink-0 text-rose-300 hover:text-rose-200 border-rose-500/40 hover:border-rose-400/60"
+            >
+              <AlertCircle className="h-4 w-4" />
+              View error
+            </Button>
+          </div>
+        </div>
+      )}
+
      {['queued', 'waking_up', 'running'].includes(job.status) && (
        <div className="flex items-center gap-3">
          <Button
@@ -314,6 +366,13 @@ export default function JobDetail() {
          </ModalFooter>
        </ModalContent>
      </Modal>
+
+      <ErrorDetailsModal
+        open={errorModal}
+        onOpenChange={setErrorModal}
+        job={job}
+        fullLog={fullErrorLog}
+      />
    </div>
  );
 }