feat: add mergerfs mover and snapraid integration

New 'Almacenamiento' page with: - Auto-detection of rsync, mergerfs, snapraid binaries and mergerfs mount - Configurable pool settings (source/dest, macOS cleanup, rsync flags) - Mergerfs mover with dry-run preview and live SSE output streaming - SnapRAID diff/sync/scrub/check with live SSE output - Async job system (1 concurrent job) with SSE streaming - Job history table Backend: - internal/storage/ package with capabilities, mergerfs, snapraid, jobs - storage_config and storage_jobs DB tables (migration 0008) - GET/PUT /api/storage/config, GET /api/storage/capabilities - POST/GET /api/storage/jobs, GET /api/storage/jobs/{id}/stream - Storage operations disabled when NASCTL_EXEC_SYSTEM=false Closes #new-feature
2026-07-06 23:23:22 -04:00
parent 319030848f
commit 27a52d2986
21 changed files with 2062 additions and 1 deletions
@@ -91,6 +91,11 @@ func (d *DB) Migrate() error {
 				return fmt.Errorf("fix zero fsids: %w", err)
 			}
 		}
+		if name == "0008_storage.sql" {
+			if err := d.EnsureStorageConfig(); err != nil {
+				return fmt.Errorf("ensure storage config: %w", err)
+			}
+		}
 	}
 	return nil
 }
@@ -0,0 +1,31 @@
+CREATE TABLE IF NOT EXISTS storage_config (
+    id INTEGER PRIMARY KEY CHECK (id = 1),
+    mover_source          TEXT NOT NULL DEFAULT '',
+    mover_dest            TEXT NOT NULL DEFAULT '',
+    mover_clean_macos     INTEGER NOT NULL DEFAULT 1,
+    mover_remove_source   INTEGER NOT NULL DEFAULT 1,
+    mover_inplace         INTEGER NOT NULL DEFAULT 1,
+    mover_rsync_options   TEXT NOT NULL DEFAULT '',
+    snapraid_content      TEXT NOT NULL DEFAULT '',
+    snapraid_data_dirs    TEXT NOT NULL DEFAULT '',
+    snapraid_parity_dir   TEXT NOT NULL DEFAULT '',
+    snapraid_scrub_plan   INTEGER NOT NULL DEFAULT 8,
+    updated_at            TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+CREATE TABLE IF NOT EXISTS storage_jobs (
+    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    kind        TEXT NOT NULL,
+    status      TEXT NOT NULL DEFAULT 'queued',
+    args_json   TEXT NOT NULL DEFAULT '{}',
+    pid         INTEGER NOT NULL DEFAULT 0,
+    started_at  TEXT,
+    finished_at TEXT,
+    exit_code   INTEGER NOT NULL DEFAULT -1,
+    output      TEXT NOT NULL DEFAULT '',
+    error       TEXT NOT NULL DEFAULT '',
+    created_at  TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_storage_jobs_created ON storage_jobs(created_at DESC);
+CREATE INDEX IF NOT EXISTS idx_storage_jobs_kind_created ON storage_jobs(kind, created_at DESC);
@@ -87,3 +87,39 @@ type ApplyLogEntry struct {
 	Success   bool      `json:"success"`
 	CreatedAt time.Time `json:"created_at"`
 }
+
+type StorageConfig struct {
+	ID                int64  `json:"id"`
+	MoverSource       string `json:"mover_source"`
+	MoverDest         string `json:"mover_dest"`
+	MoverCleanMacOS   bool   `json:"mover_clean_macos"`
+	MoverRemoveSource bool   `json:"mover_remove_source"`
+	MoverInplace      bool   `json:"mover_inplace"`
+	MoverRsyncOptions string `json:"mover_rsync_options"`
+	SnapraidContent   string `json:"snapraid_content"`
+	SnapraidDataDirs  string `json:"snapraid_data_dirs"`
+	SnapraidParityDir string `json:"snapraid_parity_dir"`
+	SnapraidScrubPlan int    `json:"snapraid_scrub_plan"`
+	UpdatedAt         string `json:"updated_at"`
+}
+
+type StorageJob struct {
+	ID         int64   `json:"id"`
+	Kind       string  `json:"kind"`
+	Status     string  `json:"status"`
+	ArgsJSON   string  `json:"args_json"`
+	PID        int     `json:"pid"`
+	StartedAt  *string `json:"started_at,omitempty"`
+	FinishedAt *string `json:"finished_at,omitempty"`
+	ExitCode   int     `json:"exit_code"`
+	Output     string  `json:"output"`
+	Error      string  `json:"error"`
+	CreatedAt  string  `json:"created_at"`
+}
+
+type StorageCapabilities struct {
+	Rsync           bool `json:"rsync"`
+	MergerfsBin     bool `json:"mergerfs"`
+	MergerfsMounted  bool `json:"mergerfs_mounted"`
+	SnapraidBin     bool `json:"snapraid"`
+}
@@ -0,0 +1,226 @@
+package db
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func scanStorageConfig(row interface {
+	Scan(dest ...any) error
+}) (StorageConfig, error) {
+	var c StorageConfig
+	var cleanMacos, removeSource, inplace int
+	var createdAt string
+	if err := row.Scan(
+		&c.ID,
+		&c.MoverSource,
+		&c.MoverDest,
+		&cleanMacos,
+		&removeSource,
+		&inplace,
+		&c.MoverRsyncOptions,
+		&c.SnapraidContent,
+		&c.SnapraidDataDirs,
+		&c.SnapraidParityDir,
+		&c.SnapraidScrubPlan,
+		&createdAt,
+	); err != nil {
+		return StorageConfig{}, err
+	}
+	c.MoverCleanMacOS = cleanMacos != 0
+	c.MoverRemoveSource = removeSource != 0
+	c.MoverInplace = inplace != 0
+	c.UpdatedAt = createdAt
+	return c, nil
+}
+
+func (d *DB) GetStorageConfig() (StorageConfig, error) {
+	row := d.conn.QueryRow(`SELECT id, mover_source, mover_dest, mover_clean_macos, mover_remove_source, mover_inplace, mover_rsync_options, snapraid_content, snapraid_data_dirs, snapraid_parity_dir, snapraid_scrub_plan, updated_at FROM storage_config WHERE id = 1`)
+	c, err := scanStorageConfig(row)
+	if err == sql.ErrNoRows {
+		return StorageConfig{ID: 1, MoverCleanMacOS: true, MoverRemoveSource: true, MoverInplace: true, SnapraidScrubPlan: 8}, nil
+	}
+	if err != nil {
+		return StorageConfig{}, fmt.Errorf("get storage config: %w", err)
+	}
+	return c, nil
+}
+
+func (d *DB) UpdateStorageConfig(c StorageConfig) error {
+	_, err := d.conn.Exec(`
+		UPDATE storage_config
+		SET mover_source=?, mover_dest=?, mover_clean_macos=?, mover_remove_source=?, mover_inplace=?,
+		    mover_rsync_options=?, snapraid_content=?, snapraid_data_dirs=?, snapraid_parity_dir=?,
+		    snapraid_scrub_plan=?, updated_at=datetime('now')
+		WHERE id=1`,
+		c.MoverSource, c.MoverDest, toInt(c.MoverCleanMacOS), toInt(c.MoverRemoveSource), toInt(c.MoverInplace),
+		c.MoverRsyncOptions, c.SnapraidContent, c.SnapraidDataDirs, c.SnapraidParityDir,
+		c.SnapraidScrubPlan, c.ID,
+	)
+	if err != nil {
+		return fmt.Errorf("update storage config: %w", err)
+	}
+	return nil
+}
+
+func (d *DB) EnsureStorageConfig() error {
+	_, err := d.conn.Exec(`INSERT OR IGNORE INTO storage_config (id) VALUES (1)`)
+	if err != nil {
+		return fmt.Errorf("ensure storage config: %w", err)
+	}
+	return nil
+}
+
+func scanStorageJob(row interface {
+	Scan(dest ...any) error
+}) (StorageJob, error) {
+	var j StorageJob
+	var startedAt, finishedAt sql.NullString
+	if err := row.Scan(
+		&j.ID, &j.Kind, &j.Status, &j.ArgsJSON, &j.PID,
+		&startedAt, &finishedAt, &j.ExitCode, &j.Output, &j.Error, &j.CreatedAt,
+	); err != nil {
+		return StorageJob{}, err
+	}
+	if startedAt.Valid {
+		j.StartedAt = &startedAt.String
+	}
+	if finishedAt.Valid {
+		j.FinishedAt = &finishedAt.String
+	}
+	return j, nil
+}
+
+func (d *DB) CreateStorageJob(kind string, argsJSON string) (StorageJob, error) {
+	result, err := d.conn.Exec(
+		`INSERT INTO storage_jobs (kind, status, args_json) VALUES (?, 'queued', ?)`,
+		kind, argsJSON,
+	)
+	if err != nil {
+		return StorageJob{}, fmt.Errorf("create storage job: %w", err)
+	}
+	id, err := result.LastInsertId()
+	if err != nil {
+		return StorageJob{}, fmt.Errorf("last insert id: %w", err)
+	}
+	return d.GetStorageJob(id)
+}
+
+func (d *DB) GetStorageJob(id int64) (StorageJob, error) {
+	row := d.conn.QueryRow(`
+		SELECT id, kind, status, args_json, pid, started_at, finished_at, exit_code, output, error, created_at
+		FROM storage_jobs WHERE id=?`, id)
+	j, err := scanStorageJob(row)
+	if err == sql.ErrNoRows {
+		return StorageJob{}, fmt.Errorf("storage job not found")
+	}
+	if err != nil {
+		return StorageJob{}, fmt.Errorf("get storage job: %w", err)
+	}
+	return j, nil
+}
+
+func (d *DB) MarkJobRunning(id int64, pid int) error {
+	_, err := d.conn.Exec(
+		`UPDATE storage_jobs SET status='running', pid=?, started_at=datetime('now') WHERE id=?`,
+		pid, id,
+	)
+	if err != nil {
+		return fmt.Errorf("mark job running: %w", err)
+	}
+	return nil
+}
+
+func (d *DB) MarkJobFinished(id int64, status string, exitCode int, output, errMsg string) error {
+	_, err := d.conn.Exec(
+		`UPDATE storage_jobs SET status=?, exit_code=?, output=?, error=?, finished_at=datetime('now') WHERE id=?`,
+		status, exitCode, output, errMsg, id,
+	)
+	if err != nil {
+		return fmt.Errorf("mark job finished: %w", err)
+	}
+	return nil
+}
+
+func (d *DB) AppendJobOutput(id int64, line string) error {
+	_, err := d.conn.Exec(
+		`UPDATE storage_jobs SET output = output || ? || char(10) WHERE id=?`,
+		line, id,
+	)
+	if err != nil {
+		return fmt.Errorf("append job output: %w", err)
+	}
+	return nil
+}
+
+func (d *DB) TrimJobOutput(id int64, maxLines int) error {
+	_ = maxLines
+	return nil
+}
+
+func (d *DB) ListStorageJobs(limit int) ([]StorageJob, error) {
+	if limit <= 0 {
+		limit = 50
+	}
+	rows, err := d.conn.Query(`
+		SELECT id, kind, status, args_json, pid, started_at, finished_at, exit_code, output, error, created_at
+		FROM storage_jobs ORDER BY created_at DESC LIMIT ?`, limit)
+	if err != nil {
+		return nil, fmt.Errorf("list storage jobs: %w", err)
+	}
+	defer rows.Close()
+
+	var jobs []StorageJob
+	for rows.Next() {
+		j, err := scanStorageJob(rows)
+		if err != nil {
+			return nil, fmt.Errorf("scan storage job: %w", err)
+		}
+		jobs = append(jobs, j)
+	}
+	return jobs, rows.Err()
+}
+
+func (d *DB) GetRunningJob() (*StorageJob, error) {
+	row := d.conn.QueryRow(`
+		SELECT id, kind, status, args_json, pid, started_at, finished_at, exit_code, output, error, created_at
+		FROM storage_jobs WHERE status IN ('queued','running') ORDER BY created_at ASC LIMIT 1`)
+	j, err := scanStorageJob(row)
+	if err == sql.ErrNoRows {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("get running job: %w", err)
+	}
+	return &j, nil
+}
+
+func (d *DB) ResetOrphanedJobs() error {
+	_, err := d.conn.Exec(
+		`UPDATE storage_jobs SET status='interrupted', finished_at=datetime('now') WHERE status IN ('queued','running')`,
+	)
+	if err != nil {
+		return fmt.Errorf("reset orphaned jobs: %w", err)
+	}
+	return nil
+}
+
+func (d *DB) SetJobCancelled(id int64) error {
+	_, err := d.conn.Exec(
+		`UPDATE storage_jobs SET status='cancelled', finished_at=datetime('now') WHERE id=?`,
+		id,
+	)
+	if err != nil {
+		return fmt.Errorf("set job cancelled: %w", err)
+	}
+	return nil
+}
+
+func toInt(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+