Add deploy script with dry-run, retry, rollback and health checks

2026-07-08 00:21:17 -04:00
parent b185686d2b
commit e322299dc3
3 changed files with 284 additions and 1 deletions
@@ -19,6 +19,9 @@ make build-all

 # Tests
 go test ./...
+
+# Deploy (full build + bump + push + LXC deploy + verify)
+./scripts/deploy.sh
 ```

 ## Version Bumping
@@ -46,6 +49,42 @@ ssh root@10.5.1.30 '/usr/bin/syncserver --version'
 ssh root@10.5.1.30 'journalctl -u syncserver -n 3 --no-pager'
 ```

+
+## Deploy Flow
+
+The `make deploy` target (or `./scripts/deploy.sh`) automates the full deployment pipeline:
+
+1. **Pre-flight** — checks git clean, on main branch, LXC reachable, disk space
+2. **Version bump** — runs `bump-version.sh`, commits and pushes automatically
+3. **Build** — `make clean && make package` (Linux amd64 .deb)
+4. **Backup** — copies current `/usr/bin/syncserver` to `/var/backups/syncserver.previous.<version>`
+5. **Deploy** — scp .deb to LXC, run `dpkg -i`
+6. **Verify** — service active, version matches, Cache-Control headers correct, HTTP 200
+
+### Deploy Options
+
+```bash
+./scripts/deploy.sh                    # Full deploy
+./scripts/deploy.sh --dry-run          # Simulate without making changes
+SKIP_BUILD=1 ./scripts/deploy.sh       # Use existing .deb
+SKIP_VERIFY=1 ./scripts/deploy.sh     # Skip post-deploy checks
+LXC_HOST=root@10.5.1.30 ./scripts/deploy.sh  # Custom target
+```
+
+### Rollback
+
+If verification fails after deploy, the script automatically rolls back to the backup binary. Manual rollback:
+
+```bash
+ssh root@10.5.1.30 "cp /var/backups/syncserver.previous.<version> /usr/bin/syncserver && systemctl restart syncserver"
+```
+
+### LXC Host
+
+- **Host**: `root@10.5.1.30`
+- **Remote dir**: `~/move-data-nas`
+- **Service**: `syncserver` (systemd)
+
 ## Critical Build Order

 1. Frontend must be built FIRST: `cd web && npm run build`
@@ -4,7 +4,7 @@ GO?=go
 LDFLAGS=-s -w -X main.version=$(VERSION) -X main.commit=$(shell git rev-parse --short HEAD 2>/dev/null || echo unknown)
 BUILD_FLAGS=CGO_ENABLED=0

-.PHONY: build build-all build-go frontend run package clean tidy test
+.PHONY: build build-all build-go frontend run package clean tidy test deploy

 frontend:
 	cd web && npm install && npm run build
@@ -47,3 +47,6 @@ tidy:

 test:
 	$(GO) test ./...
+
+deploy:
+	./scripts/deploy.sh
@@ -0,0 +1,241 @@
+#!/bin/bash
+# scripts/deploy.sh - Automated deploy for SyncServer
+#
+# Usage:
+#   ./scripts/deploy.sh                    # Full deploy
+#   ./scripts/deploy.sh --dry-run          # Simulate without making changes
+#   ./scripts/deploy.sh --help             # Show help
+#   SKIP_BUILD=1 ./scripts/deploy.sh       # Skip rebuild (use existing .deb)
+#   LXC_HOST=user@host ./scripts/deploy.sh # Custom target
+#
+# Environment variables:
+#   LXC_HOST       - SSH target (default: root@10.5.1.30)
+#   LXC_PORT       - SSH port (default: 22)
+#   REMOTE_DIR     - Remote directory (default: ~/move-data-nas)
+#   SKIP_BUILD     - Skip make package if 1
+#   SKIP_VERIFY    - Skip post-deploy verification if 1
+#   LOG_DIR        - Where to write logs (default: logs/)
+
+set -euo pipefail
+
+# === Configuration ===
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+LXC_HOST="${LXC_HOST:-root@10.5.1.30}"
+LXC_PORT="${LXC_PORT:-22}"
+REMOTE_DIR="${REMOTE_DIR:-~/move-data-nas}"
+LOG_DIR="${LOG_DIR:-$PROJECT_DIR/logs}"
+SKIP_BUILD="${SKIP_BUILD:-0}"
+SKIP_VERIFY="${SKIP_VERIFY:-0}"
+DRY_RUN=0
+HEALTH_TIMEOUT=30
+SSH_OPTS="-o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new -p $LXC_PORT"
+
+# === Setup ===
+mkdir -p "$LOG_DIR"
+LOG_FILE="$LOG_DIR/deploy_$(date +%Y%m%d_%H%M%S).log"
+exec > >(tee -a "$LOG_FILE") 2>&1
+
+# === Helpers ===
+log()     { echo "→ $*"; }
+success() { echo "✓ $*"; }
+warn()    { echo "⚠ $*" >&2; }
+die()     { echo "✗ $*" >&2; exit 1; }
+
+run_ssh() { ssh $SSH_OPTS "$LXC_HOST" "$@"; }
+run_scp() { scp -P "$LXC_PORT" "$@"; }
+
+BACKUP_PATH=""
+rollback() {
+    if [ -z "$BACKUP_PATH" ]; then
+        warn "No backup path set, cannot rollback"
+        return 1
+    fi
+    warn "Rolling back to $BACKUP_PATH..."
+    if run_ssh "systemctl stop syncserver 2>/dev/null; cp $BACKUP_PATH /usr/bin/syncserver; systemctl start syncserver 2>/dev/null || true"; then
+        warn "Rollback completed"
+    else
+        warn "Rollback failed - manual intervention required"
+    fi
+}
+
+trap 'die "Interrupted by signal"' INT TERM
+
+# === Argument parsing ===
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --dry-run) DRY_RUN=1; shift ;;
+        --help|-h)
+            sed -n '2,16p' "$0"
+            exit 0
+            ;;
+        *) die "Unknown argument: $1. Use --help for usage." ;;
+    esac
+done
+
+# === Step 1: Pre-flight ===
+log "Pre-flight checks..."
+cd "$PROJECT_DIR"
+git diff --quiet || die "Git working tree is dirty. Commit or stash first."
+git diff --cached --quiet || die "Git index has staged changes. Commit first."
+[ "$(git rev-parse --abbrev-ref HEAD)" = "main" ] || die "Must be on main branch"
+[ -f "$SCRIPT_DIR/bump-version.sh" ] || die "bump-version.sh not found"
+[ -f "Makefile" ] || die "Makefile not found"
+[ -d "web" ] || die "web/ directory not found"
+
+for tool in ssh scp make go dpkg-deb; do
+    command -v "$tool" >/dev/null || die "$tool not found"
+done
+
+if ! run_ssh 'true' 2>/dev/null; then
+    [ "$DRY_RUN" = "1" ] && warn "Cannot reach $LXC_HOST, but --dry-run so continuing" || die "Cannot ssh to $LXC_HOST"
+fi
+
+FREE_KB=$(df -k "$PROJECT_DIR" | tail -1 | awk '{print $4}')
+if [ "$FREE_KB" -lt 512000 ]; then
+    warn "Less than 500MB free on local disk"
+fi
+
+success "Pre-flight OK"
+
+# === Step 2: Bump version ===
+log "Bumping version..."
+CURRENT_VERSION=$(grep -oE 'var version = "[0-9]+\.[0-9]+\.[0-9]+"' cmd/server/main.go | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')
+[ -n "$CURRENT_VERSION" ] || die "Cannot parse current version"
+
+if [ "$DRY_RUN" = "1" ]; then
+    success "Would bump from $CURRENT_VERSION (dry run)"
+    NEW_VERSION="$CURRENT_VERSION"
+else
+    "$SCRIPT_DIR/bump-version.sh" >/dev/null
+    NEW_VERSION=$(grep -oE 'var version = "[0-9]+\.[0-9]+\.[0-9]+"' cmd/server/main.go | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')
+    [ "$NEW_VERSION" != "$CURRENT_VERSION" ] || die "Version bump failed"
+
+    git add cmd/server/main.go Makefile
+    git commit -m "Bump version to $NEW_VERSION" >/dev/null
+    git push || die "git push failed"
+    success "Bumped to $NEW_VERSION and pushed"
+fi
+
+VERSION="$NEW_VERSION"
+
+# === Step 3: Build ===
+if [ "$SKIP_BUILD" = "1" ]; then
+    log "Skipping build (SKIP_BUILD=1)"
+    DEB="dist/syncserver_${VERSION}_amd64.deb"
+    [ -f "$DEB" ] || die "SKIP_BUILD=1 but $DEB does not exist"
+else
+    log "Building..."
+    if [ "$DRY_RUN" = "1" ]; then
+        log "[dry-run] Would run: make clean && make package"
+        DEB="dist/syncserver_${VERSION}_amd64.deb"
+        if [ -f "$DEB" ]; then
+            log "[dry-run] Found existing $DEB"
+        else
+            log "[dry-run] $DEB does not exist yet (would be created by build)"
+        fi
+    else
+        make clean >/dev/null
+        make package
+        DEB="dist/syncserver_${VERSION}_amd64.deb"
+        [ -f "$DEB" ] || die "Build failed: $DEB not found"
+        success "Built: $DEB ($(du -h "$DEB" | cut -f1))"
+    fi
+fi
+
+# === Step 4: Backup on LXC ===
+BACKUP_PATH="/var/backups/syncserver.previous.${VERSION}"
+log "Backing up current binary on LXC to $BACKUP_PATH..."
+
+if [ "$DRY_RUN" = "1" ]; then
+    log "[dry-run] Would backup /usr/bin/syncserver"
+else
+    if ! run_ssh "mkdir -p /var/backups && cp /usr/bin/syncserver $BACKUP_PATH && ls -la $BACKUP_PATH" 2>/dev/null; then
+        die "Backup failed on $LXC_HOST"
+    fi
+    success "Backed up to $BACKUP_PATH"
+fi
+
+# === Step 5: Deploy ===
+log "Deploying to $LXC_HOST..."
+
+if [ "$DRY_RUN" = "1" ]; then
+    log "[dry-run] Would scp $DEB and run dpkg -i"
+    echo ""
+    echo "Dry run complete. No changes made."
+    echo "Log file: $LOG_FILE"
+    exit 0
+fi
+
+if ! run_scp "$DEB" "${LXC_HOST}:${REMOTE_DIR}/dist/" 2>/dev/null; then
+    rollback
+    die "SCP failed"
+fi
+
+if ! run_ssh "cd $REMOTE_DIR && dpkg -i dist/syncserver_${VERSION}_amd64.deb" 2>/dev/null; then
+    rollback
+    die "dpkg -i failed, rolled back"
+fi
+
+success "Deployed $DEB"
+
+# === Step 6: Verify ===
+if [ "$SKIP_VERIFY" = "1" ]; then
+    log "Skipping verify (SKIP_VERIFY=1)"
+else
+    log "Verifying deployment (timeout: ${HEALTH_TIMEOUT}s)..."
+
+    SERVICE_ACTIVE=""
+    for i in $(seq 1 $HEALTH_TIMEOUT); do
+        SERVICE_ACTIVE=$(run_ssh 'systemctl is-active syncserver' 2>/dev/null || echo "unknown")
+        if [ "$SERVICE_ACTIVE" = "active" ]; then
+            break
+        fi
+        sleep 1
+    done
+
+    if [ "$SERVICE_ACTIVE" != "active" ]; then
+        rollback
+        die "Service did not become active within ${HEALTH_TIMEOUT}s (current: $SERVICE_ACTIVE)"
+    fi
+    success "Service active"
+
+    REMOTE_VERSION=$(run_ssh '/usr/bin/syncserver --version' 2>/dev/null | tr -d '[:space:]' || echo "unknown")
+    if [ "$REMOTE_VERSION" != "$VERSION" ]; then
+        rollback
+        die "Version mismatch: deployed $VERSION, remote reports $REMOTE_VERSION"
+    fi
+    success "Version: $REMOTE_VERSION"
+
+    CACHE_HEADER=$(run_ssh 'curl -sI http://localhost:8080/' 2>/dev/null | grep -i "cache-control:" | head -1 | tr -d '\r')
+    if ! echo "$CACHE_HEADER" | grep -qi "no-cache"; then
+        warn "Cache-Control header missing or wrong: $CACHE_HEADER"
+    else
+        success "Cache headers OK"
+    fi
+
+    HTTP_CODE=$(run_ssh 'curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/' 2>/dev/null || echo "000")
+    if [ "$HTTP_CODE" != "200" ]; then
+        rollback
+        die "Health check failed: HTTP $HTTP_CODE"
+    fi
+    success "HTTP /  → 200"
+fi
+
+# === Summary ===
+echo ""
+echo "════════════════════════════════════════════"
+echo "  Deploy Summary"
+echo "════════════════════════════════════════════"
+echo "  Version:   $VERSION"
+echo "  Host:      $LXC_HOST"
+echo "  Binary:    /usr/bin/syncserver"
+echo "  Backup:    $BACKUP_PATH"
+echo "  Status:    active"
+echo "  HTTP:      200"
+echo "  Headers:   $CACHE_HEADER"
+echo "  Time:      $(date)"
+echo "  Log file:  $LOG_FILE"
+echo "════════════════════════════════════════════"
+echo ""
+echo "Next: hard reload browser once (Ctrl+Shift+R) to clear stale cache"