From 5d708e1d7bb0eaa02b89781cc092017bdd91b00c Mon Sep 17 00:00:00 2001 From: Daniel Arroyo Date: Thu, 9 Jul 2026 20:03:00 -0400 Subject: [PATCH] fix: RunRemote SSH-to-source with pre-installed dest key, eliminating base64 round-trip - Regenerate qnap.key and baby-nas.key to OpenSSH native format (387 bytes vs 119 PKCS8) - Pre-install qnap.key on Baby NAS at /var/lib/syncserver/ssh/keys/ - Pre-populate Baby NAS known_hosts with Qnap host keys - Simplify RunRemote: LXC SSH to Baby NAS, Baby NAS runs rsync with local qnap.key - Remove wrapper script approach (rsync rejects remote-to-remote) --- Makefile | 2 +- cmd/server/main.go | 2 +- internal/syncengine/engine.go | 29 ++-- internal/syncengine/rsync_runner.go | 215 +++++++++------------------- 4 files changed, 89 insertions(+), 159 deletions(-) diff --git a/Makefile b/Makefile index 1e89ac9..4f10189 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ BINARY=syncserver -VERSION?=1.0.27 +VERSION?=1.0.28 GO?=go LDFLAGS=-s -w -X main.version=$(VERSION) -X main.commit=$(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) BUILD_FLAGS=CGO_ENABLED=0 diff --git a/cmd/server/main.go b/cmd/server/main.go index 8047598..5d86ff3 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -20,7 +20,7 @@ import ( "github.com/syncserver/internal/syncengine" ) -var version = "1.0.27" +var version = "1.0.28" func main() { cfgPath := flag.String("config", "", "Path to config.yaml") diff --git a/internal/syncengine/engine.go b/internal/syncengine/engine.go index 4e5d3d4..a09c159 100644 --- a/internal/syncengine/engine.go +++ b/internal/syncengine/engine.go @@ -264,20 +264,23 @@ func (e *Engine) Run(ctx context.Context, jobID int64, pairID int64) error { destPrivKeyPath, err := e.resolveSSHKey(dstMachine) if err != nil { slog.Warn("failed to resolve destination SSH key, using server key", "error", err) - destPrivKeyPath = filepath.Join(e.cfg.SSHDir(), "id_ed25519") + destPrivKeyPath = "" } - destPrivKeyBytes, err := os.ReadFile(destPrivKeyPath) - if err != nil { - slog.Warn("failed to read destination SSH key, using empty", "error", err) - destPrivKeyBytes = []byte{} - } - result, err = runner.RunRemote(jobCtx, cfg, RemoteMachine{ - Host: srcMachine.Host, - Port: srcMachine.Port, - SSHUser: srcMachine.SSHUser, - PrivKey: privKey, - DestPrivKey: string(destPrivKeyBytes), - }, onLine) + result, err = runner.RunRemote(jobCtx, cfg, + &MachineKeys{ + Host: srcMachine.Host, + Port: srcMachine.Port, + SSHUser: srcMachine.SSHUser, + PrivKey: privKey, + }, + &MachineKeys{ + Host: dstMachine.Host, + Port: dstMachine.Port, + SSHUser: dstMachine.SSHUser, + PrivKey: destPrivKeyPath, + }, + destPrivKeyPath, + onLine) } else { result, err = runner.Run(jobCtx, cfg, onLine) } diff --git a/internal/syncengine/rsync_runner.go b/internal/syncengine/rsync_runner.go index e71fafe..bf43dad 100644 --- a/internal/syncengine/rsync_runner.go +++ b/internal/syncengine/rsync_runner.go @@ -2,10 +2,10 @@ package syncengine import ( "context" - "encoding/base64" "fmt" "io" "os/exec" + "path/filepath" "strings" ) @@ -49,6 +49,76 @@ func (r *RsyncRunner) Run(ctx context.Context, pair *SyncPairConfig, onLine func cmd.Args = append([]string{"rsync", "-e", sshCmd}, args[1:]...) } + return r.runCmd(ctx, cmd, onLine) +} + +func (r *RsyncRunner) buildArgs(pair *SyncPairConfig) []string { + var args []string + + flags := strings.Fields(pair.RsyncFlags) + args = append(args, flags...) + + for _, pattern := range pair.ExcludePatterns { + args = append(args, "--exclude="+pattern) + } + + if pair.Direction == "mirror" { + args = append(args, "--delete") + } + + if pair.Direction == "pull" { + args = append(args, pair.Dest, pair.Source) + } else { + args = append(args, pair.Source, pair.Dest) + } + + return args +} + +type MachineKeys struct { + Host string + Port int + SSHUser string + PrivKey string +} + +func (r *RsyncRunner) RunRemote(ctx context.Context, pair *SyncPairConfig, src *MachineKeys, dst *MachineKeys, destKey string, onLine func(stream string, line string)) (*RsyncResult, error) { + if src.Port == 0 { + src.Port = 22 + } + if src.PrivKey == "" { + src.PrivKey = filepath.Join(r.sshDir, "id_ed25519") + } + if destKey == "" { + destKey = filepath.Join(r.sshDir, "id_ed25519") + } + + destUserHost := fmt.Sprintf("%s@%s", dst.SSHUser, dst.Host) + args := r.buildArgs(pair) + + innerSSH := fmt.Sprintf("ssh -i %s -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=%s", + destKey, filepath.Join(r.sshDir, "known_hosts")) + rsyncFlags := strings.Join(args[:len(args)-2], " ") + sourcePath := args[len(args)-2] + destPath := args[len(args)-1] + + remoteCmd := fmt.Sprintf("rsync %s -e %q %s %s", + rsyncFlags, innerSSH, sourcePath, destUserHost+":"+destPath) + + sshArgs := []string{ + "-i", src.PrivKey, + "-o", "StrictHostKeyChecking=accept-new", + "-o", "UserKnownHostsFile=" + filepath.Join(r.sshDir, "known_hosts"), + "-p", fmt.Sprintf("%d", src.Port), + fmt.Sprintf("%s@%s", src.SSHUser, src.Host), + } + sshArgs = append(sshArgs, remoteCmd) + + cmd := exec.CommandContext(ctx, "ssh", sshArgs...) + return r.runCmd(ctx, cmd, onLine) +} + +func (r *RsyncRunner) runCmd(ctx context.Context, cmd *exec.Cmd, onLine func(stream string, line string)) (*RsyncResult, error) { stdout, err := cmd.StdoutPipe() if err != nil { return nil, fmt.Errorf("stdout pipe: %w", err) @@ -133,146 +203,3 @@ func (r *RsyncRunner) Run(ctx context.Context, pair *SyncPairConfig, onLine func return result, nil } - -func (r *RsyncRunner) buildArgs(pair *SyncPairConfig) []string { - var args []string - - flags := strings.Fields(pair.RsyncFlags) - args = append(args, flags...) - - for _, pattern := range pair.ExcludePatterns { - args = append(args, "--exclude="+pattern) - } - - if pair.Direction == "mirror" { - args = append(args, "--delete") - } - - if pair.Direction == "pull" { - args = append(args, pair.Dest, pair.Source) - } else { - args = append(args, pair.Source, pair.Dest) - } - - return args -} - -type RemoteMachine struct { - Host string - Port int - SSHUser string - PrivKey string - DestPrivKey string -} - -func (r *RsyncRunner) RunRemote(ctx context.Context, pair *SyncPairConfig, remote RemoteMachine, onLine func(stream string, line string)) (*RsyncResult, error) { - args := r.buildArgs(pair) - - var remoteCmd string - if remote.DestPrivKey != "" { - encodedKey := base64.StdEncoding.EncodeToString([]byte(remote.DestPrivKey)) - innerSSH := fmt.Sprintf(`ssh -i /tmp/syncserver-dest-key -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/dev/null`) - rsyncPart := fmt.Sprintf("rsync %s -e %q", - strings.Join(args, " "), innerSSH) - remoteCmd = fmt.Sprintf( - `echo '%s' | base64 -d > /tmp/syncserver-dest-key && chmod 600 /tmp/syncserver-dest-key && %s; STATUS=$?; rm -f /tmp/syncserver-dest-key; exit $STATUS`, - encodedKey, rsyncPart) - } else { - remoteCmd = "rsync " + strings.Join(args, " ") - } - - sshArgs := []string{ - "-i", remote.PrivKey, - "-o", "StrictHostKeyChecking=accept-new", - "-o", "UserKnownHostsFile=" + strings.TrimRight(r.sshDir, "/") + "/known_hosts", - "-p", fmt.Sprintf("%d", remote.Port), - fmt.Sprintf("%s@%s", remote.SSHUser, remote.Host), - } - sshArgs = append(sshArgs, remoteCmd) - - cmd := exec.CommandContext(ctx, "ssh", sshArgs...) - - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, fmt.Errorf("stdout pipe: %w", err) - } - stderr, err := cmd.StderrPipe() - if err != nil { - return nil, fmt.Errorf("stderr pipe: %w", err) - } - - if err := cmd.Start(); err != nil { - return nil, fmt.Errorf("starting ssh: %w", err) - } - - var outLines, errLines []string - done := make(chan struct{}) - go func() { - br := io.Reader(stdout) - buf := make([]byte, 4096) - for { - n, err := br.Read(buf) - if n > 0 { - line := strings.TrimRight(string(buf[:n]), "\r\n") - if line != "" { - outLines = append(outLines, line) - if onLine != nil { - onLine("stdout", line) - } - } - } - if err != nil { - break - } - } - select { - case <-done: - default: - close(done) - } - }() - - go func() { - br := io.Reader(stderr) - buf := make([]byte, 4096) - for { - n, err := br.Read(buf) - if n > 0 { - line := strings.TrimRight(string(buf[:n]), "\r\n") - if line != "" { - errLines = append(errLines, line) - if onLine != nil { - onLine("stderr", line) - } - } - } - if err != nil { - break - } - } - select { - case <-done: - default: - close(done) - } - }() - - err = cmd.Wait() - <-done - - result := &RsyncResult{ - ExitCode: 0, - Stdout: strings.Join(outLines, "\n"), - Stderr: strings.Join(errLines, "\n"), - Stats: ParseFinalStats(strings.Join(outLines, "\n")), - } - if err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - result.ExitCode = exitErr.ExitCode() - } else { - result.ExitCode = -1 - } - } - - return result, nil -}