fix: RunRemote SSH-to-source with pre-installed dest key, eliminating base64 round-trip
- Regenerate qnap.key and baby-nas.key to OpenSSH native format (387 bytes vs 119 PKCS8) - Pre-install qnap.key on Baby NAS at /var/lib/syncserver/ssh/keys/ - Pre-populate Baby NAS known_hosts with Qnap host keys - Simplify RunRemote: LXC SSH to Baby NAS, Baby NAS runs rsync with local qnap.key - Remove wrapper script approach (rsync rejects remote-to-remote)
This commit is contained in:
@@ -1,5 +1,5 @@
|
|||||||
BINARY=syncserver
|
BINARY=syncserver
|
||||||
VERSION?=1.0.27
|
VERSION?=1.0.28
|
||||||
GO?=go
|
GO?=go
|
||||||
LDFLAGS=-s -w -X main.version=$(VERSION) -X main.commit=$(shell git rev-parse --short HEAD 2>/dev/null || echo unknown)
|
LDFLAGS=-s -w -X main.version=$(VERSION) -X main.commit=$(shell git rev-parse --short HEAD 2>/dev/null || echo unknown)
|
||||||
BUILD_FLAGS=CGO_ENABLED=0
|
BUILD_FLAGS=CGO_ENABLED=0
|
||||||
|
|||||||
+1
-1
@@ -20,7 +20,7 @@ import (
|
|||||||
"github.com/syncserver/internal/syncengine"
|
"github.com/syncserver/internal/syncengine"
|
||||||
)
|
)
|
||||||
|
|
||||||
var version = "1.0.27"
|
var version = "1.0.28"
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
cfgPath := flag.String("config", "", "Path to config.yaml")
|
cfgPath := flag.String("config", "", "Path to config.yaml")
|
||||||
|
|||||||
@@ -264,20 +264,23 @@ func (e *Engine) Run(ctx context.Context, jobID int64, pairID int64) error {
|
|||||||
destPrivKeyPath, err := e.resolveSSHKey(dstMachine)
|
destPrivKeyPath, err := e.resolveSSHKey(dstMachine)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Warn("failed to resolve destination SSH key, using server key", "error", err)
|
slog.Warn("failed to resolve destination SSH key, using server key", "error", err)
|
||||||
destPrivKeyPath = filepath.Join(e.cfg.SSHDir(), "id_ed25519")
|
destPrivKeyPath = ""
|
||||||
}
|
}
|
||||||
destPrivKeyBytes, err := os.ReadFile(destPrivKeyPath)
|
result, err = runner.RunRemote(jobCtx, cfg,
|
||||||
if err != nil {
|
&MachineKeys{
|
||||||
slog.Warn("failed to read destination SSH key, using empty", "error", err)
|
Host: srcMachine.Host,
|
||||||
destPrivKeyBytes = []byte{}
|
Port: srcMachine.Port,
|
||||||
}
|
SSHUser: srcMachine.SSHUser,
|
||||||
result, err = runner.RunRemote(jobCtx, cfg, RemoteMachine{
|
PrivKey: privKey,
|
||||||
Host: srcMachine.Host,
|
},
|
||||||
Port: srcMachine.Port,
|
&MachineKeys{
|
||||||
SSHUser: srcMachine.SSHUser,
|
Host: dstMachine.Host,
|
||||||
PrivKey: privKey,
|
Port: dstMachine.Port,
|
||||||
DestPrivKey: string(destPrivKeyBytes),
|
SSHUser: dstMachine.SSHUser,
|
||||||
}, onLine)
|
PrivKey: destPrivKeyPath,
|
||||||
|
},
|
||||||
|
destPrivKeyPath,
|
||||||
|
onLine)
|
||||||
} else {
|
} else {
|
||||||
result, err = runner.Run(jobCtx, cfg, onLine)
|
result, err = runner.Run(jobCtx, cfg, onLine)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,10 +2,10 @@ package syncengine
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/base64"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -49,6 +49,76 @@ func (r *RsyncRunner) Run(ctx context.Context, pair *SyncPairConfig, onLine func
|
|||||||
cmd.Args = append([]string{"rsync", "-e", sshCmd}, args[1:]...)
|
cmd.Args = append([]string{"rsync", "-e", sshCmd}, args[1:]...)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return r.runCmd(ctx, cmd, onLine)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *RsyncRunner) buildArgs(pair *SyncPairConfig) []string {
|
||||||
|
var args []string
|
||||||
|
|
||||||
|
flags := strings.Fields(pair.RsyncFlags)
|
||||||
|
args = append(args, flags...)
|
||||||
|
|
||||||
|
for _, pattern := range pair.ExcludePatterns {
|
||||||
|
args = append(args, "--exclude="+pattern)
|
||||||
|
}
|
||||||
|
|
||||||
|
if pair.Direction == "mirror" {
|
||||||
|
args = append(args, "--delete")
|
||||||
|
}
|
||||||
|
|
||||||
|
if pair.Direction == "pull" {
|
||||||
|
args = append(args, pair.Dest, pair.Source)
|
||||||
|
} else {
|
||||||
|
args = append(args, pair.Source, pair.Dest)
|
||||||
|
}
|
||||||
|
|
||||||
|
return args
|
||||||
|
}
|
||||||
|
|
||||||
|
type MachineKeys struct {
|
||||||
|
Host string
|
||||||
|
Port int
|
||||||
|
SSHUser string
|
||||||
|
PrivKey string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *RsyncRunner) RunRemote(ctx context.Context, pair *SyncPairConfig, src *MachineKeys, dst *MachineKeys, destKey string, onLine func(stream string, line string)) (*RsyncResult, error) {
|
||||||
|
if src.Port == 0 {
|
||||||
|
src.Port = 22
|
||||||
|
}
|
||||||
|
if src.PrivKey == "" {
|
||||||
|
src.PrivKey = filepath.Join(r.sshDir, "id_ed25519")
|
||||||
|
}
|
||||||
|
if destKey == "" {
|
||||||
|
destKey = filepath.Join(r.sshDir, "id_ed25519")
|
||||||
|
}
|
||||||
|
|
||||||
|
destUserHost := fmt.Sprintf("%s@%s", dst.SSHUser, dst.Host)
|
||||||
|
args := r.buildArgs(pair)
|
||||||
|
|
||||||
|
innerSSH := fmt.Sprintf("ssh -i %s -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=%s",
|
||||||
|
destKey, filepath.Join(r.sshDir, "known_hosts"))
|
||||||
|
rsyncFlags := strings.Join(args[:len(args)-2], " ")
|
||||||
|
sourcePath := args[len(args)-2]
|
||||||
|
destPath := args[len(args)-1]
|
||||||
|
|
||||||
|
remoteCmd := fmt.Sprintf("rsync %s -e %q %s %s",
|
||||||
|
rsyncFlags, innerSSH, sourcePath, destUserHost+":"+destPath)
|
||||||
|
|
||||||
|
sshArgs := []string{
|
||||||
|
"-i", src.PrivKey,
|
||||||
|
"-o", "StrictHostKeyChecking=accept-new",
|
||||||
|
"-o", "UserKnownHostsFile=" + filepath.Join(r.sshDir, "known_hosts"),
|
||||||
|
"-p", fmt.Sprintf("%d", src.Port),
|
||||||
|
fmt.Sprintf("%s@%s", src.SSHUser, src.Host),
|
||||||
|
}
|
||||||
|
sshArgs = append(sshArgs, remoteCmd)
|
||||||
|
|
||||||
|
cmd := exec.CommandContext(ctx, "ssh", sshArgs...)
|
||||||
|
return r.runCmd(ctx, cmd, onLine)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *RsyncRunner) runCmd(ctx context.Context, cmd *exec.Cmd, onLine func(stream string, line string)) (*RsyncResult, error) {
|
||||||
stdout, err := cmd.StdoutPipe()
|
stdout, err := cmd.StdoutPipe()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("stdout pipe: %w", err)
|
return nil, fmt.Errorf("stdout pipe: %w", err)
|
||||||
@@ -133,146 +203,3 @@ func (r *RsyncRunner) Run(ctx context.Context, pair *SyncPairConfig, onLine func
|
|||||||
|
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *RsyncRunner) buildArgs(pair *SyncPairConfig) []string {
|
|
||||||
var args []string
|
|
||||||
|
|
||||||
flags := strings.Fields(pair.RsyncFlags)
|
|
||||||
args = append(args, flags...)
|
|
||||||
|
|
||||||
for _, pattern := range pair.ExcludePatterns {
|
|
||||||
args = append(args, "--exclude="+pattern)
|
|
||||||
}
|
|
||||||
|
|
||||||
if pair.Direction == "mirror" {
|
|
||||||
args = append(args, "--delete")
|
|
||||||
}
|
|
||||||
|
|
||||||
if pair.Direction == "pull" {
|
|
||||||
args = append(args, pair.Dest, pair.Source)
|
|
||||||
} else {
|
|
||||||
args = append(args, pair.Source, pair.Dest)
|
|
||||||
}
|
|
||||||
|
|
||||||
return args
|
|
||||||
}
|
|
||||||
|
|
||||||
type RemoteMachine struct {
|
|
||||||
Host string
|
|
||||||
Port int
|
|
||||||
SSHUser string
|
|
||||||
PrivKey string
|
|
||||||
DestPrivKey string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *RsyncRunner) RunRemote(ctx context.Context, pair *SyncPairConfig, remote RemoteMachine, onLine func(stream string, line string)) (*RsyncResult, error) {
|
|
||||||
args := r.buildArgs(pair)
|
|
||||||
|
|
||||||
var remoteCmd string
|
|
||||||
if remote.DestPrivKey != "" {
|
|
||||||
encodedKey := base64.StdEncoding.EncodeToString([]byte(remote.DestPrivKey))
|
|
||||||
innerSSH := fmt.Sprintf(`ssh -i /tmp/syncserver-dest-key -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/dev/null`)
|
|
||||||
rsyncPart := fmt.Sprintf("rsync %s -e %q",
|
|
||||||
strings.Join(args, " "), innerSSH)
|
|
||||||
remoteCmd = fmt.Sprintf(
|
|
||||||
`echo '%s' | base64 -d > /tmp/syncserver-dest-key && chmod 600 /tmp/syncserver-dest-key && %s; STATUS=$?; rm -f /tmp/syncserver-dest-key; exit $STATUS`,
|
|
||||||
encodedKey, rsyncPart)
|
|
||||||
} else {
|
|
||||||
remoteCmd = "rsync " + strings.Join(args, " ")
|
|
||||||
}
|
|
||||||
|
|
||||||
sshArgs := []string{
|
|
||||||
"-i", remote.PrivKey,
|
|
||||||
"-o", "StrictHostKeyChecking=accept-new",
|
|
||||||
"-o", "UserKnownHostsFile=" + strings.TrimRight(r.sshDir, "/") + "/known_hosts",
|
|
||||||
"-p", fmt.Sprintf("%d", remote.Port),
|
|
||||||
fmt.Sprintf("%s@%s", remote.SSHUser, remote.Host),
|
|
||||||
}
|
|
||||||
sshArgs = append(sshArgs, remoteCmd)
|
|
||||||
|
|
||||||
cmd := exec.CommandContext(ctx, "ssh", sshArgs...)
|
|
||||||
|
|
||||||
stdout, err := cmd.StdoutPipe()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("stdout pipe: %w", err)
|
|
||||||
}
|
|
||||||
stderr, err := cmd.StderrPipe()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("stderr pipe: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := cmd.Start(); err != nil {
|
|
||||||
return nil, fmt.Errorf("starting ssh: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var outLines, errLines []string
|
|
||||||
done := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
br := io.Reader(stdout)
|
|
||||||
buf := make([]byte, 4096)
|
|
||||||
for {
|
|
||||||
n, err := br.Read(buf)
|
|
||||||
if n > 0 {
|
|
||||||
line := strings.TrimRight(string(buf[:n]), "\r\n")
|
|
||||||
if line != "" {
|
|
||||||
outLines = append(outLines, line)
|
|
||||||
if onLine != nil {
|
|
||||||
onLine("stdout", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
default:
|
|
||||||
close(done)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
br := io.Reader(stderr)
|
|
||||||
buf := make([]byte, 4096)
|
|
||||||
for {
|
|
||||||
n, err := br.Read(buf)
|
|
||||||
if n > 0 {
|
|
||||||
line := strings.TrimRight(string(buf[:n]), "\r\n")
|
|
||||||
if line != "" {
|
|
||||||
errLines = append(errLines, line)
|
|
||||||
if onLine != nil {
|
|
||||||
onLine("stderr", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
default:
|
|
||||||
close(done)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
err = cmd.Wait()
|
|
||||||
<-done
|
|
||||||
|
|
||||||
result := &RsyncResult{
|
|
||||||
ExitCode: 0,
|
|
||||||
Stdout: strings.Join(outLines, "\n"),
|
|
||||||
Stderr: strings.Join(errLines, "\n"),
|
|
||||||
Stats: ParseFinalStats(strings.Join(outLines, "\n")),
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
|
||||||
result.ExitCode = exitErr.ExitCode()
|
|
||||||
} else {
|
|
||||||
result.ExitCode = -1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|||||||
Reference in New Issue
Block a user