Bump version to 1.0.51
This commit is contained in:
@@ -1,5 +1,5 @@
|
|||||||
BINARY=syncserver
|
BINARY=syncserver
|
||||||
VERSION?=1.0.50
|
VERSION?=1.0.51
|
||||||
GO?=go
|
GO?=go
|
||||||
LDFLAGS=-s -w -X main.version=$(VERSION) -X main.commit=$(shell git rev-parse --short HEAD 2>/dev/null || echo unknown)
|
LDFLAGS=-s -w -X main.version=$(VERSION) -X main.commit=$(shell git rev-parse --short HEAD 2>/dev/null || echo unknown)
|
||||||
BUILD_FLAGS=CGO_ENABLED=0
|
BUILD_FLAGS=CGO_ENABLED=0
|
||||||
|
|||||||
+1
-1
@@ -20,7 +20,7 @@ import (
|
|||||||
"github.com/syncserver/internal/syncengine"
|
"github.com/syncserver/internal/syncengine"
|
||||||
)
|
)
|
||||||
|
|
||||||
var version = "1.0.50"
|
var version = "1.0.51"
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
cfgPath := flag.String("config", "", "Path to config.yaml")
|
cfgPath := flag.String("config", "", "Path to config.yaml")
|
||||||
|
|||||||
@@ -84,6 +84,8 @@ type JobResponse struct {
|
|||||||
ErrorCode *string `json:"error_code,omitempty"`
|
ErrorCode *string `json:"error_code,omitempty"`
|
||||||
DurationSeconds *int64 `json:"duration_seconds,omitempty"`
|
DurationSeconds *int64 `json:"duration_seconds,omitempty"`
|
||||||
LogLineCount *int64 `json:"log_line_count,omitempty"`
|
LogLineCount *int64 `json:"log_line_count,omitempty"`
|
||||||
|
TotalSizeBytes *int64 `json:"total_size_bytes,omitempty"`
|
||||||
|
SentBytes *int64 `json:"sent_bytes,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type LogLineResponse struct {
|
type LogLineResponse struct {
|
||||||
|
|||||||
@@ -254,6 +254,12 @@ func jobToResp(j models.Job) JobResponse {
|
|||||||
s := j.FinishedAt.Format(time.RFC3339)
|
s := j.FinishedAt.Format(time.RFC3339)
|
||||||
resp.FinishedAt = &s
|
resp.FinishedAt = &s
|
||||||
}
|
}
|
||||||
|
if j.TotalSizeBytes > 0 {
|
||||||
|
resp.TotalSizeBytes = &j.TotalSizeBytes
|
||||||
|
}
|
||||||
|
if j.SentBytes > 0 {
|
||||||
|
resp.SentBytes = &j.SentBytes
|
||||||
|
}
|
||||||
return resp
|
return resp
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
-- 0006_progress_totals.sql
|
||||||
|
|
||||||
|
ALTER TABLE jobs ADD COLUMN total_size_bytes INTEGER DEFAULT 0;
|
||||||
|
ALTER TABLE jobs ADD COLUMN sent_bytes INTEGER DEFAULT 0;
|
||||||
+18
-6
@@ -15,6 +15,8 @@ type Job struct {
|
|||||||
LogFile *string `db:"log_file" json:"log_file"`
|
LogFile *string `db:"log_file" json:"log_file"`
|
||||||
ErrorMessage *string `db:"error_message" json:"error_message,omitempty"`
|
ErrorMessage *string `db:"error_message" json:"error_message,omitempty"`
|
||||||
ErrorCode *string `db:"error_code" json:"error_code,omitempty"`
|
ErrorCode *string `db:"error_code" json:"error_code,omitempty"`
|
||||||
|
TotalSizeBytes int64 `db:"total_size_bytes" json:"total_size_bytes,omitempty"`
|
||||||
|
SentBytes int64 `db:"sent_bytes" json:"sent_bytes,omitempty"`
|
||||||
CreatedAt time.Time `db:"created_at" json:"created_at"`
|
CreatedAt time.Time `db:"created_at" json:"created_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -43,9 +45,10 @@ func (r *JobRepository) GetByID(id int64) (*Job, error) {
|
|||||||
var logFile, errMsg, errCode sql.NullString
|
var logFile, errMsg, errCode sql.NullString
|
||||||
err := r.db.QueryRow(`
|
err := r.db.QueryRow(`
|
||||||
SELECT id, sync_pair_id, trigger_type, status, started_at, finished_at,
|
SELECT id, sync_pair_id, trigger_type, status, started_at, finished_at,
|
||||||
log_file, error_message, error_code, created_at FROM jobs WHERE id = ?`, id).Scan(
|
log_file, error_message, error_code, total_size_bytes, sent_bytes, created_at
|
||||||
|
FROM jobs WHERE id = ?`, id).Scan(
|
||||||
&j.ID, &j.SyncPairID, &j.TriggerType, &j.Status, &started, &finished,
|
&j.ID, &j.SyncPairID, &j.TriggerType, &j.Status, &started, &finished,
|
||||||
&logFile, &errMsg, &errCode, &j.CreatedAt)
|
&logFile, &errMsg, &errCode, &j.TotalSizeBytes, &j.SentBytes, &j.CreatedAt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -70,7 +73,7 @@ func (r *JobRepository) GetByID(id int64) (*Job, error) {
|
|||||||
func (r *JobRepository) GetAll(limit, offset int) ([]Job, error) {
|
func (r *JobRepository) GetAll(limit, offset int) ([]Job, error) {
|
||||||
rows, err := r.db.Query(`
|
rows, err := r.db.Query(`
|
||||||
SELECT id, sync_pair_id, trigger_type, status, started_at, finished_at,
|
SELECT id, sync_pair_id, trigger_type, status, started_at, finished_at,
|
||||||
log_file, error_message, error_code, created_at
|
log_file, error_message, error_code, total_size_bytes, sent_bytes, created_at
|
||||||
FROM jobs ORDER BY created_at DESC LIMIT ? OFFSET ?`,
|
FROM jobs ORDER BY created_at DESC LIMIT ? OFFSET ?`,
|
||||||
limit, offset)
|
limit, offset)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -84,7 +87,8 @@ func (r *JobRepository) GetAll(limit, offset int) ([]Job, error) {
|
|||||||
var started, finished sql.NullTime
|
var started, finished sql.NullTime
|
||||||
var logFile, errMsg, errCode sql.NullString
|
var logFile, errMsg, errCode sql.NullString
|
||||||
if err := rows.Scan(&j.ID, &j.SyncPairID, &j.TriggerType, &j.Status,
|
if err := rows.Scan(&j.ID, &j.SyncPairID, &j.TriggerType, &j.Status,
|
||||||
&started, &finished, &logFile, &errMsg, &errCode, &j.CreatedAt); err != nil {
|
&started, &finished, &logFile, &errMsg, &errCode,
|
||||||
|
&j.TotalSizeBytes, &j.SentBytes, &j.CreatedAt); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if started.Valid {
|
if started.Valid {
|
||||||
@@ -144,11 +148,11 @@ func (r *JobRepository) GetRunningBySyncPair(syncPairID int64) (*Job, error) {
|
|||||||
var logFile, errMsg, errCode sql.NullString
|
var logFile, errMsg, errCode sql.NullString
|
||||||
err := r.db.QueryRow(`
|
err := r.db.QueryRow(`
|
||||||
SELECT id, sync_pair_id, trigger_type, status, started_at, finished_at,
|
SELECT id, sync_pair_id, trigger_type, status, started_at, finished_at,
|
||||||
log_file, error_message, error_code, created_at FROM jobs
|
log_file, error_message, error_code, total_size_bytes, sent_bytes, created_at FROM jobs
|
||||||
WHERE sync_pair_id = ? AND status IN ('queued','waking_up','running')
|
WHERE sync_pair_id = ? AND status IN ('queued','waking_up','running')
|
||||||
ORDER BY created_at DESC LIMIT 1`, syncPairID).Scan(
|
ORDER BY created_at DESC LIMIT 1`, syncPairID).Scan(
|
||||||
&j.ID, &j.SyncPairID, &j.TriggerType, &j.Status, &started,
|
&j.ID, &j.SyncPairID, &j.TriggerType, &j.Status, &started,
|
||||||
&j.FinishedAt, &logFile, &errMsg, &errCode, &j.CreatedAt)
|
&j.FinishedAt, &logFile, &errMsg, &errCode, &j.TotalSizeBytes, &j.SentBytes, &j.CreatedAt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -180,3 +184,11 @@ func (r *JobRepository) DeleteFinishedBefore(before time.Time) (int64, error) {
|
|||||||
}
|
}
|
||||||
return res.RowsAffected()
|
return res.RowsAffected()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *JobRepository) SetTotals(id int64, totalSize, sentBytes int64) error {
|
||||||
|
_, err := r.db.Exec(
|
||||||
|
"UPDATE jobs SET total_size_bytes = ?, sent_bytes = ? WHERE id = ?",
|
||||||
|
totalSize, sentBytes, id,
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|||||||
@@ -85,6 +85,19 @@ func (r *JobLogRepository) DeleteBefore(before time.Time) (int64, error) {
|
|||||||
return res.RowsAffected()
|
return res.RowsAffected()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *JobLogRepository) TruncateKeepingHeaderTail(jobID int64, head, tail int) error {
|
||||||
|
_, err := r.db.Exec(`
|
||||||
|
DELETE FROM job_logs
|
||||||
|
WHERE job_id = ?
|
||||||
|
AND id NOT IN (
|
||||||
|
SELECT id FROM job_logs WHERE job_id = ? ORDER BY id ASC LIMIT ?
|
||||||
|
UNION ALL
|
||||||
|
SELECT id FROM job_logs WHERE job_id = ? ORDER BY id DESC LIMIT ?
|
||||||
|
)`,
|
||||||
|
jobID, jobID, head, jobID, tail)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
type JobWithStats struct {
|
type JobWithStats struct {
|
||||||
Job
|
Job
|
||||||
DurationSeconds *int64 `db:"duration_seconds" json:"duration_seconds"`
|
DurationSeconds *int64 `db:"duration_seconds" json:"duration_seconds"`
|
||||||
@@ -125,7 +138,7 @@ func (r *JobLogRepository) GetAllFiltered(limit, offset int, syncPairID *int64,
|
|||||||
SELECT
|
SELECT
|
||||||
j.id, j.sync_pair_id, j.trigger_type, j.status,
|
j.id, j.sync_pair_id, j.trigger_type, j.status,
|
||||||
j.started_at, j.finished_at, j.log_file,
|
j.started_at, j.finished_at, j.log_file,
|
||||||
j.error_message, j.error_code, j.created_at,
|
j.error_message, j.error_code, j.total_size_bytes, j.sent_bytes, j.created_at,
|
||||||
CASE WHEN j.finished_at IS NOT NULL AND j.started_at IS NOT NULL
|
CASE WHEN j.finished_at IS NOT NULL AND j.started_at IS NOT NULL
|
||||||
THEN (j.finished_at - j.started_at) ELSE NULL END as duration_seconds,
|
THEN (j.finished_at - j.started_at) ELSE NULL END as duration_seconds,
|
||||||
(SELECT COUNT(*) FROM job_logs WHERE job_id = j.id) as log_line_count
|
(SELECT COUNT(*) FROM job_logs WHERE job_id = j.id) as log_line_count
|
||||||
@@ -148,7 +161,7 @@ func (r *JobLogRepository) GetAllFiltered(limit, offset int, syncPairID *int64,
|
|||||||
var durationSeconds sql.NullInt64
|
var durationSeconds sql.NullInt64
|
||||||
if err := rows.Scan(&j.ID, &j.SyncPairID, &j.TriggerType, &j.Status,
|
if err := rows.Scan(&j.ID, &j.SyncPairID, &j.TriggerType, &j.Status,
|
||||||
&started, &finished, &logFile,
|
&started, &finished, &logFile,
|
||||||
&errMsg, &errCode, &j.CreatedAt,
|
&errMsg, &errCode, &j.TotalSizeBytes, &j.SentBytes, &j.CreatedAt,
|
||||||
&durationSeconds, &j.LogLineCount); err != nil {
|
&durationSeconds, &j.LogLineCount); err != nil {
|
||||||
return nil, 0, err
|
return nil, 0, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,6 +35,9 @@ type Event struct {
|
|||||||
Value string
|
Value string
|
||||||
Line string
|
Line string
|
||||||
Stream string
|
Stream string
|
||||||
|
Progress *ProgressFields
|
||||||
|
TotalBytes int64
|
||||||
|
SentBytes int64
|
||||||
}
|
}
|
||||||
|
|
||||||
func New(database interface{ SQLDB() *sql.DB }, cfg *config.Config) *Engine {
|
func New(database interface{ SQLDB() *sql.DB }, cfg *config.Config) *Engine {
|
||||||
@@ -243,7 +246,17 @@ func (e *Engine) Run(ctx context.Context, jobID int64, pairID int64) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var lastFileName string
|
||||||
onLine := func(stream, line string) {
|
onLine := func(stream, line string) {
|
||||||
|
if stream == "stdout" && isProgressOnlyLine(line) {
|
||||||
|
if p := parseProgressFields(line); p != nil {
|
||||||
|
e.emit(Event{Type: "progress", JobID: jobID, Line: line, Stream: stream, Progress: p, Value: lastFileName})
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if stream == "stdout" && isFileNameLine(line) {
|
||||||
|
lastFileName = line
|
||||||
|
}
|
||||||
f, _ := os.OpenFile(logPath, os.O_APPEND|os.O_WRONLY, 0644)
|
f, _ := os.OpenFile(logPath, os.O_APPEND|os.O_WRONLY, 0644)
|
||||||
if f != nil {
|
if f != nil {
|
||||||
fmt.Fprintln(f, line)
|
fmt.Fprintln(f, line)
|
||||||
@@ -286,6 +299,11 @@ func (e *Engine) Run(ctx context.Context, jobID int64, pairID int64) error {
|
|||||||
}
|
}
|
||||||
flush()
|
flush()
|
||||||
|
|
||||||
|
stats := result.Stats
|
||||||
|
if stats != nil && stats.TotalSize > 0 {
|
||||||
|
e.emit(Event{Type: "progress_total", JobID: jobID, TotalBytes: stats.TotalSize, SentBytes: stats.SentBytes})
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if jobCtx.Err() != nil {
|
if jobCtx.Err() != nil {
|
||||||
code := "cancelled_shutdown"
|
code := "cancelled_shutdown"
|
||||||
@@ -297,6 +315,7 @@ func (e *Engine) Run(ctx context.Context, jobID int64, pairID int64) error {
|
|||||||
e.setJobError(jobID, code, msg)
|
e.setJobError(jobID, code, msg)
|
||||||
e.setJobStatus(jobID, "cancelled")
|
e.setJobStatus(jobID, "cancelled")
|
||||||
e.emit(Event{Type: "status", JobID: jobID, Key: "status", Value: "cancelled", Line: msg})
|
e.emit(Event{Type: "status", JobID: jobID, Key: "status", Value: "cancelled", Line: msg})
|
||||||
|
e.persistAndClose(jobID)
|
||||||
return jobCtx.Err()
|
return jobCtx.Err()
|
||||||
}
|
}
|
||||||
e.setJobStatus(jobID, "failed")
|
e.setJobStatus(jobID, "failed")
|
||||||
@@ -315,11 +334,15 @@ func (e *Engine) Run(ctx context.Context, jobID int64, pairID int64) error {
|
|||||||
e.setJobStatus(jobID, "failed")
|
e.setJobStatus(jobID, "failed")
|
||||||
e.setJobError(jobID, errCode, errMsg)
|
e.setJobError(jobID, errCode, errMsg)
|
||||||
e.emit(Event{Type: "status", JobID: jobID, Key: "status", Value: "failed", Line: errMsg})
|
e.emit(Event{Type: "status", JobID: jobID, Key: "status", Value: "failed", Line: errMsg})
|
||||||
|
e.persistAndClose(jobID)
|
||||||
return fmt.Errorf("rsync exited with code %d: %s", result.ExitCode, result.Stderr)
|
return fmt.Errorf("rsync exited with code %d: %s", result.ExitCode, result.Stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
e.setJobStatus(jobID, "success")
|
e.setJobStatus(jobID, "success")
|
||||||
e.emit(Event{Type: "status", JobID: jobID, Key: "status", Value: "success"})
|
e.emit(Event{Type: "status", JobID: jobID, Key: "status", Value: "success"})
|
||||||
|
if stats != nil {
|
||||||
|
e.persistJobTotals(jobID, stats)
|
||||||
|
}
|
||||||
slog.Info("job completed", "job_id", jobID, "pair", pair.Name)
|
slog.Info("job completed", "job_id", jobID, "pair", pair.Name)
|
||||||
e.persistAndClose(jobID)
|
e.persistAndClose(jobID)
|
||||||
return nil
|
return nil
|
||||||
@@ -327,6 +350,23 @@ func (e *Engine) Run(ctx context.Context, jobID int64, pairID int64) error {
|
|||||||
|
|
||||||
func (e *Engine) persistAndClose(jobID int64) {
|
func (e *Engine) persistAndClose(jobID int64) {
|
||||||
e.eventBus.CloseJobChannels(jobID)
|
e.eventBus.CloseJobChannels(jobID)
|
||||||
|
logRepo := models.NewJobLogRepository(e.db)
|
||||||
|
count, err := logRepo.CountByJobID(jobID)
|
||||||
|
if err == nil && count > 2000 {
|
||||||
|
if truncateErr := logRepo.TruncateKeepingHeaderTail(jobID, 50, 100); truncateErr != nil {
|
||||||
|
slog.Warn("failed to truncate job logs", "job_id", jobID, "error", truncateErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) persistJobTotals(jobID int64, stats *RsyncStats) {
|
||||||
|
if stats == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
jobRepo := models.NewJobRepository(e.db)
|
||||||
|
if err := jobRepo.SetTotals(jobID, stats.TotalSize, stats.SentBytes); err != nil {
|
||||||
|
slog.Warn("failed to persist job totals", "job_id", jobID, "error", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) resolveSSHKey(machine *models.Machine) (string, error) {
|
func (e *Engine) resolveSSHKey(machine *models.Machine) (string, error) {
|
||||||
|
|||||||
@@ -24,6 +24,15 @@ type ProgressLine struct {
|
|||||||
XferedBytes int64
|
XferedBytes int64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ProgressFields struct {
|
||||||
|
FileBytes int64
|
||||||
|
Pct int
|
||||||
|
SpeedBps int64
|
||||||
|
EtaSeconds int
|
||||||
|
XfrDone int
|
||||||
|
XfrTotal int
|
||||||
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
progressRegex = regexp.MustCompile(`\s*([\d,]+)\s+([\d,]+)\s+([\d%]+)\s*`)
|
progressRegex = regexp.MustCompile(`\s*([\d,]+)\s+([\d,]+)\s+([\d%]+)\s*`)
|
||||||
sentRegex = regexp.MustCompile(`sent\s+([\d,]+)\s+bytes`)
|
sentRegex = regexp.MustCompile(`sent\s+([\d,]+)\s+bytes`)
|
||||||
@@ -32,6 +41,24 @@ var (
|
|||||||
filesRegex = regexp.MustCompile(`Number of files: ([\d,]+)`)
|
filesRegex = regexp.MustCompile(`Number of files: ([\d,]+)`)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var perFileProgressRegex = regexp.MustCompile(
|
||||||
|
`^\s*(\d{1,3}(?:,\d{3})+)\s+(\d+)%\s+(\d+\.\d+)([kMG])B/s\s+(\d+:\d{2}:\d{2})(.*)`,
|
||||||
|
)
|
||||||
|
|
||||||
|
var xfrRegex = regexp.MustCompile(`xfr#(\d+).*to-chk=(\d+)/(\d+)`)
|
||||||
|
|
||||||
|
func parseXfrSuffix(suffix string) (done, total int) {
|
||||||
|
m := xfrRegex.FindStringSubmatch(suffix)
|
||||||
|
if m == nil {
|
||||||
|
return 0, 0
|
||||||
|
}
|
||||||
|
done, _ = strconv.Atoi(m[1])
|
||||||
|
t, _ := strconv.Atoi(m[2])
|
||||||
|
_ = t
|
||||||
|
total, _ = strconv.Atoi(m[3])
|
||||||
|
return done, total
|
||||||
|
}
|
||||||
|
|
||||||
func ParseProgressLine(line string) *ProgressLine {
|
func ParseProgressLine(line string) *ProgressLine {
|
||||||
if strings.Contains(line, "files to consider") || strings.Contains(line, "files...") {
|
if strings.Contains(line, "files to consider") || strings.Contains(line, "files...") {
|
||||||
return &ProgressLine{Phase: "scanning"}
|
return &ProgressLine{Phase: "scanning"}
|
||||||
@@ -74,3 +101,71 @@ func ParseFinalStats(output string) *RsyncStats {
|
|||||||
}
|
}
|
||||||
return stats
|
return stats
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isProgressOnlyLine(line string) bool {
|
||||||
|
return perFileProgressRegex.MatchString(line)
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseProgressFields(line string) *ProgressFields {
|
||||||
|
m := perFileProgressRegex.FindStringSubmatch(line)
|
||||||
|
if m == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
bytes, _ := strconv.ParseInt(strings.ReplaceAll(m[1], ",", ""), 10, 64)
|
||||||
|
pct, _ := strconv.Atoi(m[2])
|
||||||
|
speed, _ := strconv.ParseFloat(m[3], 64)
|
||||||
|
unit := m[4]
|
||||||
|
eta := m[5]
|
||||||
|
suffix := m[6]
|
||||||
|
|
||||||
|
speedBps := int64(speed * 1e6)
|
||||||
|
switch unit {
|
||||||
|
case "k", "K":
|
||||||
|
speedBps = int64(speed * 1e3)
|
||||||
|
case "m", "M":
|
||||||
|
speedBps = int64(speed * 1e6)
|
||||||
|
case "g", "G":
|
||||||
|
speedBps = int64(speed * 1e9)
|
||||||
|
}
|
||||||
|
|
||||||
|
etaSecs := 0
|
||||||
|
parts := strings.Split(eta, ":")
|
||||||
|
if len(parts) == 3 {
|
||||||
|
h, _ := strconv.Atoi(parts[0])
|
||||||
|
m, _ := strconv.Atoi(parts[1])
|
||||||
|
s, _ := strconv.Atoi(parts[2])
|
||||||
|
etaSecs = h*3600 + m*60 + s
|
||||||
|
}
|
||||||
|
|
||||||
|
pf := &ProgressFields{
|
||||||
|
FileBytes: bytes,
|
||||||
|
Pct: pct,
|
||||||
|
SpeedBps: speedBps,
|
||||||
|
EtaSeconds: etaSecs,
|
||||||
|
}
|
||||||
|
|
||||||
|
if suffix != "" {
|
||||||
|
done, total := parseXfrSuffix(suffix)
|
||||||
|
pf.XfrDone = done
|
||||||
|
pf.XfrTotal = total
|
||||||
|
}
|
||||||
|
|
||||||
|
return pf
|
||||||
|
}
|
||||||
|
|
||||||
|
func isFileNameLine(line string) bool {
|
||||||
|
if line == "" || strings.TrimSpace(line) == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if strings.Contains(line, "sending incremental file list") ||
|
||||||
|
strings.Contains(line, "building file list") ||
|
||||||
|
strings.Contains(line, "cannot open") ||
|
||||||
|
strings.Contains(line, "skipping non-regular") ||
|
||||||
|
strings.HasPrefix(line, "sent ") ||
|
||||||
|
strings.HasPrefix(line, "total ") ||
|
||||||
|
strings.HasPrefix(line, "Number of files:") ||
|
||||||
|
strings.Contains(line, "bytes received") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return !isProgressOnlyLine(line)
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
package syncengine
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestIsProgressOnlyLine(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
line string
|
||||||
|
expect bool
|
||||||
|
}{
|
||||||
|
{"per-file progress 0%", " 32,768 0% 0.00kB/s 0:00:00", true},
|
||||||
|
{"per-file progress 7%", " 2,260,893,696 7% 51.14MB/s 0:08:45", true},
|
||||||
|
{"per-file progress with xfr suffix", " 67,141,632 0% 32.02MB/s 0:15:06 (xfr#1, to-chk=4/10)", true},
|
||||||
|
{"filename line", "Dragon Ball Sleeping Princess in Devil's Castle (1987)/", false},
|
||||||
|
{"sending incremental file list header", "sending incremental file list", false},
|
||||||
|
{"sent bytes stats", "sent 123,456 bytes received 789 bytes 12.34kB/s", false},
|
||||||
|
{"total size stats", "total size is 999,999,999 speedup is 1.23", false},
|
||||||
|
{"Number of files stats", "Number of files: 10", false},
|
||||||
|
{"building file list", "building file list ...", false},
|
||||||
|
{"empty line", "", false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got := isProgressOnlyLine(tc.line)
|
||||||
|
if got != tc.expect {
|
||||||
|
t.Errorf("isProgressOnlyLine(%q) = %v, want %v", tc.line, got, tc.expect)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseProgressFields(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
line string
|
||||||
|
wantPct int
|
||||||
|
wantSpeedBps int64
|
||||||
|
wantEtaSeconds int
|
||||||
|
wantXfrDone int
|
||||||
|
wantXfrTotal int
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "progress 0% with kB/s",
|
||||||
|
line: " 32,768 0% 0.00kB/s 0:00:00",
|
||||||
|
wantPct: 0,
|
||||||
|
wantSpeedBps: 0,
|
||||||
|
wantEtaSeconds: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "progress 7% with MB/s",
|
||||||
|
line: " 2,260,893,696 7% 51.14MB/s 0:08:45",
|
||||||
|
wantPct: 7,
|
||||||
|
wantSpeedBps: 51_140_000,
|
||||||
|
wantEtaSeconds: 8*60 + 45,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "progress with xfr suffix",
|
||||||
|
line: " 67,141,632 0% 32.02MB/s 0:15:06 (xfr#1, to-chk=4/10)",
|
||||||
|
wantPct: 0,
|
||||||
|
wantSpeedBps: 32_020_000,
|
||||||
|
wantEtaSeconds: 15*60 + 6,
|
||||||
|
wantXfrDone: 1,
|
||||||
|
wantXfrTotal: 10,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "progress with GB/s",
|
||||||
|
line: " 1,234,567,890 50% 1.23GB/s 0:01:30",
|
||||||
|
wantPct: 50,
|
||||||
|
wantSpeedBps: 1_230_000_000,
|
||||||
|
wantEtaSeconds: 1*60 + 30,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
p := parseProgressFields(tc.line)
|
||||||
|
if p == nil {
|
||||||
|
t.Fatalf("parseProgressFields(%q) returned nil, want non-nil", tc.line)
|
||||||
|
}
|
||||||
|
if p.Pct != tc.wantPct {
|
||||||
|
t.Errorf("pct = %d, want %d", p.Pct, tc.wantPct)
|
||||||
|
}
|
||||||
|
if p.SpeedBps != tc.wantSpeedBps {
|
||||||
|
t.Errorf("speedBps = %d, want %d", p.SpeedBps, tc.wantSpeedBps)
|
||||||
|
}
|
||||||
|
if p.EtaSeconds != tc.wantEtaSeconds {
|
||||||
|
t.Errorf("etaSeconds = %d, want %d", p.EtaSeconds, tc.wantEtaSeconds)
|
||||||
|
}
|
||||||
|
if tc.wantXfrTotal > 0 && p.XfrDone != tc.wantXfrDone {
|
||||||
|
t.Errorf("xfrDone = %d, want %d", p.XfrDone, tc.wantXfrDone)
|
||||||
|
}
|
||||||
|
if tc.wantXfrTotal > 0 && p.XfrTotal != tc.wantXfrTotal {
|
||||||
|
t.Errorf("xfrTotal = %d, want %d", p.XfrTotal, tc.wantXfrTotal)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIsFileNameLine(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
line string
|
||||||
|
expect bool
|
||||||
|
}{
|
||||||
|
{"directory path", "Dragon Ball Sleeping Princess in Devil's Castle (1987)/", true},
|
||||||
|
{"file path", "Dragon Ball Sleeping Princess in Devil's Castle (1987)/Dragon Ball...WEBDL-2160p.mkv", true},
|
||||||
|
{"sending incremental file list header", "sending incremental file list", false},
|
||||||
|
{"sent stats", "sent 12,345 bytes received 1,234 bytes", false},
|
||||||
|
{"total size stats", "total size is 999,999,999", false},
|
||||||
|
{"Number of files", "Number of files: 10", false},
|
||||||
|
{"progress line", " 2,260,893,696 7% 51.14MB/s 0:08:45", false},
|
||||||
|
{"empty", "", false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got := isFileNameLine(tc.line)
|
||||||
|
if got != tc.expect {
|
||||||
|
t.Errorf("isFileNameLine(%q) = %v, want %v", tc.line, got, tc.expect)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -28,18 +28,32 @@ import {
|
|||||||
Terminal,
|
Terminal,
|
||||||
AlertCircle,
|
AlertCircle,
|
||||||
Ban,
|
Ban,
|
||||||
|
ChevronDown,
|
||||||
|
Activity,
|
||||||
} from 'lucide-react';
|
} from 'lucide-react';
|
||||||
import { toast } from 'sonner';
|
import { toast } from 'sonner';
|
||||||
import { statusVariant, statusLabel, getErrorCodeInfo } from '@/lib/status';
|
import { statusVariant, statusLabel, getErrorCodeInfo } from '@/lib/status';
|
||||||
import { formatDuration } from '@/lib/utils';
|
import { formatDuration } from '@/lib/utils';
|
||||||
import { cn } from '@/lib/utils';
|
import { cn } from '@/lib/utils';
|
||||||
|
|
||||||
|
interface SSEProgress {
|
||||||
|
fileBytes: number;
|
||||||
|
pct: number;
|
||||||
|
speedBps: number;
|
||||||
|
etaSeconds: number;
|
||||||
|
xfrDone: number;
|
||||||
|
xfrTotal: number;
|
||||||
|
}
|
||||||
|
|
||||||
interface SSEEvent {
|
interface SSEEvent {
|
||||||
type: string;
|
type: string;
|
||||||
job_id: number;
|
job_id: number;
|
||||||
status?: string;
|
status?: string;
|
||||||
line?: string;
|
line?: string;
|
||||||
stream?: string;
|
stream?: string;
|
||||||
|
progress?: SSEProgress;
|
||||||
|
totalBytes?: number;
|
||||||
|
sentBytes?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export default function JobDetail() {
|
export default function JobDetail() {
|
||||||
@@ -56,6 +70,10 @@ export default function JobDetail() {
|
|||||||
const [cancelModal, setCancelModal] = useState(false);
|
const [cancelModal, setCancelModal] = useState(false);
|
||||||
const [cancelReason, setCancelReason] = useState('');
|
const [cancelReason, setCancelReason] = useState('');
|
||||||
const [errorModal, setErrorModal] = useState(false);
|
const [errorModal, setErrorModal] = useState(false);
|
||||||
|
const [progress, setProgress] = useState<SSEProgress | null>(null);
|
||||||
|
const [finalTotals, setFinalTotals] = useState<{ totalBytes: number; sentBytes: number } | null>(null);
|
||||||
|
const [hasMoreLogs, setHasMoreLogs] = useState(false);
|
||||||
|
const [loadingMore, setLoadingMore] = useState(false);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
loadJob();
|
loadJob();
|
||||||
@@ -78,6 +96,13 @@ export default function JobDetail() {
|
|||||||
return updated;
|
return updated;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
if (evt.type === 'progress' && evt.progress) {
|
||||||
|
setProgress(evt.progress);
|
||||||
|
}
|
||||||
|
if (evt.type === 'progress_total' && evt.totalBytes !== undefined && evt.sentBytes !== undefined) {
|
||||||
|
setFinalTotals({ totalBytes: evt.totalBytes, sentBytes: evt.sentBytes });
|
||||||
|
setProgress(null);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return () => esRef.current?.close();
|
return () => esRef.current?.close();
|
||||||
@@ -109,8 +134,10 @@ export default function JobDetail() {
|
|||||||
)) ?? [];
|
)) ?? [];
|
||||||
if (offset === 0) {
|
if (offset === 0) {
|
||||||
setLogs(ls);
|
setLogs(ls);
|
||||||
|
setHasMoreLogs(ls.length === 1000);
|
||||||
} else {
|
} else {
|
||||||
setLogs(prev => [...prev, ...ls]);
|
setLogs(prev => [...prev, ...ls]);
|
||||||
|
setHasMoreLogs(ls.length === 1000);
|
||||||
}
|
}
|
||||||
} catch {}
|
} catch {}
|
||||||
}
|
}
|
||||||
@@ -321,6 +348,10 @@ export default function JobDetail() {
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{(progress || finalTotals) && (
|
||||||
|
<TransferProgress progress={progress} finalTotals={finalTotals} />
|
||||||
|
)}
|
||||||
|
|
||||||
<Card className="flex flex-col min-h-0">
|
<Card className="flex flex-col min-h-0">
|
||||||
<div className="flex items-center justify-between px-4 py-3 border-b border-border shrink-0">
|
<div className="flex items-center justify-between px-4 py-3 border-b border-border shrink-0">
|
||||||
<div className="flex items-center gap-2">
|
<div className="flex items-center gap-2">
|
||||||
@@ -378,6 +409,23 @@ export default function JobDetail() {
|
|||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
<div ref={logEndRef} />
|
<div ref={logEndRef} />
|
||||||
|
{hasMoreLogs && (
|
||||||
|
<div className="flex justify-center py-2">
|
||||||
|
<Button
|
||||||
|
variant="secondary"
|
||||||
|
size="sm"
|
||||||
|
onClick={async () => {
|
||||||
|
setLoadingMore(true);
|
||||||
|
await loadLogs(logs.length);
|
||||||
|
setLoadingMore(false);
|
||||||
|
}}
|
||||||
|
disabled={loadingMore}
|
||||||
|
>
|
||||||
|
<ChevronDown className="h-4 w-4" />
|
||||||
|
{loadingMore ? 'Loading...' : 'Load more'}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
</Card>
|
</Card>
|
||||||
|
|
||||||
@@ -452,3 +500,86 @@ function LogLine({
|
|||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatBytes(bytes: number): string {
|
||||||
|
if (bytes === 0) return '0 B';
|
||||||
|
const units = ['B', 'kB', 'MB', 'GB', 'TB'];
|
||||||
|
const i = Math.floor(Math.log(bytes) / Math.log(1024));
|
||||||
|
return `${(bytes / Math.pow(1024, i)).toFixed(1)} ${units[i]}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatSpeed(bps: number): string {
|
||||||
|
if (bps === 0) return '0 B/s';
|
||||||
|
const units = ['B/s', 'kB/s', 'MB/s', 'GB/s'];
|
||||||
|
const i = Math.floor(Math.log(bps) / Math.log(1000));
|
||||||
|
return `${(bps / Math.pow(1000, i)).toFixed(1)} ${units[i]}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function TransferProgress({
|
||||||
|
progress,
|
||||||
|
finalTotals,
|
||||||
|
}: {
|
||||||
|
progress: SSEProgress | null;
|
||||||
|
finalTotals: { totalBytes: number; sentBytes: number } | null;
|
||||||
|
}) {
|
||||||
|
const globalPct = finalTotals && finalTotals.totalBytes > 0
|
||||||
|
? Math.round((finalTotals.sentBytes / finalTotals.totalBytes) * 100)
|
||||||
|
: null;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Card>
|
||||||
|
<div className="p-4 space-y-3">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<Activity className="h-4 w-4 text-accent" />
|
||||||
|
<span className="text-xs font-semibold text-fg-muted uppercase tracking-wider">
|
||||||
|
Transfer Progress
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{progress && (
|
||||||
|
<>
|
||||||
|
<div className="space-y-1.5">
|
||||||
|
<div className="flex justify-between text-xs text-fg-muted">
|
||||||
|
<span>Current file</span>
|
||||||
|
<span>{progress.pct}%</span>
|
||||||
|
</div>
|
||||||
|
<div className="h-2 bg-border rounded-full overflow-hidden">
|
||||||
|
<div
|
||||||
|
className="h-full bg-accent transition-all duration-300 rounded-full"
|
||||||
|
style={{ width: `${progress.pct}%` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex justify-between text-xs text-fg-muted">
|
||||||
|
<span className="font-mono">
|
||||||
|
xfr#{(progress.xfrDone).toLocaleString()}/{progress.xfrTotal > 0 ? progress.xfrTotal.toLocaleString() : '?'}
|
||||||
|
</span>
|
||||||
|
<span className="font-mono">
|
||||||
|
{formatSpeed(progress.speedBps)}
|
||||||
|
</span>
|
||||||
|
<span className="font-mono">
|
||||||
|
ETA {progress.etaSeconds > 0 ? `${Math.floor(progress.etaSeconds / 60)}m ${progress.etaSeconds % 60}s` : '-'}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{finalTotals && (
|
||||||
|
<div className="space-y-1.5">
|
||||||
|
<div className="flex justify-between text-xs text-fg-muted">
|
||||||
|
<span>Total transferred</span>
|
||||||
|
<span>{globalPct}% — {formatBytes(finalTotals.sentBytes)} / {formatBytes(finalTotals.totalBytes)}</span>
|
||||||
|
</div>
|
||||||
|
<div className="h-2 bg-border rounded-full overflow-hidden">
|
||||||
|
<div
|
||||||
|
className="h-full bg-emerald-500 transition-all duration-300 rounded-full"
|
||||||
|
style={{ width: `${globalPct ?? 0}%` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</Card>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user