// Package progress renders human-facing run progress for long archives. // // A Reporter receives events from the single-threaded run loop and renders // them from a background ticker, so blocking phases (a slow scan, fsync, lag // waiting) still produce a heartbeat. Rendering never touches scanner or // writer state directly: the run loop copies what it needs into the Reporter. package progress import ( "fmt" "io" "os" "strconv" "strings" "sync" "time" ) // rateWindow is how far back a rate sample may sit to be considered current. const rateWindow = 5 * time.Second // Estimate describes the numeric range used for percentage and ETA. // Kind is "day" (position is a day boundary in unix seconds) or "key" // (position is a numeric scan key). type Estimate struct { Kind string Lo, Hi float64 } // Options configures a Reporter. type Options struct { Out io.Writer // destination, default os.Stderr Enabled bool // whether progress lines are produced at all Live bool // render in place with carriage returns Interval time.Duration // minimum refresh interval, default 1s RowEvery int64 // extra refresh every N rows; 0 disables Width int // live line limit, default 100 Now func() time.Time } type sample struct { t time.Time rows int64 pos float64 hasPos bool } type data struct { started time.Time phase string reason string phaseT time.Time base int64 read int64 pos float64 hasPos bool originP float64 originT time.Time hasOrig bool estimate *Estimate shard int64 hasShard bool bytes int64 hasBytes bool shards int64 hasShardN bool samples []sample rowMark int64 lastDraw time.Time } func (d *data) rows() int64 { return d.base + d.read } func (d *data) clone() data { c := *d c.samples = append([]sample(nil), d.samples...) return c } // Reporter accumulates run metrics and renders them. type Reporter struct { out io.Writer now func() time.Time enabled bool live bool interval time.Duration rowEvery int64 width int mu sync.Mutex d data started bool running bool stopCh chan struct{} doneCh chan struct{} wmu sync.Mutex lastWidth int } // New builds a Reporter. A disabled Reporter still carries messages (warnings) // to Out but emits no progress lines. func New(o Options) *Reporter { if o.Out == nil { o.Out = os.Stderr } if o.Now == nil { o.Now = time.Now } if o.Interval <= 0 { o.Interval = time.Second } if o.Width <= 0 { o.Width = 100 } return &Reporter{ out: o.Out, now: o.Now, enabled: o.Enabled, live: o.Live, interval: o.Interval, rowEvery: o.RowEvery, width: o.Width, stopCh: make(chan struct{}), doneCh: make(chan struct{}), } } // Start begins the refresh goroutine. It is a no-op for a disabled Reporter. func (r *Reporter) Start() { r.mu.Lock() if r.started { r.mu.Unlock() return } r.started = true r.initLocked(r.now()) if !r.enabled { r.mu.Unlock() return } r.running = true r.mu.Unlock() go r.loop() } func (r *Reporter) initLocked(now time.Time) { r.d.started = now r.d.phaseT = now r.d.lastDraw = time.Time{} if r.rowEvery > 0 { r.d.rowMark = r.d.read / r.rowEvery } r.pushLocked(now) } func (r *Reporter) loop() { t := time.NewTicker(r.interval) defer t.Stop() defer close(r.doneCh) for { select { case <-r.stopCh: return case <-t.C: r.maybeRender(false) } } } // SetBase installs the row count carried over from a previous run. Resetting // the sample ring keeps the rate measured from this run's work rather than // jumping by the inherited count on the first chunk. func (r *Reporter) SetBase(n int64) { r.mu.Lock() r.d.base = n r.d.samples = nil r.pushLocked(r.now()) r.mu.Unlock() } // AddRows accounts for rows read in this run. func (r *Reporter) AddRows(n int64) { if n == 0 { return } r.mu.Lock() r.d.read += n now := r.now() r.pushLocked(now) crossed := false if r.rowEvery > 0 { mark := r.d.read / r.rowEvery if mark > r.d.rowMark { r.d.rowMark = mark crossed = true } } r.mu.Unlock() if crossed { // The threshold is an extra line at a chunk boundary; draw it inline so // it is not lost if the run finishes before the next tick. r.maybeRender(true) } } // SetPosition publishes the current numeric position (day seconds or key). func (r *Reporter) SetPosition(pos float64) { r.mu.Lock() r.d.pos = pos r.d.hasPos = true if !r.d.hasOrig { r.d.originP = pos r.d.originT = r.now() r.d.hasOrig = true } r.mu.Unlock() } // SetEstimate publishes or clears the range used for percentage and ETA. func (r *Reporter) SetEstimate(e *Estimate) { r.mu.Lock() r.d.estimate = e r.mu.Unlock() } // SetShard publishes the current shard sequence number. func (r *Reporter) SetShard(seq int64) { r.mu.Lock() r.d.shard = seq r.d.hasShard = true r.mu.Unlock() } // SetBytes publishes bytes written so far. func (r *Reporter) SetBytes(n int64) { r.mu.Lock() r.d.bytes = n r.d.hasBytes = true r.mu.Unlock() } // SetShards publishes the sealed shard count for the done summary. func (r *Reporter) SetShards(n int64) { r.mu.Lock() r.d.shards = n r.d.hasShardN = true r.mu.Unlock() } // Phase records the current phase and an optional waiting reason. Repeating // the same phase (for example lag-wait reported on every poll) keeps the phase // clock running so its duration is visible. func (r *Reporter) Phase(name, reason string) { r.mu.Lock() if r.d.phase != name { r.d.phase = name r.d.phaseT = r.now() } r.d.reason = reason r.mu.Unlock() } // Message writes a complete diagnostic line, first clearing any live line. func (r *Reporter) Message(format string, args ...any) { line := strings.TrimRight(fmt.Sprintf(format, args...), "\n") r.wmu.Lock() defer r.wmu.Unlock() if r.live { r.clearLocked() } fmt.Fprintln(r.out, line) } // Stop ends refreshing and clears the live line without printing a summary. func (r *Reporter) Stop() { r.stopTicker() r.wmu.Lock() r.clearLocked() r.wmu.Unlock() } // Done prints the final summary line on normal completion. func (r *Reporter) Done() { r.stopTicker() now := r.now() r.mu.Lock() s := r.d.clone() r.mu.Unlock() r.wmu.Lock() defer r.wmu.Unlock() r.clearLocked() if r.enabled { fmt.Fprintln(r.out, buildDone(s, now)) } } // Interrupted prints the final snapshot line after a signal. func (r *Reporter) Interrupted() { r.stopTicker() now := r.now() r.mu.Lock() s := r.d.clone() r.mu.Unlock() r.wmu.Lock() defer r.wmu.Unlock() r.clearLocked() if r.enabled { fmt.Fprintln(r.out, buildLine(s, now)) } } func (r *Reporter) stopTicker() { r.mu.Lock() if !r.running { r.mu.Unlock() return } r.running = false close(r.stopCh) done := r.doneCh r.mu.Unlock() <-done } func (r *Reporter) pushLocked(now time.Time) { r.d.samples = append(r.d.samples, sample{ t: now, rows: r.d.rows(), pos: r.d.pos, hasPos: r.d.hasPos, }) cut := now.Add(-2 * rateWindow) i := 0 for i < len(r.d.samples) && r.d.samples[i].t.Before(cut) { i++ } if i > 0 { r.d.samples = append(r.d.samples[:0], r.d.samples[i:]...) } if len(r.d.samples) > 256 { r.d.samples = r.d.samples[len(r.d.samples)-256:] } } // maybeRender draws a frame. force bypasses the interval throttle so a row // threshold can produce an extra line. func (r *Reporter) maybeRender(force bool) { if !r.enabled { return } now := r.now() r.mu.Lock() if !force && !r.d.lastDraw.IsZero() && now.Sub(r.d.lastDraw) < r.interval { r.mu.Unlock() return } r.d.lastDraw = now s := r.d.clone() r.mu.Unlock() var line string if r.live { line = r.buildLive(s, now) } else { line = buildLine(s, now) } r.wmu.Lock() defer r.wmu.Unlock() if r.live { if line == "" { line = " " } pad := 0 if r.lastWidth > len(line) { pad = r.lastWidth - len(line) } fmt.Fprintf(r.out, "\r%s%s", line, strings.Repeat(" ", pad)) r.lastWidth = len(line) return } fmt.Fprintln(r.out, line) } func (r *Reporter) clearLocked() { if r.live && r.lastWidth > 0 { fmt.Fprintf(r.out, "\r%s\r", strings.Repeat(" ", r.lastWidth)) r.lastWidth = 0 } } func (r *Reporter) buildLive(s data, now time.Time) string { // Field priority when narrowing: drop bytes, then the shard number, then // shrink the bar. The line never wraps. type shape struct { bar int bytes bool shard bool } shapes := []shape{ {20, true, true}, {20, false, true}, {20, false, false}, {12, false, false}, {8, false, false}, {0, false, false}, } var last string for _, sh := range shapes { last = liveLine(s, now, sh.bar, sh.bytes, sh.shard) if len(last) <= r.width { return last } } return truncate(last, r.width) } func liveLine(s data, now time.Time, barWidth int, showBytes, showShard bool) string { var parts []string pct, eta, hasEta, hasEst := calc(s, now) if hasEst { if barWidth > 0 { filled := int(pct*float64(barWidth) + 0.5) parts = append(parts, "["+strings.Repeat("#", filled)+strings.Repeat("-", barWidth-filled)+"]") } parts = append(parts, fmt.Sprintf("~%.0f%%", pct*100)) if s.estimate.Kind == "day" { cur, total := dayProgress(s) parts = append(parts, fmt.Sprintf("day %d/%d", cur, total)) } } parts = append(parts, humanRows(s.rows())+" rows") if rr, ok := rate(s.samples, now, false); ok && rr > 0 { parts = append(parts, humanRate(rr)) } if s.phase != "" { parts = append(parts, fmt.Sprintf("%s %s", s.phase, humanDuration(now.Sub(s.phaseT)))) } if s.reason != "" { parts = append(parts, "("+s.reason+")") } if showShard && s.hasShard { parts = append(parts, fmt.Sprintf("shard %d", s.shard)) } if showBytes && s.hasBytes { parts = append(parts, humanBytes(s.bytes)) } if hasEta { parts = append(parts, "eta ~"+humanDuration(eta)) } parts = append(parts, "elapsed "+humanDuration(now.Sub(s.started))) return strings.Join(parts, " ") } func buildLine(s data, now time.Time) string { var b strings.Builder b.WriteString("progress: rows=") b.WriteString(strconv.FormatInt(s.rows(), 10)) b.WriteString(" rate=") if rr, ok := rate(s.samples, now, false); ok && rr > 0 { b.WriteString(humanRate(rr)) } else { b.WriteString(humanRate(cumulativeRows(s, now))) } b.WriteString(" phase=") if s.phase == "" { b.WriteString("start") } else { b.WriteString(s.phase) } b.WriteString(" phase_elapsed=") b.WriteString(humanDuration(now.Sub(s.phaseT))) b.WriteString(" elapsed=") b.WriteString(humanDuration(now.Sub(s.started))) if s.reason != "" { b.WriteString(" reason=") b.WriteString(strings.Join(strings.Fields(s.reason), "_")) } if s.hasShard { fmt.Fprintf(&b, " shard=%d", s.shard) } if s.hasBytes { fmt.Fprintf(&b, " bytes=%d", s.bytes) } pct, eta, hasEta, hasEst := calc(s, now) if !hasEst { b.WriteString(" estimate=none") return b.String() } fmt.Fprintf(&b, " estimate=%s pct=~%.1f", s.estimate.Kind, pct*100) if s.estimate.Kind == "day" { cur, total := dayProgress(s) fmt.Fprintf(&b, " day=%d/%d", cur, total) } if hasEta { b.WriteString(" eta=~") b.WriteString(humanDuration(eta)) } return b.String() } func buildDone(s data, now time.Time) string { var b strings.Builder b.WriteString("done rows=") b.WriteString(strconv.FormatInt(s.rows(), 10)) b.WriteString(" elapsed=") b.WriteString(humanDuration(now.Sub(s.started))) if s.hasShardN { fmt.Fprintf(&b, " shards=%d", s.shards) } if s.hasBytes { fmt.Fprintf(&b, " bytes=%d", s.bytes) } return b.String() } // calc returns percentage, ETA and whether an estimate is displayable. func calc(s data, now time.Time) (pct float64, eta time.Duration, hasEta, ok bool) { if s.estimate == nil || !s.hasPos { return 0, 0, false, false } span := s.estimate.Hi - s.estimate.Lo if span <= 0 { return 0, 0, false, false } pct = (s.pos - s.estimate.Lo) / span if pct < 0 { pct = 0 } if pct > 1 { pct = 1 } if pr, rok := positionRate(s, now); rok && pr > 0 { remaining := s.estimate.Hi - s.pos if remaining < 0 { remaining = 0 } secs := remaining / pr if secs >= 0 && secs < 9e9 { eta = time.Duration(secs * float64(time.Second)) hasEta = true } } return pct, eta, hasEta, true } func dayProgress(s data) (int, int) { const day = 86400.0 total := int((s.estimate.Hi-s.estimate.Lo)/day) + 1 cur := int((s.pos-s.estimate.Lo)/day) + 1 if cur < 1 { cur = 1 } if cur > total { cur = total } return cur, total } func rate(samples []sample, now time.Time, usePos bool) (float64, bool) { cut := now.Add(-rateWindow) var first *sample for i := range samples { if usePos && !samples[i].hasPos { continue } if !samples[i].t.Before(cut) { first = &samples[i] break } } if first == nil { for i := len(samples) - 1; i >= 0; i-- { if !usePos || samples[i].hasPos { first = &samples[i] break } } } if first == nil || len(samples) == 0 { return 0, false } last := &samples[len(samples)-1] if usePos && !last.hasPos { return 0, false } if first == last { return 0, false } dt := last.t.Sub(first.t).Seconds() if dt <= 0 { return 0, false } var d float64 if usePos { d = last.pos - first.pos } else { d = float64(last.rows - first.rows) } if d < 0 { return 0, false } return d / dt, true } func positionRate(s data, now time.Time) (float64, bool) { if v, ok := rate(s.samples, now, true); ok { return v, true } if s.hasOrig && now.After(s.originT) { return (s.pos - s.originP) / now.Sub(s.originT).Seconds(), true } return 0, false } func cumulativeRows(s data, now time.Time) float64 { el := now.Sub(s.started).Seconds() if el <= 0 { return 0 } return float64(s.read) / el } func humanRows(n int64) string { switch { case n >= 1_000_000_000: return fmt.Sprintf("%.2fG", float64(n)/1e9) case n >= 1_000_000: return fmt.Sprintf("%.2fM", float64(n)/1e6) case n >= 10_000: return fmt.Sprintf("%.1fk", float64(n)/1e3) default: return strconv.FormatInt(n, 10) } } func humanRate(v float64) string { switch { case v >= 1_000_000: return fmt.Sprintf("%.2fM/s", v/1e6) case v >= 10_000: return fmt.Sprintf("%.1fk/s", v/1e3) case v >= 1: return fmt.Sprintf("%.0f/s", v) default: return "0/s" } } func humanBytes(n int64) string { const unit = 1024 if n < unit { return fmt.Sprintf("%dB", n) } div, exp := int64(unit), 0 for v := n / unit; v >= unit; v /= unit { div *= unit exp++ } return fmt.Sprintf("%.1f%ciB", float64(n)/float64(div), "KMGTPE"[exp]) } func humanDuration(d time.Duration) string { if d < 0 { d = 0 } if d < time.Second { return fmt.Sprintf("%dms", d.Milliseconds()) } secs := int(d.Seconds()) switch { case secs < 60: return fmt.Sprintf("%ds", secs) case secs < 3600: return fmt.Sprintf("%dm%02ds", secs/60, secs%60) default: return fmt.Sprintf("%dh%02dm", secs/3600, (secs%3600)/60) } } func truncate(s string, max int) string { if len(s) <= max { return s } if max <= 0 { return "" } runes := []rune(s) if len(runes) <= max { return s } return string(runes[:max]) } // IsTerminal reports whether w is a character device (a terminal). func IsTerminal(w io.Writer) bool { f, ok := w.(*os.File) if !ok { return false } st, err := f.Stat() if err != nil { return false } return st.Mode()&os.ModeCharDevice != 0 }