Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
708 lines
15 KiB
Go
708 lines
15 KiB
Go
// Package progress renders human-facing run progress for long archives.
|
|
//
|
|
// A Reporter receives events from the single-threaded run loop and renders
|
|
// them from a background ticker, so blocking phases (a slow scan, fsync, lag
|
|
// waiting) still produce a heartbeat. Rendering never touches scanner or
|
|
// writer state directly: the run loop copies what it needs into the Reporter.
|
|
package progress
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// rateWindow is how far back a rate sample may sit to be considered current.
|
|
const rateWindow = 5 * time.Second
|
|
|
|
// Estimate describes the numeric range used for percentage and ETA.
|
|
// Kind is "day" (position is a day boundary in unix seconds) or "key"
|
|
// (position is a numeric scan key).
|
|
type Estimate struct {
|
|
Kind string
|
|
Lo, Hi float64
|
|
}
|
|
|
|
// Options configures a Reporter.
|
|
type Options struct {
|
|
Out io.Writer // destination, default os.Stderr
|
|
Enabled bool // whether progress lines are produced at all
|
|
Live bool // render in place with carriage returns
|
|
Interval time.Duration // minimum refresh interval, default 1s
|
|
RowEvery int64 // extra refresh every N rows; 0 disables
|
|
Width int // live line limit, default 100
|
|
Now func() time.Time
|
|
}
|
|
|
|
type sample struct {
|
|
t time.Time
|
|
rows int64
|
|
pos float64
|
|
hasPos bool
|
|
}
|
|
|
|
type data struct {
|
|
started time.Time
|
|
phase string
|
|
reason string
|
|
phaseT time.Time
|
|
base int64
|
|
read int64
|
|
pos float64
|
|
hasPos bool
|
|
originP float64
|
|
originT time.Time
|
|
hasOrig bool
|
|
estimate *Estimate
|
|
shard int64
|
|
hasShard bool
|
|
bytes int64
|
|
hasBytes bool
|
|
shards int64
|
|
hasShardN bool
|
|
samples []sample
|
|
rowMark int64
|
|
lastDraw time.Time
|
|
}
|
|
|
|
func (d *data) rows() int64 { return d.base + d.read }
|
|
|
|
func (d *data) clone() data {
|
|
c := *d
|
|
c.samples = append([]sample(nil), d.samples...)
|
|
return c
|
|
}
|
|
|
|
// Reporter accumulates run metrics and renders them.
|
|
type Reporter struct {
|
|
out io.Writer
|
|
now func() time.Time
|
|
enabled bool
|
|
live bool
|
|
interval time.Duration
|
|
rowEvery int64
|
|
width int
|
|
|
|
mu sync.Mutex
|
|
d data
|
|
started bool
|
|
running bool
|
|
stopCh chan struct{}
|
|
doneCh chan struct{}
|
|
|
|
wmu sync.Mutex
|
|
lastWidth int
|
|
}
|
|
|
|
// New builds a Reporter. A disabled Reporter still carries messages (warnings)
|
|
// to Out but emits no progress lines.
|
|
func New(o Options) *Reporter {
|
|
if o.Out == nil {
|
|
o.Out = os.Stderr
|
|
}
|
|
if o.Now == nil {
|
|
o.Now = time.Now
|
|
}
|
|
if o.Interval <= 0 {
|
|
o.Interval = time.Second
|
|
}
|
|
if o.Width <= 0 {
|
|
o.Width = 100
|
|
}
|
|
return &Reporter{
|
|
out: o.Out,
|
|
now: o.Now,
|
|
enabled: o.Enabled,
|
|
live: o.Live,
|
|
interval: o.Interval,
|
|
rowEvery: o.RowEvery,
|
|
width: o.Width,
|
|
stopCh: make(chan struct{}),
|
|
doneCh: make(chan struct{}),
|
|
}
|
|
}
|
|
|
|
// Start begins the refresh goroutine. It is a no-op for a disabled Reporter.
|
|
func (r *Reporter) Start() {
|
|
r.mu.Lock()
|
|
if r.started {
|
|
r.mu.Unlock()
|
|
return
|
|
}
|
|
r.started = true
|
|
r.initLocked(r.now())
|
|
if !r.enabled {
|
|
r.mu.Unlock()
|
|
return
|
|
}
|
|
r.running = true
|
|
r.mu.Unlock()
|
|
go r.loop()
|
|
}
|
|
|
|
func (r *Reporter) initLocked(now time.Time) {
|
|
r.d.started = now
|
|
r.d.phaseT = now
|
|
r.d.lastDraw = time.Time{}
|
|
if r.rowEvery > 0 {
|
|
r.d.rowMark = r.d.read / r.rowEvery
|
|
}
|
|
r.pushLocked(now)
|
|
}
|
|
|
|
func (r *Reporter) loop() {
|
|
t := time.NewTicker(r.interval)
|
|
defer t.Stop()
|
|
defer close(r.doneCh)
|
|
for {
|
|
select {
|
|
case <-r.stopCh:
|
|
return
|
|
case <-t.C:
|
|
r.maybeRender(false)
|
|
}
|
|
}
|
|
}
|
|
|
|
// SetBase installs the row count carried over from a previous run. Resetting
|
|
// the sample ring keeps the rate measured from this run's work rather than
|
|
// jumping by the inherited count on the first chunk.
|
|
func (r *Reporter) SetBase(n int64) {
|
|
r.mu.Lock()
|
|
r.d.base = n
|
|
r.d.samples = nil
|
|
r.pushLocked(r.now())
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
// AddRows accounts for rows read in this run.
|
|
func (r *Reporter) AddRows(n int64) {
|
|
if n == 0 {
|
|
return
|
|
}
|
|
r.mu.Lock()
|
|
r.d.read += n
|
|
now := r.now()
|
|
r.pushLocked(now)
|
|
crossed := false
|
|
if r.rowEvery > 0 {
|
|
mark := r.d.read / r.rowEvery
|
|
if mark > r.d.rowMark {
|
|
r.d.rowMark = mark
|
|
crossed = true
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
if crossed {
|
|
// The threshold is an extra line at a chunk boundary; draw it inline so
|
|
// it is not lost if the run finishes before the next tick.
|
|
r.maybeRender(true)
|
|
}
|
|
}
|
|
|
|
// SetPosition publishes the current numeric position (day seconds or key).
|
|
func (r *Reporter) SetPosition(pos float64) {
|
|
r.mu.Lock()
|
|
r.d.pos = pos
|
|
r.d.hasPos = true
|
|
if !r.d.hasOrig {
|
|
r.d.originP = pos
|
|
r.d.originT = r.now()
|
|
r.d.hasOrig = true
|
|
}
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
// SetEstimate publishes or clears the range used for percentage and ETA.
|
|
func (r *Reporter) SetEstimate(e *Estimate) {
|
|
r.mu.Lock()
|
|
r.d.estimate = e
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
// SetShard publishes the current shard sequence number.
|
|
func (r *Reporter) SetShard(seq int64) {
|
|
r.mu.Lock()
|
|
r.d.shard = seq
|
|
r.d.hasShard = true
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
// SetBytes publishes bytes written so far.
|
|
func (r *Reporter) SetBytes(n int64) {
|
|
r.mu.Lock()
|
|
r.d.bytes = n
|
|
r.d.hasBytes = true
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
// SetShards publishes the sealed shard count for the done summary.
|
|
func (r *Reporter) SetShards(n int64) {
|
|
r.mu.Lock()
|
|
r.d.shards = n
|
|
r.d.hasShardN = true
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
// Phase records the current phase and an optional waiting reason. Repeating
|
|
// the same phase (for example lag-wait reported on every poll) keeps the phase
|
|
// clock running so its duration is visible.
|
|
func (r *Reporter) Phase(name, reason string) {
|
|
r.mu.Lock()
|
|
if r.d.phase != name {
|
|
r.d.phase = name
|
|
r.d.phaseT = r.now()
|
|
}
|
|
r.d.reason = reason
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
// Message writes a complete diagnostic line, first clearing any live line.
|
|
func (r *Reporter) Message(format string, args ...any) {
|
|
line := strings.TrimRight(fmt.Sprintf(format, args...), "\n")
|
|
r.wmu.Lock()
|
|
defer r.wmu.Unlock()
|
|
if r.live {
|
|
r.clearLocked()
|
|
}
|
|
fmt.Fprintln(r.out, line)
|
|
}
|
|
|
|
// Stop ends refreshing and clears the live line without printing a summary.
|
|
func (r *Reporter) Stop() {
|
|
r.stopTicker()
|
|
r.wmu.Lock()
|
|
r.clearLocked()
|
|
r.wmu.Unlock()
|
|
}
|
|
|
|
// Done prints the final summary line on normal completion.
|
|
func (r *Reporter) Done() {
|
|
r.stopTicker()
|
|
now := r.now()
|
|
r.mu.Lock()
|
|
s := r.d.clone()
|
|
r.mu.Unlock()
|
|
r.wmu.Lock()
|
|
defer r.wmu.Unlock()
|
|
r.clearLocked()
|
|
if r.enabled {
|
|
fmt.Fprintln(r.out, buildDone(s, now))
|
|
}
|
|
}
|
|
|
|
// Interrupted prints the final snapshot line after a signal.
|
|
func (r *Reporter) Interrupted() {
|
|
r.stopTicker()
|
|
now := r.now()
|
|
r.mu.Lock()
|
|
s := r.d.clone()
|
|
r.mu.Unlock()
|
|
r.wmu.Lock()
|
|
defer r.wmu.Unlock()
|
|
r.clearLocked()
|
|
if r.enabled {
|
|
fmt.Fprintln(r.out, buildLine(s, now))
|
|
}
|
|
}
|
|
|
|
func (r *Reporter) stopTicker() {
|
|
r.mu.Lock()
|
|
if !r.running {
|
|
r.mu.Unlock()
|
|
return
|
|
}
|
|
r.running = false
|
|
close(r.stopCh)
|
|
done := r.doneCh
|
|
r.mu.Unlock()
|
|
<-done
|
|
}
|
|
|
|
func (r *Reporter) pushLocked(now time.Time) {
|
|
r.d.samples = append(r.d.samples, sample{
|
|
t: now,
|
|
rows: r.d.rows(),
|
|
pos: r.d.pos,
|
|
hasPos: r.d.hasPos,
|
|
})
|
|
cut := now.Add(-2 * rateWindow)
|
|
i := 0
|
|
for i < len(r.d.samples) && r.d.samples[i].t.Before(cut) {
|
|
i++
|
|
}
|
|
if i > 0 {
|
|
r.d.samples = append(r.d.samples[:0], r.d.samples[i:]...)
|
|
}
|
|
if len(r.d.samples) > 256 {
|
|
r.d.samples = r.d.samples[len(r.d.samples)-256:]
|
|
}
|
|
}
|
|
|
|
// maybeRender draws a frame. force bypasses the interval throttle so a row
|
|
// threshold can produce an extra line.
|
|
func (r *Reporter) maybeRender(force bool) {
|
|
if !r.enabled {
|
|
return
|
|
}
|
|
now := r.now()
|
|
r.mu.Lock()
|
|
if !force && !r.d.lastDraw.IsZero() && now.Sub(r.d.lastDraw) < r.interval {
|
|
r.mu.Unlock()
|
|
return
|
|
}
|
|
r.d.lastDraw = now
|
|
s := r.d.clone()
|
|
r.mu.Unlock()
|
|
|
|
var line string
|
|
if r.live {
|
|
line = r.buildLive(s, now)
|
|
} else {
|
|
line = buildLine(s, now)
|
|
}
|
|
|
|
r.wmu.Lock()
|
|
defer r.wmu.Unlock()
|
|
if r.live {
|
|
if line == "" {
|
|
line = " "
|
|
}
|
|
pad := 0
|
|
if r.lastWidth > len(line) {
|
|
pad = r.lastWidth - len(line)
|
|
}
|
|
fmt.Fprintf(r.out, "\r%s%s", line, strings.Repeat(" ", pad))
|
|
r.lastWidth = len(line)
|
|
return
|
|
}
|
|
fmt.Fprintln(r.out, line)
|
|
}
|
|
|
|
func (r *Reporter) clearLocked() {
|
|
if r.live && r.lastWidth > 0 {
|
|
fmt.Fprintf(r.out, "\r%s\r", strings.Repeat(" ", r.lastWidth))
|
|
r.lastWidth = 0
|
|
}
|
|
}
|
|
|
|
func (r *Reporter) buildLive(s data, now time.Time) string {
|
|
// Field priority when narrowing: drop bytes, then the shard number, then
|
|
// shrink the bar. The line never wraps.
|
|
type shape struct {
|
|
bar int
|
|
bytes bool
|
|
shard bool
|
|
}
|
|
shapes := []shape{
|
|
{20, true, true},
|
|
{20, false, true},
|
|
{20, false, false},
|
|
{12, false, false},
|
|
{8, false, false},
|
|
{0, false, false},
|
|
}
|
|
var last string
|
|
for _, sh := range shapes {
|
|
last = liveLine(s, now, sh.bar, sh.bytes, sh.shard)
|
|
if len(last) <= r.width {
|
|
return last
|
|
}
|
|
}
|
|
return truncate(last, r.width)
|
|
}
|
|
|
|
func liveLine(s data, now time.Time, barWidth int, showBytes, showShard bool) string {
|
|
var parts []string
|
|
pct, eta, hasEta, hasEst := calc(s, now)
|
|
if hasEst {
|
|
if barWidth > 0 {
|
|
filled := int(pct*float64(barWidth) + 0.5)
|
|
parts = append(parts, "["+strings.Repeat("#", filled)+strings.Repeat("-", barWidth-filled)+"]")
|
|
}
|
|
parts = append(parts, fmt.Sprintf("~%.0f%%", pct*100))
|
|
if s.estimate.Kind == "day" {
|
|
cur, total := dayProgress(s)
|
|
parts = append(parts, fmt.Sprintf("day %d/%d", cur, total))
|
|
}
|
|
}
|
|
parts = append(parts, humanRows(s.rows())+" rows")
|
|
if rr, ok := rate(s.samples, now, false); ok && rr > 0 {
|
|
parts = append(parts, humanRate(rr))
|
|
}
|
|
if s.phase != "" {
|
|
parts = append(parts, fmt.Sprintf("%s %s", s.phase, humanDuration(now.Sub(s.phaseT))))
|
|
}
|
|
if s.reason != "" {
|
|
parts = append(parts, "("+s.reason+")")
|
|
}
|
|
if showShard && s.hasShard {
|
|
parts = append(parts, fmt.Sprintf("shard %d", s.shard))
|
|
}
|
|
if showBytes && s.hasBytes {
|
|
parts = append(parts, humanBytes(s.bytes))
|
|
}
|
|
if hasEta {
|
|
parts = append(parts, "eta ~"+humanDuration(eta))
|
|
}
|
|
parts = append(parts, "elapsed "+humanDuration(now.Sub(s.started)))
|
|
return strings.Join(parts, " ")
|
|
}
|
|
|
|
func buildLine(s data, now time.Time) string {
|
|
var b strings.Builder
|
|
b.WriteString("progress: rows=")
|
|
b.WriteString(strconv.FormatInt(s.rows(), 10))
|
|
b.WriteString(" rate=")
|
|
if rr, ok := rate(s.samples, now, false); ok && rr > 0 {
|
|
b.WriteString(humanRate(rr))
|
|
} else {
|
|
b.WriteString(humanRate(cumulativeRows(s, now)))
|
|
}
|
|
b.WriteString(" phase=")
|
|
if s.phase == "" {
|
|
b.WriteString("start")
|
|
} else {
|
|
b.WriteString(s.phase)
|
|
}
|
|
b.WriteString(" phase_elapsed=")
|
|
b.WriteString(humanDuration(now.Sub(s.phaseT)))
|
|
b.WriteString(" elapsed=")
|
|
b.WriteString(humanDuration(now.Sub(s.started)))
|
|
if s.reason != "" {
|
|
b.WriteString(" reason=")
|
|
b.WriteString(strings.Join(strings.Fields(s.reason), "_"))
|
|
}
|
|
if s.hasShard {
|
|
fmt.Fprintf(&b, " shard=%d", s.shard)
|
|
}
|
|
if s.hasBytes {
|
|
fmt.Fprintf(&b, " bytes=%d", s.bytes)
|
|
}
|
|
pct, eta, hasEta, hasEst := calc(s, now)
|
|
if !hasEst {
|
|
b.WriteString(" estimate=none")
|
|
return b.String()
|
|
}
|
|
fmt.Fprintf(&b, " estimate=%s pct=~%.1f", s.estimate.Kind, pct*100)
|
|
if s.estimate.Kind == "day" {
|
|
cur, total := dayProgress(s)
|
|
fmt.Fprintf(&b, " day=%d/%d", cur, total)
|
|
}
|
|
if hasEta {
|
|
b.WriteString(" eta=~")
|
|
b.WriteString(humanDuration(eta))
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func buildDone(s data, now time.Time) string {
|
|
var b strings.Builder
|
|
b.WriteString("done rows=")
|
|
b.WriteString(strconv.FormatInt(s.rows(), 10))
|
|
b.WriteString(" elapsed=")
|
|
b.WriteString(humanDuration(now.Sub(s.started)))
|
|
if s.hasShardN {
|
|
fmt.Fprintf(&b, " shards=%d", s.shards)
|
|
}
|
|
if s.hasBytes {
|
|
fmt.Fprintf(&b, " bytes=%d", s.bytes)
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// calc returns percentage, ETA and whether an estimate is displayable.
|
|
func calc(s data, now time.Time) (pct float64, eta time.Duration, hasEta, ok bool) {
|
|
if s.estimate == nil || !s.hasPos {
|
|
return 0, 0, false, false
|
|
}
|
|
span := s.estimate.Hi - s.estimate.Lo
|
|
if span <= 0 {
|
|
return 0, 0, false, false
|
|
}
|
|
pct = (s.pos - s.estimate.Lo) / span
|
|
if pct < 0 {
|
|
pct = 0
|
|
}
|
|
if pct > 1 {
|
|
pct = 1
|
|
}
|
|
if pr, rok := positionRate(s, now); rok && pr > 0 {
|
|
remaining := s.estimate.Hi - s.pos
|
|
if remaining < 0 {
|
|
remaining = 0
|
|
}
|
|
secs := remaining / pr
|
|
if secs >= 0 && secs < 9e9 {
|
|
eta = time.Duration(secs * float64(time.Second))
|
|
hasEta = true
|
|
}
|
|
}
|
|
return pct, eta, hasEta, true
|
|
}
|
|
|
|
func dayProgress(s data) (int, int) {
|
|
const day = 86400.0
|
|
total := int((s.estimate.Hi-s.estimate.Lo)/day) + 1
|
|
cur := int((s.pos-s.estimate.Lo)/day) + 1
|
|
if cur < 1 {
|
|
cur = 1
|
|
}
|
|
if cur > total {
|
|
cur = total
|
|
}
|
|
return cur, total
|
|
}
|
|
|
|
func rate(samples []sample, now time.Time, usePos bool) (float64, bool) {
|
|
cut := now.Add(-rateWindow)
|
|
var first *sample
|
|
for i := range samples {
|
|
if usePos && !samples[i].hasPos {
|
|
continue
|
|
}
|
|
if !samples[i].t.Before(cut) {
|
|
first = &samples[i]
|
|
break
|
|
}
|
|
}
|
|
if first == nil {
|
|
for i := len(samples) - 1; i >= 0; i-- {
|
|
if !usePos || samples[i].hasPos {
|
|
first = &samples[i]
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if first == nil || len(samples) == 0 {
|
|
return 0, false
|
|
}
|
|
last := &samples[len(samples)-1]
|
|
if usePos && !last.hasPos {
|
|
return 0, false
|
|
}
|
|
if first == last {
|
|
return 0, false
|
|
}
|
|
dt := last.t.Sub(first.t).Seconds()
|
|
if dt <= 0 {
|
|
return 0, false
|
|
}
|
|
var d float64
|
|
if usePos {
|
|
d = last.pos - first.pos
|
|
} else {
|
|
d = float64(last.rows - first.rows)
|
|
}
|
|
if d < 0 {
|
|
return 0, false
|
|
}
|
|
return d / dt, true
|
|
}
|
|
|
|
func positionRate(s data, now time.Time) (float64, bool) {
|
|
if v, ok := rate(s.samples, now, true); ok {
|
|
return v, true
|
|
}
|
|
if s.hasOrig && now.After(s.originT) {
|
|
return (s.pos - s.originP) / now.Sub(s.originT).Seconds(), true
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
func cumulativeRows(s data, now time.Time) float64 {
|
|
el := now.Sub(s.started).Seconds()
|
|
if el <= 0 {
|
|
return 0
|
|
}
|
|
return float64(s.read) / el
|
|
}
|
|
|
|
func humanRows(n int64) string {
|
|
switch {
|
|
case n >= 1_000_000_000:
|
|
return fmt.Sprintf("%.2fG", float64(n)/1e9)
|
|
case n >= 1_000_000:
|
|
return fmt.Sprintf("%.2fM", float64(n)/1e6)
|
|
case n >= 10_000:
|
|
return fmt.Sprintf("%.1fk", float64(n)/1e3)
|
|
default:
|
|
return strconv.FormatInt(n, 10)
|
|
}
|
|
}
|
|
|
|
func humanRate(v float64) string {
|
|
switch {
|
|
case v >= 1_000_000:
|
|
return fmt.Sprintf("%.2fM/s", v/1e6)
|
|
case v >= 10_000:
|
|
return fmt.Sprintf("%.1fk/s", v/1e3)
|
|
case v >= 1:
|
|
return fmt.Sprintf("%.0f/s", v)
|
|
default:
|
|
return "0/s"
|
|
}
|
|
}
|
|
|
|
func humanBytes(n int64) string {
|
|
const unit = 1024
|
|
if n < unit {
|
|
return fmt.Sprintf("%dB", n)
|
|
}
|
|
div, exp := int64(unit), 0
|
|
for v := n / unit; v >= unit; v /= unit {
|
|
div *= unit
|
|
exp++
|
|
}
|
|
return fmt.Sprintf("%.1f%ciB", float64(n)/float64(div), "KMGTPE"[exp])
|
|
}
|
|
|
|
func humanDuration(d time.Duration) string {
|
|
if d < 0 {
|
|
d = 0
|
|
}
|
|
if d < time.Second {
|
|
return fmt.Sprintf("%dms", d.Milliseconds())
|
|
}
|
|
secs := int(d.Seconds())
|
|
switch {
|
|
case secs < 60:
|
|
return fmt.Sprintf("%ds", secs)
|
|
case secs < 3600:
|
|
return fmt.Sprintf("%dm%02ds", secs/60, secs%60)
|
|
default:
|
|
return fmt.Sprintf("%dh%02dm", secs/3600, (secs%3600)/60)
|
|
}
|
|
}
|
|
|
|
func truncate(s string, max int) string {
|
|
if len(s) <= max {
|
|
return s
|
|
}
|
|
if max <= 0 {
|
|
return ""
|
|
}
|
|
runes := []rune(s)
|
|
if len(runes) <= max {
|
|
return s
|
|
}
|
|
return string(runes[:max])
|
|
}
|
|
|
|
// IsTerminal reports whether w is a character device (a terminal).
|
|
func IsTerminal(w io.Writer) bool {
|
|
f, ok := w.(*os.File)
|
|
if !ok {
|
|
return false
|
|
}
|
|
st, err := f.Stat()
|
|
if err != nil {
|
|
return false
|
|
}
|
|
return st.Mode()&os.ModeCharDevice != 0
|
|
}
|