Initial commit: archer, a Go-native MySQL/MariaDB table archiver

Renamed from arcdb: module path, cmd/archer, internal/archererr,
docs, build scripts and ARCHER_* environment variables.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-06 22:12:17 +08:00
commit bc4a6b504f
99 files changed
+12752

No files matched your search

+707
View File
@@ -0,0 +1,707 @@
// Package progress renders human-facing run progress for long archives.
//
// A Reporter receives events from the single-threaded run loop and renders
// them from a background ticker, so blocking phases (a slow scan, fsync, lag
// waiting) still produce a heartbeat. Rendering never touches scanner or
// writer state directly: the run loop copies what it needs into the Reporter.
package progress
import (
"fmt"
"io"
"os"
"strconv"
"strings"
"sync"
"time"
)
// rateWindow is how far back a rate sample may sit to be considered current.
const rateWindow = 5 * time.Second
// Estimate describes the numeric range used for percentage and ETA.
// Kind is "day" (position is a day boundary in unix seconds) or "key"
// (position is a numeric scan key).
type Estimate struct {
Kind string
Lo, Hi float64
}
// Options configures a Reporter.
type Options struct {
Out io.Writer // destination, default os.Stderr
Enabled bool // whether progress lines are produced at all
Live bool // render in place with carriage returns
Interval time.Duration // minimum refresh interval, default 1s
RowEvery int64 // extra refresh every N rows; 0 disables
Width int // live line limit, default 100
Now func() time.Time
}
type sample struct {
t time.Time
rows int64
pos float64
hasPos bool
}
type data struct {
started time.Time
phase string
reason string
phaseT time.Time
base int64
read int64
pos float64
hasPos bool
originP float64
originT time.Time
hasOrig bool
estimate *Estimate
shard int64
hasShard bool
bytes int64
hasBytes bool
shards int64
hasShardN bool
samples []sample
rowMark int64
lastDraw time.Time
}
func (d *data) rows() int64 { return d.base + d.read }
func (d *data) clone() data {
c := *d
c.samples = append([]sample(nil), d.samples...)
return c
}
// Reporter accumulates run metrics and renders them.
type Reporter struct {
out io.Writer
now func() time.Time
enabled bool
live bool
interval time.Duration
rowEvery int64
width int
mu sync.Mutex
d data
started bool
running bool
stopCh chan struct{}
doneCh chan struct{}
wmu sync.Mutex
lastWidth int
}
// New builds a Reporter. A disabled Reporter still carries messages (warnings)
// to Out but emits no progress lines.
func New(o Options) *Reporter {
if o.Out == nil {
o.Out = os.Stderr
}
if o.Now == nil {
o.Now = time.Now
}
if o.Interval <= 0 {
o.Interval = time.Second
}
if o.Width <= 0 {
o.Width = 100
}
return &Reporter{
out: o.Out,
now: o.Now,
enabled: o.Enabled,
live: o.Live,
interval: o.Interval,
rowEvery: o.RowEvery,
width: o.Width,
stopCh: make(chan struct{}),
doneCh: make(chan struct{}),
}
}
// Start begins the refresh goroutine. It is a no-op for a disabled Reporter.
func (r *Reporter) Start() {
r.mu.Lock()
if r.started {
r.mu.Unlock()
return
}
r.started = true
r.initLocked(r.now())
if !r.enabled {
r.mu.Unlock()
return
}
r.running = true
r.mu.Unlock()
go r.loop()
}
func (r *Reporter) initLocked(now time.Time) {
r.d.started = now
r.d.phaseT = now
r.d.lastDraw = time.Time{}
if r.rowEvery > 0 {
r.d.rowMark = r.d.read / r.rowEvery
}
r.pushLocked(now)
}
func (r *Reporter) loop() {
t := time.NewTicker(r.interval)
defer t.Stop()
defer close(r.doneCh)
for {
select {
case <-r.stopCh:
return
case <-t.C:
r.maybeRender(false)
}
}
}
// SetBase installs the row count carried over from a previous run. Resetting
// the sample ring keeps the rate measured from this run's work rather than
// jumping by the inherited count on the first chunk.
func (r *Reporter) SetBase(n int64) {
r.mu.Lock()
r.d.base = n
r.d.samples = nil
r.pushLocked(r.now())
r.mu.Unlock()
}
// AddRows accounts for rows read in this run.
func (r *Reporter) AddRows(n int64) {
if n == 0 {
return
}
r.mu.Lock()
r.d.read += n
now := r.now()
r.pushLocked(now)
crossed := false
if r.rowEvery > 0 {
mark := r.d.read / r.rowEvery
if mark > r.d.rowMark {
r.d.rowMark = mark
crossed = true
}
}
r.mu.Unlock()
if crossed {
// The threshold is an extra line at a chunk boundary; draw it inline so
// it is not lost if the run finishes before the next tick.
r.maybeRender(true)
}
}
// SetPosition publishes the current numeric position (day seconds or key).
func (r *Reporter) SetPosition(pos float64) {
r.mu.Lock()
r.d.pos = pos
r.d.hasPos = true
if !r.d.hasOrig {
r.d.originP = pos
r.d.originT = r.now()
r.d.hasOrig = true
}
r.mu.Unlock()
}
// SetEstimate publishes or clears the range used for percentage and ETA.
func (r *Reporter) SetEstimate(e *Estimate) {
r.mu.Lock()
r.d.estimate = e
r.mu.Unlock()
}
// SetShard publishes the current shard sequence number.
func (r *Reporter) SetShard(seq int64) {
r.mu.Lock()
r.d.shard = seq
r.d.hasShard = true
r.mu.Unlock()
}
// SetBytes publishes bytes written so far.
func (r *Reporter) SetBytes(n int64) {
r.mu.Lock()
r.d.bytes = n
r.d.hasBytes = true
r.mu.Unlock()
}
// SetShards publishes the sealed shard count for the done summary.
func (r *Reporter) SetShards(n int64) {
r.mu.Lock()
r.d.shards = n
r.d.hasShardN = true
r.mu.Unlock()
}
// Phase records the current phase and an optional waiting reason. Repeating
// the same phase (for example lag-wait reported on every poll) keeps the phase
// clock running so its duration is visible.
func (r *Reporter) Phase(name, reason string) {
r.mu.Lock()
if r.d.phase != name {
r.d.phase = name
r.d.phaseT = r.now()
}
r.d.reason = reason
r.mu.Unlock()
}
// Message writes a complete diagnostic line, first clearing any live line.
func (r *Reporter) Message(format string, args ...any) {
line := strings.TrimRight(fmt.Sprintf(format, args...), "\n")
r.wmu.Lock()
defer r.wmu.Unlock()
if r.live {
r.clearLocked()
}
fmt.Fprintln(r.out, line)
}
// Stop ends refreshing and clears the live line without printing a summary.
func (r *Reporter) Stop() {
r.stopTicker()
r.wmu.Lock()
r.clearLocked()
r.wmu.Unlock()
}
// Done prints the final summary line on normal completion.
func (r *Reporter) Done() {
r.stopTicker()
now := r.now()
r.mu.Lock()
s := r.d.clone()
r.mu.Unlock()
r.wmu.Lock()
defer r.wmu.Unlock()
r.clearLocked()
if r.enabled {
fmt.Fprintln(r.out, buildDone(s, now))
}
}
// Interrupted prints the final snapshot line after a signal.
func (r *Reporter) Interrupted() {
r.stopTicker()
now := r.now()
r.mu.Lock()
s := r.d.clone()
r.mu.Unlock()
r.wmu.Lock()
defer r.wmu.Unlock()
r.clearLocked()
if r.enabled {
fmt.Fprintln(r.out, buildLine(s, now))
}
}
func (r *Reporter) stopTicker() {
r.mu.Lock()
if !r.running {
r.mu.Unlock()
return
}
r.running = false
close(r.stopCh)
done := r.doneCh
r.mu.Unlock()
<-done
}
func (r *Reporter) pushLocked(now time.Time) {
r.d.samples = append(r.d.samples, sample{
t: now,
rows: r.d.rows(),
pos: r.d.pos,
hasPos: r.d.hasPos,
})
cut := now.Add(-2 * rateWindow)
i := 0
for i < len(r.d.samples) && r.d.samples[i].t.Before(cut) {
i++
}
if i > 0 {
r.d.samples = append(r.d.samples[:0], r.d.samples[i:]...)
}
if len(r.d.samples) > 256 {
r.d.samples = r.d.samples[len(r.d.samples)-256:]
}
}
// maybeRender draws a frame. force bypasses the interval throttle so a row
// threshold can produce an extra line.
func (r *Reporter) maybeRender(force bool) {
if !r.enabled {
return
}
now := r.now()
r.mu.Lock()
if !force && !r.d.lastDraw.IsZero() && now.Sub(r.d.lastDraw) < r.interval {
r.mu.Unlock()
return
}
r.d.lastDraw = now
s := r.d.clone()
r.mu.Unlock()
var line string
if r.live {
line = r.buildLive(s, now)
} else {
line = buildLine(s, now)
}
r.wmu.Lock()
defer r.wmu.Unlock()
if r.live {
if line == "" {
line = " "
}
pad := 0
if r.lastWidth > len(line) {
pad = r.lastWidth - len(line)
}
fmt.Fprintf(r.out, "\r%s%s", line, strings.Repeat(" ", pad))
r.lastWidth = len(line)
return
}
fmt.Fprintln(r.out, line)
}
func (r *Reporter) clearLocked() {
if r.live && r.lastWidth > 0 {
fmt.Fprintf(r.out, "\r%s\r", strings.Repeat(" ", r.lastWidth))
r.lastWidth = 0
}
}
func (r *Reporter) buildLive(s data, now time.Time) string {
// Field priority when narrowing: drop bytes, then the shard number, then
// shrink the bar. The line never wraps.
type shape struct {
bar int
bytes bool
shard bool
}
shapes := []shape{
{20, true, true},
{20, false, true},
{20, false, false},
{12, false, false},
{8, false, false},
{0, false, false},
}
var last string
for _, sh := range shapes {
last = liveLine(s, now, sh.bar, sh.bytes, sh.shard)
if len(last) <= r.width {
return last
}
}
return truncate(last, r.width)
}
func liveLine(s data, now time.Time, barWidth int, showBytes, showShard bool) string {
var parts []string
pct, eta, hasEta, hasEst := calc(s, now)
if hasEst {
if barWidth > 0 {
filled := int(pct*float64(barWidth) + 0.5)
parts = append(parts, "["+strings.Repeat("#", filled)+strings.Repeat("-", barWidth-filled)+"]")
}
parts = append(parts, fmt.Sprintf("~%.0f%%", pct*100))
if s.estimate.Kind == "day" {
cur, total := dayProgress(s)
parts = append(parts, fmt.Sprintf("day %d/%d", cur, total))
}
}
parts = append(parts, humanRows(s.rows())+" rows")
if rr, ok := rate(s.samples, now, false); ok && rr > 0 {
parts = append(parts, humanRate(rr))
}
if s.phase != "" {
parts = append(parts, fmt.Sprintf("%s %s", s.phase, humanDuration(now.Sub(s.phaseT))))
}
if s.reason != "" {
parts = append(parts, "("+s.reason+")")
}
if showShard && s.hasShard {
parts = append(parts, fmt.Sprintf("shard %d", s.shard))
}
if showBytes && s.hasBytes {
parts = append(parts, humanBytes(s.bytes))
}
if hasEta {
parts = append(parts, "eta ~"+humanDuration(eta))
}
parts = append(parts, "elapsed "+humanDuration(now.Sub(s.started)))
return strings.Join(parts, " ")
}
func buildLine(s data, now time.Time) string {
var b strings.Builder
b.WriteString("progress: rows=")
b.WriteString(strconv.FormatInt(s.rows(), 10))
b.WriteString(" rate=")
if rr, ok := rate(s.samples, now, false); ok && rr > 0 {
b.WriteString(humanRate(rr))
} else {
b.WriteString(humanRate(cumulativeRows(s, now)))
}
b.WriteString(" phase=")
if s.phase == "" {
b.WriteString("start")
} else {
b.WriteString(s.phase)
}
b.WriteString(" phase_elapsed=")
b.WriteString(humanDuration(now.Sub(s.phaseT)))
b.WriteString(" elapsed=")
b.WriteString(humanDuration(now.Sub(s.started)))
if s.reason != "" {
b.WriteString(" reason=")
b.WriteString(strings.Join(strings.Fields(s.reason), "_"))
}
if s.hasShard {
fmt.Fprintf(&b, " shard=%d", s.shard)
}
if s.hasBytes {
fmt.Fprintf(&b, " bytes=%d", s.bytes)
}
pct, eta, hasEta, hasEst := calc(s, now)
if !hasEst {
b.WriteString(" estimate=none")
return b.String()
}
fmt.Fprintf(&b, " estimate=%s pct=~%.1f", s.estimate.Kind, pct*100)
if s.estimate.Kind == "day" {
cur, total := dayProgress(s)
fmt.Fprintf(&b, " day=%d/%d", cur, total)
}
if hasEta {
b.WriteString(" eta=~")
b.WriteString(humanDuration(eta))
}
return b.String()
}
func buildDone(s data, now time.Time) string {
var b strings.Builder
b.WriteString("done rows=")
b.WriteString(strconv.FormatInt(s.rows(), 10))
b.WriteString(" elapsed=")
b.WriteString(humanDuration(now.Sub(s.started)))
if s.hasShardN {
fmt.Fprintf(&b, " shards=%d", s.shards)
}
if s.hasBytes {
fmt.Fprintf(&b, " bytes=%d", s.bytes)
}
return b.String()
}
// calc returns percentage, ETA and whether an estimate is displayable.
func calc(s data, now time.Time) (pct float64, eta time.Duration, hasEta, ok bool) {
if s.estimate == nil || !s.hasPos {
return 0, 0, false, false
}
span := s.estimate.Hi - s.estimate.Lo
if span <= 0 {
return 0, 0, false, false
}
pct = (s.pos - s.estimate.Lo) / span
if pct < 0 {
pct = 0
}
if pct > 1 {
pct = 1
}
if pr, rok := positionRate(s, now); rok && pr > 0 {
remaining := s.estimate.Hi - s.pos
if remaining < 0 {
remaining = 0
}
secs := remaining / pr
if secs >= 0 && secs < 9e9 {
eta = time.Duration(secs * float64(time.Second))
hasEta = true
}
}
return pct, eta, hasEta, true
}
func dayProgress(s data) (int, int) {
const day = 86400.0
total := int((s.estimate.Hi-s.estimate.Lo)/day) + 1
cur := int((s.pos-s.estimate.Lo)/day) + 1
if cur < 1 {
cur = 1
}
if cur > total {
cur = total
}
return cur, total
}
func rate(samples []sample, now time.Time, usePos bool) (float64, bool) {
cut := now.Add(-rateWindow)
var first *sample
for i := range samples {
if usePos && !samples[i].hasPos {
continue
}
if !samples[i].t.Before(cut) {
first = &samples[i]
break
}
}
if first == nil {
for i := len(samples) - 1; i >= 0; i-- {
if !usePos || samples[i].hasPos {
first = &samples[i]
break
}
}
}
if first == nil || len(samples) == 0 {
return 0, false
}
last := &samples[len(samples)-1]
if usePos && !last.hasPos {
return 0, false
}
if first == last {
return 0, false
}
dt := last.t.Sub(first.t).Seconds()
if dt <= 0 {
return 0, false
}
var d float64
if usePos {
d = last.pos - first.pos
} else {
d = float64(last.rows - first.rows)
}
if d < 0 {
return 0, false
}
return d / dt, true
}
func positionRate(s data, now time.Time) (float64, bool) {
if v, ok := rate(s.samples, now, true); ok {
return v, true
}
if s.hasOrig && now.After(s.originT) {
return (s.pos - s.originP) / now.Sub(s.originT).Seconds(), true
}
return 0, false
}
func cumulativeRows(s data, now time.Time) float64 {
el := now.Sub(s.started).Seconds()
if el <= 0 {
return 0
}
return float64(s.read) / el
}
func humanRows(n int64) string {
switch {
case n >= 1_000_000_000:
return fmt.Sprintf("%.2fG", float64(n)/1e9)
case n >= 1_000_000:
return fmt.Sprintf("%.2fM", float64(n)/1e6)
case n >= 10_000:
return fmt.Sprintf("%.1fk", float64(n)/1e3)
default:
return strconv.FormatInt(n, 10)
}
}
func humanRate(v float64) string {
switch {
case v >= 1_000_000:
return fmt.Sprintf("%.2fM/s", v/1e6)
case v >= 10_000:
return fmt.Sprintf("%.1fk/s", v/1e3)
case v >= 1:
return fmt.Sprintf("%.0f/s", v)
default:
return "0/s"
}
}
func humanBytes(n int64) string {
const unit = 1024
if n < unit {
return fmt.Sprintf("%dB", n)
}
div, exp := int64(unit), 0
for v := n / unit; v >= unit; v /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f%ciB", float64(n)/float64(div), "KMGTPE"[exp])
}
func humanDuration(d time.Duration) string {
if d < 0 {
d = 0
}
if d < time.Second {
return fmt.Sprintf("%dms", d.Milliseconds())
}
secs := int(d.Seconds())
switch {
case secs < 60:
return fmt.Sprintf("%ds", secs)
case secs < 3600:
return fmt.Sprintf("%dm%02ds", secs/60, secs%60)
default:
return fmt.Sprintf("%dh%02dm", secs/3600, (secs%3600)/60)
}
}
func truncate(s string, max int) string {
if len(s) <= max {
return s
}
if max <= 0 {
return ""
}
runes := []rune(s)
if len(runes) <= max {
return s
}
return string(runes[:max])
}
// IsTerminal reports whether w is a character device (a terminal).
func IsTerminal(w io.Writer) bool {
f, ok := w.(*os.File)
if !ok {
return false
}
st, err := f.Stat()
if err != nil {
return false
}
return st.Mode()&os.ModeCharDevice != 0
}