Files
archer/internal/app/app.go
T
voidandClaude Sonnet 4.6 bc4a6b504f Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr,
docs, build scripts and ARCHER_* environment variables.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-11-06 22:12:17 +08:00

706 lines
18 KiB
Go

// Package app wires the CLI, schema probe, scanners, shard writer and
// database targets into one run.
package app
import (
"context"
"crypto/rand"
"database/sql"
"encoding/hex"
"fmt"
"io"
"os"
"strconv"
"strings"
"time"
"github.com/go-sql-driver/mysql"
"archer/internal/archererr"
"archer/internal/checkpoint"
"archer/internal/cli"
"archer/internal/lag"
"archer/internal/progress"
"archer/internal/purge"
"archer/internal/row"
"archer/internal/scan"
"archer/internal/shard"
"archer/internal/sink"
)
// Stats collects counters and phase timings for --statistics.
type Stats struct {
Started time.Time
RowsRead int64
RowsCommitted int64
RowsInserted int64
RowsDeleted int64
ScanDur time.Duration
FileDur time.Duration
DBDur time.Duration
CheckpointDur time.Duration
Shards int64
Bytes int64
}
// App holds a run's state.
type App struct {
cfg *cli.Config
srcDB *sql.DB
destDB *sql.DB
srcCfg *mysql.Config
destCfg *mysql.Config
loc *time.Location
probe *probeResult
cfgHash string
cfgFields map[string]string
scanner scan.Scanner
fileSink *sink.File
writer *shard.Writer
manifest *shard.Manifest
destSink *sink.DB
purger *purge.Purger
lagChecker *lag.Checker
sameInstance bool
resume *scan.Resume
alreadyComplete bool
st *checkpoint.State
stats Stats
reporter *progress.Reporter
estimate *progress.Estimate
baseRows int64
}
// Run executes one archiving run.
func Run(ctx context.Context, cfg *cli.Config) error {
if err := cfg.Validate(); err != nil {
return err
}
a := &App{cfg: cfg, stats: Stats{Started: time.Now()}}
defer a.close()
if !cfg.DryRun {
a.startReporter()
defer a.reporter.Stop()
}
if err := a.setupSource(ctx); err != nil {
return err
}
if cfg.DryRun {
return a.printDryRun(os.Stdout)
}
if err := a.prepare(ctx); err != nil {
return err
}
if a.alreadyComplete {
fmt.Fprintf(os.Stdout, "archer: output in %s is already complete; nothing to do\n", cfg.FileDir)
return nil
}
return a.execute(ctx)
}
func (a *App) startReporter() {
tty := progress.IsTerminal(os.Stderr)
enabled := a.cfg.Progress.Enabled(tty)
a.reporter = progress.New(progress.Options{
Out: os.Stderr,
Enabled: enabled,
Live: enabled && tty,
Interval: a.cfg.ProgressInterval,
RowEvery: a.cfg.Progress.Rows,
})
a.reporter.Start()
}
func (a *App) phase(name, reason string) {
if a.reporter != nil {
a.reporter.Phase(name, reason)
}
}
func (a *App) message(format string, args ...any) {
if a.reporter != nil {
a.reporter.Message(format, args...)
return
}
fmt.Fprintf(os.Stderr, format+"\n", args...)
}
func (a *App) setupSource(ctx context.Context) error {
a.phase("probe", "")
loc, err := time.LoadLocation(a.cfg.ShardTimeZone)
if err != nil {
return archererr.Newf(archererr.KindConfig, "invalid --shard-time-zone: %v", err)
}
a.loc = loc
srcCfg, err := a.cfg.SourceConfig()
if err != nil {
return archererr.Wrap(archererr.KindConfig, err)
}
if srcCfg.DBName == "" {
return archererr.Newf(archererr.KindConfig, "--source must include a database name")
}
a.srcCfg = srcCfg
a.srcDB, err = openDB(srcCfg)
if err != nil {
return err
}
a.probe, err = probeTable(ctx, a.srcDB, a.cfg, srcCfg.DBName, loc)
if err != nil {
return err
}
a.cfgHash, a.cfgFields, err = a.cfg.Fingerprint()
if err != nil {
return archererr.Wrap(archererr.KindConfig, err)
}
return nil
}
func (a *App) prepare(ctx context.Context) error {
a.phase("resume", "")
if err := a.openDest(ctx); err != nil {
return err
}
if a.cfg.Purge {
autoInc := false
if len(a.probe.key.Columns) == 1 {
if c, ok := a.probe.table.Column(a.probe.key.Columns[0]); ok {
autoInc = c.IsAutoIncrement()
}
}
a.purger = purge.New(a.srcDB, a.srcCfg.DBName, a.cfg.Table, a.probe.key.Columns,
a.cfg.Where, a.cfg.BulkDelete, a.cfg.SafeAutoIncrement, autoInc)
}
if a.cfg.CheckReplicaLag != "" {
c, err := lag.Open(a.cfg.CheckReplicaLag, a.cfg.MaxLag, a.cfg.CheckInterval)
if err != nil {
return archererr.Wrap(archererr.KindConnection, err)
}
c.OnWait = func(reason string) {
a.phase("lag-wait", reason)
}
a.lagChecker = c
}
if a.cfg.HasFile() {
if err := a.prepareFile(ctx); err != nil {
return err
}
}
if a.alreadyComplete {
return nil
}
a.phase("estimate", "")
a.estimate = a.queryEstimate(ctx)
if a.reporter != nil {
a.reporter.SetEstimate(a.estimate)
}
return a.openScanner(ctx)
}
func (a *App) openDest(ctx context.Context) error {
if a.cfg.DestDSN == "" {
return nil
}
destCfg, err := a.cfg.DestConfig()
if err != nil {
return archererr.Wrap(archererr.KindConfig, err)
}
if destCfg.DBName == "" {
return archererr.Newf(archererr.KindConfig, "--dest must include a database name")
}
a.destCfg = destCfg
a.sameInstance = cli.SameInstance(a.srcCfg, destCfg)
a.destDB, err = openDB(destCfg)
if err != nil {
return err
}
names := make([]string, len(a.probe.outCols))
for i, c := range a.probe.outCols {
names[i] = c.Name
}
s, err := sink.NewDB(ctx, a.destDB, destCfg.DBName, a.cfg.DestTable, names, a.cfg.InsertMode, a.probe.outIdx)
if err != nil {
return archererr.Wrap(archererr.KindValidation, err)
}
a.destSink = s
return nil
}
func (a *App) prepareFile(ctx context.Context) error {
if err := os.MkdirAll(a.cfg.FileDir, 0o755); err != nil {
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("create output directory: %w", err))
}
comp, err := shard.NewCompressor(a.cfg.Compress, a.cfg.CompressLevel)
if err != nil {
return archererr.Wrap(archererr.KindConfig, err)
}
man, err := shard.OpenManifest(a.cfg.FileDir)
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
a.manifest = man
maxSeq, err := man.Adopt(a.cfg.FileDir, a.cfg.Format, comp)
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
st, err := checkpoint.Load(a.cfg.FileDir)
if err != nil {
return archererr.Wrap(archererr.KindValidation, err)
}
startBucket, startSeq := "", int64(0)
if st != nil {
if st.ConfigHash != a.cfgHash {
if !a.cfg.ForceResume {
diffs := cli.DiffFields(st.Config, a.cfgFields)
if len(diffs) == 0 {
diffs = []string{"fingerprint changed but no field differences were recorded"}
}
return archererr.Newf(archererr.KindValidation,
"checkpoint in %s was created with a different configuration:\n %s\nre-run with --force-resume to ignore this",
a.cfg.FileDir, strings.Join(diffs, "\n "))
}
a.message("warning: --force-resume ignores the changed configuration fingerprint")
}
if st.Complete {
a.alreadyComplete = true
a.st = st
return nil
}
a.st = st
a.baseRows = st.RowsDone
a.resume = &scan.Resume{Phase: st.Cursor.Phase, Key: st.Cursor.Key, DayStart: st.Cursor.DayStart}
startBucket, startSeq = st.LastBucket, st.LastShardSeq
} else {
a.st = &checkpoint.State{RunID: newRunID(), ShardBy: a.cfg.ShardBy}
}
if a.reporter != nil {
a.reporter.SetBase(a.baseRows)
}
if maxSeq[startBucket] > startSeq {
startSeq = maxSeq[startBucket]
}
var keep string
var resumeMeta *shard.PartialMeta
if a.cfg.Purge {
if m, err := shard.LoadPartialMeta(a.cfg.FileDir, startBucket, startSeq+1); err != nil {
return archererr.Wrap(archererr.KindValidation, err)
} else if m != nil {
resumeMeta = m
keep = shard.PartialPath(a.cfg.FileDir, startBucket, startSeq+1)
}
}
if err := shard.CleanupPartials(a.cfg.FileDir, keep); err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
w, err := shard.Open(shard.Options{
Dir: a.cfg.FileDir,
Mode: a.cfg.ShardBy,
ShardRows: int64(a.cfg.ShardRows),
Format: a.cfg.Format,
Compressor: comp,
Columns: a.probe.selectMeta,
OutIdx: a.probe.outIdx,
CSVNull: a.cfg.CSVNull,
StartBucket: startBucket,
StartSeq: startSeq,
}, man)
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
if resumeMeta != nil {
if err := w.Resume(resumeMeta); err != nil {
return archererr.Wrap(archererr.KindValidation, err)
}
}
a.writer = w
a.fileSink = &sink.File{W: w}
return nil
}
func (a *App) openScanner(ctx context.Context) error {
opts := scan.Options{
DB: a.srcDB,
DBName: a.srcCfg.DBName,
Table: a.cfg.Table,
Where: a.cfg.Where,
SelectCols: a.probe.selectCols,
KeyCols: a.probe.key.Columns,
KeyIdx: a.probe.keyIdx,
ChunkSize: a.cfg.ChunkSize,
Shard: a.probe.shard,
ShardIdx: a.probe.shardIdx,
ShardLoc: a.loc,
ZeroDate: a.cfg.ZeroDate,
Resume: a.resume,
}
switch a.probe.path {
case "A":
a.scanner = scan.NewDateA(opts)
case "B":
s, err := scan.NewDateB(ctx, opts)
if err != nil {
return archererr.Wrap(archererr.KindConnection, err)
}
a.scanner = s
default:
a.scanner = scan.NewPK(opts)
}
return nil
}
func (a *App) execute(ctx context.Context) error {
if a.cfg.HasFile() {
a.message("note: file output is at-least-once; a crash and rerun may duplicate rows in shard files")
}
if a.cfg.Purge {
a.message("warning: --purge deletes archived rows from the source table")
}
if a.cfg.BulkDelete {
a.message("warning: --bulk-delete removes all rows in the scanned key range that match --where, including concurrent inserts")
}
for {
if err := ctx.Err(); err != nil {
return a.interrupt(err)
}
if a.lagChecker != nil {
if err := a.lagChecker.Wait(ctx); err != nil {
if ctx.Err() != nil {
return a.interrupt(err)
}
return archererr.Wrap(archererr.KindConnection, err)
}
}
a.phase("scan", "")
start := time.Now()
chunk, err := a.scanner.Next(ctx)
a.stats.ScanDur += time.Since(start)
if err == io.EOF {
break
}
if err != nil {
if ctx.Err() != nil {
return a.interrupt(err)
}
return archererr.Wrap(archererr.KindOther, fmt.Errorf("scan: %w", err))
}
if err := a.processChunk(ctx, chunk); err != nil {
if ctx.Err() != nil {
return a.interrupt(err)
}
return err
}
a.stats.RowsRead += int64(chunk.Len())
a.publishProgress(chunk)
}
return a.finish()
}
func (a *App) processChunk(ctx context.Context, chunk *row.Chunk) error {
fileStart := time.Now()
if a.fileSink != nil {
a.phase("file", "")
if err := a.fileSink.WriteChunk(ctx, chunk); err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
if a.cfg.Purge {
if err := a.fileSink.Checkpoint(); err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
}
}
a.stats.FileDur += time.Since(fileStart)
dbStart := time.Now()
if a.destSink != nil || a.purger != nil {
a.phase("db", "")
}
if err := a.runDB(ctx, chunk); err != nil {
return err
}
a.stats.DBDur += time.Since(dbStart)
if a.fileSink != nil && !a.cfg.Purge {
if err := a.fileSink.Flush(ctx); err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
}
a.stats.RowsCommitted += int64(chunk.Len())
if a.cfg.HasFile() {
a.phase("checkpoint", "")
}
cpStart := time.Now()
if err := a.saveState(false); err != nil {
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("save checkpoint: %w", err))
}
a.stats.CheckpointDur += time.Since(cpStart)
return nil
}
func (a *App) runDB(ctx context.Context, chunk *row.Chunk) error {
attempts := a.cfg.Retries + 1
if a.destSink != nil && a.purger != nil && a.sameInstance {
err := retry(ctx, attempts, func() error {
tx, err := a.destDB.BeginTx(ctx, nil)
if err != nil {
return err
}
n, err := a.destSink.InsertChunk(ctx, tx, chunk)
if err != nil {
_ = tx.Rollback()
return err
}
d, err := a.purger.DeleteChunk(ctx, tx, chunk)
if err != nil {
_ = tx.Rollback()
return err
}
if err := tx.Commit(); err != nil {
return err
}
a.stats.RowsInserted += n
a.stats.RowsDeleted += d
return nil
})
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
return nil
}
if a.destSink != nil {
err := retry(ctx, attempts, func() error {
tx, err := a.destDB.BeginTx(ctx, nil)
if err != nil {
return err
}
n, err := a.destSink.InsertChunk(ctx, tx, chunk)
if err != nil {
_ = tx.Rollback()
return err
}
if err := tx.Commit(); err != nil {
return err
}
a.stats.RowsInserted += n
return nil
})
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
}
if a.purger != nil {
err := retry(ctx, attempts, func() error {
tx, err := a.srcDB.BeginTx(ctx, nil)
if err != nil {
return err
}
d, err := a.purger.DeleteChunk(ctx, tx, chunk)
if err != nil {
_ = tx.Rollback()
return err
}
if err := tx.Commit(); err != nil {
return err
}
a.stats.RowsDeleted += d
return nil
})
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
}
return nil
}
func retry(ctx context.Context, attempts int, fn func() error) error {
var err error
for i := 0; i < attempts; i++ {
if i > 0 {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Duration(i) * time.Second):
}
}
if err = fn(); err == nil {
return nil
}
}
return err
}
func (a *App) saveState(complete bool) error {
if a.cfg.FileDir == "" || a.st == nil || a.alreadyComplete {
return nil
}
st := a.st
st.ConfigHash = a.cfgHash
st.Config = a.cfgFields
st.ShardBy = a.cfg.ShardBy
st.RowsDone = a.baseRows + a.stats.RowsCommitted
if a.scanner != nil {
cs := a.scanner.ResumeState()
st.Cursor = checkpoint.Cursor{Phase: cs.Phase, Key: cs.Key, DayStart: cs.DayStart}
}
if a.writer != nil {
st.LastBucket, st.LastShardSeq = a.writer.StatePosition()
}
st.Complete = complete
return checkpoint.Save(a.cfg.FileDir, st)
}
func (a *App) finish() error {
a.phase("finalize", "")
if a.writer != nil {
if err := a.writer.Close(); err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
var shards, bytes int64
_, shards, bytes = a.writer.Stats()
a.stats.Shards, a.stats.Bytes = shards, bytes
}
if err := a.saveState(true); err != nil {
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("save checkpoint: %w", err))
}
a.printStats()
if a.reporter != nil {
if a.writer != nil {
a.reporter.SetShards(a.stats.Shards)
a.reporter.SetBytes(a.stats.Bytes)
}
a.reporter.Done()
}
return nil
}
func (a *App) interrupt(cause error) error {
if a.reporter != nil {
a.reporter.Interrupted()
}
if a.writer != nil {
if err := a.writer.SealCurrent(); err != nil {
a.message("warning: could not seal current shard: %v", err)
}
}
if err := a.saveState(false); err != nil {
a.message("warning: could not save checkpoint: %v", err)
}
return archererr.Wrap(archererr.KindInterrupted, cause)
}
func (a *App) publishProgress(chunk *row.Chunk) {
if a.reporter == nil {
return
}
if a.estimate != nil {
switch a.estimate.Kind {
case "day":
if len(chunk.Buckets) > 0 {
b := chunk.Buckets[len(chunk.Buckets)-1]
if b != "" && b != "_null" && !strings.HasPrefix(b, "0000-") {
if t, err := time.ParseInLocation("2006-01-02", b, a.loc); err == nil {
a.reporter.SetPosition(float64(t.Unix()))
}
}
}
case "key":
if len(chunk.LastKey) == 1 && !chunk.LastKey[0].Null {
if n, err := strconv.ParseInt(chunk.LastKey[0].Val, 10, 64); err == nil {
a.reporter.SetPosition(float64(n))
}
}
}
}
if a.writer != nil {
a.reporter.SetShard(a.writer.CurrentShard())
a.reporter.SetBytes(a.writer.WrittenBytes())
}
a.reporter.AddRows(int64(chunk.Len()))
}
func (a *App) printStats() {
if !a.cfg.Statistics {
return
}
fmt.Fprintln(os.Stdout, "this run:")
fmt.Fprintf(os.Stdout, " rows read: %d\n", a.stats.RowsRead)
fmt.Fprintf(os.Stdout, " rows committed: %d\n", a.stats.RowsCommitted)
fmt.Fprintf(os.Stdout, " rows inserted: %d\n", a.stats.RowsInserted)
fmt.Fprintf(os.Stdout, " rows deleted: %d\n", a.stats.RowsDeleted)
if a.writer != nil {
fmt.Fprintf(os.Stdout, " shards: %d\n", a.stats.Shards)
fmt.Fprintf(os.Stdout, " bytes written: %d\n", a.stats.Bytes)
}
fmt.Fprintf(os.Stdout, " elapsed: %s (scan %s, file %s, db %s, checkpoint %s)\n",
time.Since(a.stats.Started).Round(time.Millisecond),
a.stats.ScanDur.Round(time.Millisecond), a.stats.FileDur.Round(time.Millisecond),
a.stats.DBDur.Round(time.Millisecond), a.stats.CheckpointDur.Round(time.Millisecond))
fmt.Fprintln(os.Stdout, "cumulative (including prior runs):")
fmt.Fprintf(os.Stdout, " rows read: %d\n", a.baseRows+a.stats.RowsRead)
fmt.Fprintf(os.Stdout, " rows committed: %d\n", a.baseRows+a.stats.RowsCommitted)
if a.writer != nil && a.manifest != nil {
var rows, bytes int64
entries := a.manifest.Entries()
for _, e := range entries {
rows += e.Rows
bytes += e.Bytes
}
fmt.Fprintf(os.Stdout, " shards: %d\n", len(entries))
fmt.Fprintf(os.Stdout, " rows written: %d\n", rows)
fmt.Fprintf(os.Stdout, " bytes written: %d\n", bytes)
}
}
func (a *App) close() {
if a.scanner != nil {
_ = a.scanner.Close()
}
if a.writer != nil {
_ = a.writer.Abandon()
} else if a.manifest != nil {
_ = a.manifest.Close()
}
if a.lagChecker != nil {
_ = a.lagChecker.Close()
}
if a.destDB != nil {
_ = a.destDB.Close()
}
if a.srcDB != nil {
_ = a.srcDB.Close()
}
}
func openDB(cfg *mysql.Config) (*sql.DB, error) {
db, err := sql.Open("mysql", cfg.FormatDSN())
if err != nil {
return nil, archererr.Wrap(archererr.KindConnection, err)
}
db.SetMaxOpenConns(8)
db.SetMaxIdleConns(4)
db.SetConnMaxLifetime(time.Hour)
if err := db.Ping(); err != nil {
db.Close()
return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("connect %s: %w", cfg.Addr, err))
}
return db, nil
}
func newRunID() string {
b := make([]byte, 8)
if _, err := rand.Read(b); err != nil {
return time.Now().UTC().Format("20060102T150405Z")
}
return time.Now().UTC().Format("20060102T150405Z") + "-" + hex.EncodeToString(b)
}