Initial commit: archer, a Go-native MySQL/MariaDB table archiver

Renamed from arcdb: module path, cmd/archer, internal/archererr,
docs, build scripts and ARCHER_* environment variables.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-06 22:12:17 +08:00
commit bc4a6b504f
99 files changed
+12752

No files matched your search

+705
View File
@@ -0,0 +1,705 @@
// Package app wires the CLI, schema probe, scanners, shard writer and
// database targets into one run.
package app
import (
"context"
"crypto/rand"
"database/sql"
"encoding/hex"
"fmt"
"io"
"os"
"strconv"
"strings"
"time"
"github.com/go-sql-driver/mysql"
"archer/internal/archererr"
"archer/internal/checkpoint"
"archer/internal/cli"
"archer/internal/lag"
"archer/internal/progress"
"archer/internal/purge"
"archer/internal/row"
"archer/internal/scan"
"archer/internal/shard"
"archer/internal/sink"
)
// Stats collects counters and phase timings for --statistics.
type Stats struct {
Started time.Time
RowsRead int64
RowsCommitted int64
RowsInserted int64
RowsDeleted int64
ScanDur time.Duration
FileDur time.Duration
DBDur time.Duration
CheckpointDur time.Duration
Shards int64
Bytes int64
}
// App holds a run's state.
type App struct {
cfg *cli.Config
srcDB *sql.DB
destDB *sql.DB
srcCfg *mysql.Config
destCfg *mysql.Config
loc *time.Location
probe *probeResult
cfgHash string
cfgFields map[string]string
scanner scan.Scanner
fileSink *sink.File
writer *shard.Writer
manifest *shard.Manifest
destSink *sink.DB
purger *purge.Purger
lagChecker *lag.Checker
sameInstance bool
resume *scan.Resume
alreadyComplete bool
st *checkpoint.State
stats Stats
reporter *progress.Reporter
estimate *progress.Estimate
baseRows int64
}
// Run executes one archiving run.
func Run(ctx context.Context, cfg *cli.Config) error {
if err := cfg.Validate(); err != nil {
return err
}
a := &App{cfg: cfg, stats: Stats{Started: time.Now()}}
defer a.close()
if !cfg.DryRun {
a.startReporter()
defer a.reporter.Stop()
}
if err := a.setupSource(ctx); err != nil {
return err
}
if cfg.DryRun {
return a.printDryRun(os.Stdout)
}
if err := a.prepare(ctx); err != nil {
return err
}
if a.alreadyComplete {
fmt.Fprintf(os.Stdout, "archer: output in %s is already complete; nothing to do\n", cfg.FileDir)
return nil
}
return a.execute(ctx)
}
func (a *App) startReporter() {
tty := progress.IsTerminal(os.Stderr)
enabled := a.cfg.Progress.Enabled(tty)
a.reporter = progress.New(progress.Options{
Out: os.Stderr,
Enabled: enabled,
Live: enabled && tty,
Interval: a.cfg.ProgressInterval,
RowEvery: a.cfg.Progress.Rows,
})
a.reporter.Start()
}
func (a *App) phase(name, reason string) {
if a.reporter != nil {
a.reporter.Phase(name, reason)
}
}
func (a *App) message(format string, args ...any) {
if a.reporter != nil {
a.reporter.Message(format, args...)
return
}
fmt.Fprintf(os.Stderr, format+"\n", args...)
}
func (a *App) setupSource(ctx context.Context) error {
a.phase("probe", "")
loc, err := time.LoadLocation(a.cfg.ShardTimeZone)
if err != nil {
return archererr.Newf(archererr.KindConfig, "invalid --shard-time-zone: %v", err)
}
a.loc = loc
srcCfg, err := a.cfg.SourceConfig()
if err != nil {
return archererr.Wrap(archererr.KindConfig, err)
}
if srcCfg.DBName == "" {
return archererr.Newf(archererr.KindConfig, "--source must include a database name")
}
a.srcCfg = srcCfg
a.srcDB, err = openDB(srcCfg)
if err != nil {
return err
}
a.probe, err = probeTable(ctx, a.srcDB, a.cfg, srcCfg.DBName, loc)
if err != nil {
return err
}
a.cfgHash, a.cfgFields, err = a.cfg.Fingerprint()
if err != nil {
return archererr.Wrap(archererr.KindConfig, err)
}
return nil
}
func (a *App) prepare(ctx context.Context) error {
a.phase("resume", "")
if err := a.openDest(ctx); err != nil {
return err
}
if a.cfg.Purge {
autoInc := false
if len(a.probe.key.Columns) == 1 {
if c, ok := a.probe.table.Column(a.probe.key.Columns[0]); ok {
autoInc = c.IsAutoIncrement()
}
}
a.purger = purge.New(a.srcDB, a.srcCfg.DBName, a.cfg.Table, a.probe.key.Columns,
a.cfg.Where, a.cfg.BulkDelete, a.cfg.SafeAutoIncrement, autoInc)
}
if a.cfg.CheckReplicaLag != "" {
c, err := lag.Open(a.cfg.CheckReplicaLag, a.cfg.MaxLag, a.cfg.CheckInterval)
if err != nil {
return archererr.Wrap(archererr.KindConnection, err)
}
c.OnWait = func(reason string) {
a.phase("lag-wait", reason)
}
a.lagChecker = c
}
if a.cfg.HasFile() {
if err := a.prepareFile(ctx); err != nil {
return err
}
}
if a.alreadyComplete {
return nil
}
a.phase("estimate", "")
a.estimate = a.queryEstimate(ctx)
if a.reporter != nil {
a.reporter.SetEstimate(a.estimate)
}
return a.openScanner(ctx)
}
func (a *App) openDest(ctx context.Context) error {
if a.cfg.DestDSN == "" {
return nil
}
destCfg, err := a.cfg.DestConfig()
if err != nil {
return archererr.Wrap(archererr.KindConfig, err)
}
if destCfg.DBName == "" {
return archererr.Newf(archererr.KindConfig, "--dest must include a database name")
}
a.destCfg = destCfg
a.sameInstance = cli.SameInstance(a.srcCfg, destCfg)
a.destDB, err = openDB(destCfg)
if err != nil {
return err
}
names := make([]string, len(a.probe.outCols))
for i, c := range a.probe.outCols {
names[i] = c.Name
}
s, err := sink.NewDB(ctx, a.destDB, destCfg.DBName, a.cfg.DestTable, names, a.cfg.InsertMode, a.probe.outIdx)
if err != nil {
return archererr.Wrap(archererr.KindValidation, err)
}
a.destSink = s
return nil
}
func (a *App) prepareFile(ctx context.Context) error {
if err := os.MkdirAll(a.cfg.FileDir, 0o755); err != nil {
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("create output directory: %w", err))
}
comp, err := shard.NewCompressor(a.cfg.Compress, a.cfg.CompressLevel)
if err != nil {
return archererr.Wrap(archererr.KindConfig, err)
}
man, err := shard.OpenManifest(a.cfg.FileDir)
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
a.manifest = man
maxSeq, err := man.Adopt(a.cfg.FileDir, a.cfg.Format, comp)
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
st, err := checkpoint.Load(a.cfg.FileDir)
if err != nil {
return archererr.Wrap(archererr.KindValidation, err)
}
startBucket, startSeq := "", int64(0)
if st != nil {
if st.ConfigHash != a.cfgHash {
if !a.cfg.ForceResume {
diffs := cli.DiffFields(st.Config, a.cfgFields)
if len(diffs) == 0 {
diffs = []string{"fingerprint changed but no field differences were recorded"}
}
return archererr.Newf(archererr.KindValidation,
"checkpoint in %s was created with a different configuration:\n %s\nre-run with --force-resume to ignore this",
a.cfg.FileDir, strings.Join(diffs, "\n "))
}
a.message("warning: --force-resume ignores the changed configuration fingerprint")
}
if st.Complete {
a.alreadyComplete = true
a.st = st
return nil
}
a.st = st
a.baseRows = st.RowsDone
a.resume = &scan.Resume{Phase: st.Cursor.Phase, Key: st.Cursor.Key, DayStart: st.Cursor.DayStart}
startBucket, startSeq = st.LastBucket, st.LastShardSeq
} else {
a.st = &checkpoint.State{RunID: newRunID(), ShardBy: a.cfg.ShardBy}
}
if a.reporter != nil {
a.reporter.SetBase(a.baseRows)
}
if maxSeq[startBucket] > startSeq {
startSeq = maxSeq[startBucket]
}
var keep string
var resumeMeta *shard.PartialMeta
if a.cfg.Purge {
if m, err := shard.LoadPartialMeta(a.cfg.FileDir, startBucket, startSeq+1); err != nil {
return archererr.Wrap(archererr.KindValidation, err)
} else if m != nil {
resumeMeta = m
keep = shard.PartialPath(a.cfg.FileDir, startBucket, startSeq+1)
}
}
if err := shard.CleanupPartials(a.cfg.FileDir, keep); err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
w, err := shard.Open(shard.Options{
Dir: a.cfg.FileDir,
Mode: a.cfg.ShardBy,
ShardRows: int64(a.cfg.ShardRows),
Format: a.cfg.Format,
Compressor: comp,
Columns: a.probe.selectMeta,
OutIdx: a.probe.outIdx,
CSVNull: a.cfg.CSVNull,
StartBucket: startBucket,
StartSeq: startSeq,
}, man)
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
if resumeMeta != nil {
if err := w.Resume(resumeMeta); err != nil {
return archererr.Wrap(archererr.KindValidation, err)
}
}
a.writer = w
a.fileSink = &sink.File{W: w}
return nil
}
func (a *App) openScanner(ctx context.Context) error {
opts := scan.Options{
DB: a.srcDB,
DBName: a.srcCfg.DBName,
Table: a.cfg.Table,
Where: a.cfg.Where,
SelectCols: a.probe.selectCols,
KeyCols: a.probe.key.Columns,
KeyIdx: a.probe.keyIdx,
ChunkSize: a.cfg.ChunkSize,
Shard: a.probe.shard,
ShardIdx: a.probe.shardIdx,
ShardLoc: a.loc,
ZeroDate: a.cfg.ZeroDate,
Resume: a.resume,
}
switch a.probe.path {
case "A":
a.scanner = scan.NewDateA(opts)
case "B":
s, err := scan.NewDateB(ctx, opts)
if err != nil {
return archererr.Wrap(archererr.KindConnection, err)
}
a.scanner = s
default:
a.scanner = scan.NewPK(opts)
}
return nil
}
func (a *App) execute(ctx context.Context) error {
if a.cfg.HasFile() {
a.message("note: file output is at-least-once; a crash and rerun may duplicate rows in shard files")
}
if a.cfg.Purge {
a.message("warning: --purge deletes archived rows from the source table")
}
if a.cfg.BulkDelete {
a.message("warning: --bulk-delete removes all rows in the scanned key range that match --where, including concurrent inserts")
}
for {
if err := ctx.Err(); err != nil {
return a.interrupt(err)
}
if a.lagChecker != nil {
if err := a.lagChecker.Wait(ctx); err != nil {
if ctx.Err() != nil {
return a.interrupt(err)
}
return archererr.Wrap(archererr.KindConnection, err)
}
}
a.phase("scan", "")
start := time.Now()
chunk, err := a.scanner.Next(ctx)
a.stats.ScanDur += time.Since(start)
if err == io.EOF {
break
}
if err != nil {
if ctx.Err() != nil {
return a.interrupt(err)
}
return archererr.Wrap(archererr.KindOther, fmt.Errorf("scan: %w", err))
}
if err := a.processChunk(ctx, chunk); err != nil {
if ctx.Err() != nil {
return a.interrupt(err)
}
return err
}
a.stats.RowsRead += int64(chunk.Len())
a.publishProgress(chunk)
}
return a.finish()
}
func (a *App) processChunk(ctx context.Context, chunk *row.Chunk) error {
fileStart := time.Now()
if a.fileSink != nil {
a.phase("file", "")
if err := a.fileSink.WriteChunk(ctx, chunk); err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
if a.cfg.Purge {
if err := a.fileSink.Checkpoint(); err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
}
}
a.stats.FileDur += time.Since(fileStart)
dbStart := time.Now()
if a.destSink != nil || a.purger != nil {
a.phase("db", "")
}
if err := a.runDB(ctx, chunk); err != nil {
return err
}
a.stats.DBDur += time.Since(dbStart)
if a.fileSink != nil && !a.cfg.Purge {
if err := a.fileSink.Flush(ctx); err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
}
a.stats.RowsCommitted += int64(chunk.Len())
if a.cfg.HasFile() {
a.phase("checkpoint", "")
}
cpStart := time.Now()
if err := a.saveState(false); err != nil {
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("save checkpoint: %w", err))
}
a.stats.CheckpointDur += time.Since(cpStart)
return nil
}
func (a *App) runDB(ctx context.Context, chunk *row.Chunk) error {
attempts := a.cfg.Retries + 1
if a.destSink != nil && a.purger != nil && a.sameInstance {
err := retry(ctx, attempts, func() error {
tx, err := a.destDB.BeginTx(ctx, nil)
if err != nil {
return err
}
n, err := a.destSink.InsertChunk(ctx, tx, chunk)
if err != nil {
_ = tx.Rollback()
return err
}
d, err := a.purger.DeleteChunk(ctx, tx, chunk)
if err != nil {
_ = tx.Rollback()
return err
}
if err := tx.Commit(); err != nil {
return err
}
a.stats.RowsInserted += n
a.stats.RowsDeleted += d
return nil
})
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
return nil
}
if a.destSink != nil {
err := retry(ctx, attempts, func() error {
tx, err := a.destDB.BeginTx(ctx, nil)
if err != nil {
return err
}
n, err := a.destSink.InsertChunk(ctx, tx, chunk)
if err != nil {
_ = tx.Rollback()
return err
}
if err := tx.Commit(); err != nil {
return err
}
a.stats.RowsInserted += n
return nil
})
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
}
if a.purger != nil {
err := retry(ctx, attempts, func() error {
tx, err := a.srcDB.BeginTx(ctx, nil)
if err != nil {
return err
}
d, err := a.purger.DeleteChunk(ctx, tx, chunk)
if err != nil {
_ = tx.Rollback()
return err
}
if err := tx.Commit(); err != nil {
return err
}
a.stats.RowsDeleted += d
return nil
})
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
}
return nil
}
func retry(ctx context.Context, attempts int, fn func() error) error {
var err error
for i := 0; i < attempts; i++ {
if i > 0 {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Duration(i) * time.Second):
}
}
if err = fn(); err == nil {
return nil
}
}
return err
}
func (a *App) saveState(complete bool) error {
if a.cfg.FileDir == "" || a.st == nil || a.alreadyComplete {
return nil
}
st := a.st
st.ConfigHash = a.cfgHash
st.Config = a.cfgFields
st.ShardBy = a.cfg.ShardBy
st.RowsDone = a.baseRows + a.stats.RowsCommitted
if a.scanner != nil {
cs := a.scanner.ResumeState()
st.Cursor = checkpoint.Cursor{Phase: cs.Phase, Key: cs.Key, DayStart: cs.DayStart}
}
if a.writer != nil {
st.LastBucket, st.LastShardSeq = a.writer.StatePosition()
}
st.Complete = complete
return checkpoint.Save(a.cfg.FileDir, st)
}
func (a *App) finish() error {
a.phase("finalize", "")
if a.writer != nil {
if err := a.writer.Close(); err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
var shards, bytes int64
_, shards, bytes = a.writer.Stats()
a.stats.Shards, a.stats.Bytes = shards, bytes
}
if err := a.saveState(true); err != nil {
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("save checkpoint: %w", err))
}
a.printStats()
if a.reporter != nil {
if a.writer != nil {
a.reporter.SetShards(a.stats.Shards)
a.reporter.SetBytes(a.stats.Bytes)
}
a.reporter.Done()
}
return nil
}
func (a *App) interrupt(cause error) error {
if a.reporter != nil {
a.reporter.Interrupted()
}
if a.writer != nil {
if err := a.writer.SealCurrent(); err != nil {
a.message("warning: could not seal current shard: %v", err)
}
}
if err := a.saveState(false); err != nil {
a.message("warning: could not save checkpoint: %v", err)
}
return archererr.Wrap(archererr.KindInterrupted, cause)
}
func (a *App) publishProgress(chunk *row.Chunk) {
if a.reporter == nil {
return
}
if a.estimate != nil {
switch a.estimate.Kind {
case "day":
if len(chunk.Buckets) > 0 {
b := chunk.Buckets[len(chunk.Buckets)-1]
if b != "" && b != "_null" && !strings.HasPrefix(b, "0000-") {
if t, err := time.ParseInLocation("2006-01-02", b, a.loc); err == nil {
a.reporter.SetPosition(float64(t.Unix()))
}
}
}
case "key":
if len(chunk.LastKey) == 1 && !chunk.LastKey[0].Null {
if n, err := strconv.ParseInt(chunk.LastKey[0].Val, 10, 64); err == nil {
a.reporter.SetPosition(float64(n))
}
}
}
}
if a.writer != nil {
a.reporter.SetShard(a.writer.CurrentShard())
a.reporter.SetBytes(a.writer.WrittenBytes())
}
a.reporter.AddRows(int64(chunk.Len()))
}
func (a *App) printStats() {
if !a.cfg.Statistics {
return
}
fmt.Fprintln(os.Stdout, "this run:")
fmt.Fprintf(os.Stdout, " rows read: %d\n", a.stats.RowsRead)
fmt.Fprintf(os.Stdout, " rows committed: %d\n", a.stats.RowsCommitted)
fmt.Fprintf(os.Stdout, " rows inserted: %d\n", a.stats.RowsInserted)
fmt.Fprintf(os.Stdout, " rows deleted: %d\n", a.stats.RowsDeleted)
if a.writer != nil {
fmt.Fprintf(os.Stdout, " shards: %d\n", a.stats.Shards)
fmt.Fprintf(os.Stdout, " bytes written: %d\n", a.stats.Bytes)
}
fmt.Fprintf(os.Stdout, " elapsed: %s (scan %s, file %s, db %s, checkpoint %s)\n",
time.Since(a.stats.Started).Round(time.Millisecond),
a.stats.ScanDur.Round(time.Millisecond), a.stats.FileDur.Round(time.Millisecond),
a.stats.DBDur.Round(time.Millisecond), a.stats.CheckpointDur.Round(time.Millisecond))
fmt.Fprintln(os.Stdout, "cumulative (including prior runs):")
fmt.Fprintf(os.Stdout, " rows read: %d\n", a.baseRows+a.stats.RowsRead)
fmt.Fprintf(os.Stdout, " rows committed: %d\n", a.baseRows+a.stats.RowsCommitted)
if a.writer != nil && a.manifest != nil {
var rows, bytes int64
entries := a.manifest.Entries()
for _, e := range entries {
rows += e.Rows
bytes += e.Bytes
}
fmt.Fprintf(os.Stdout, " shards: %d\n", len(entries))
fmt.Fprintf(os.Stdout, " rows written: %d\n", rows)
fmt.Fprintf(os.Stdout, " bytes written: %d\n", bytes)
}
}
func (a *App) close() {
if a.scanner != nil {
_ = a.scanner.Close()
}
if a.writer != nil {
_ = a.writer.Abandon()
} else if a.manifest != nil {
_ = a.manifest.Close()
}
if a.lagChecker != nil {
_ = a.lagChecker.Close()
}
if a.destDB != nil {
_ = a.destDB.Close()
}
if a.srcDB != nil {
_ = a.srcDB.Close()
}
}
func openDB(cfg *mysql.Config) (*sql.DB, error) {
db, err := sql.Open("mysql", cfg.FormatDSN())
if err != nil {
return nil, archererr.Wrap(archererr.KindConnection, err)
}
db.SetMaxOpenConns(8)
db.SetMaxIdleConns(4)
db.SetConnMaxLifetime(time.Hour)
if err := db.Ping(); err != nil {
db.Close()
return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("connect %s: %w", cfg.Addr, err))
}
return db, nil
}
func newRunID() string {
b := make([]byte, 8)
if _, err := rand.Read(b); err != nil {
return time.Now().UTC().Format("20060102T150405Z")
}
return time.Now().UTC().Format("20060102T150405Z") + "-" + hex.EncodeToString(b)
}
+162
View File
@@ -0,0 +1,162 @@
package app
import (
"fmt"
"io"
"strings"
"github.com/go-sql-driver/mysql"
"archer/internal/purge"
"archer/internal/shard"
"archer/internal/sqlutil"
)
func (a *App) printDryRun(w io.Writer) error {
cfg := a.cfg
comp, err := shard.NewCompressor(cfg.Compress, cfg.CompressLevel)
if err != nil {
return err
}
fmt.Fprintln(w, "== resolved configuration ==")
fmt.Fprintf(w, "source: %s\n", redactDSN(cfg.SourceDSN))
if cfg.DestDSN != "" {
fmt.Fprintf(w, "dest: %s\n", redactDSN(cfg.DestDSN))
}
fmt.Fprintf(w, "table: %s\n", cfg.Table)
if cfg.DestDSN != "" {
fmt.Fprintf(w, "dest table: %s\n", cfg.DestTable)
}
fmt.Fprintf(w, "where: %s\n", cfg.Where)
if len(cfg.Columns) > 0 {
fmt.Fprintf(w, "columns: %s\n", strings.Join(cfg.Columns, ", "))
} else {
fmt.Fprintf(w, "columns: (all)\n")
}
targets := []string{}
if cfg.HasFile() {
targets = append(targets, "file="+cfg.FileDir)
}
if cfg.DestDSN != "" {
targets = append(targets, "dest")
}
if cfg.Purge {
targets = append(targets, "purge")
}
fmt.Fprintf(w, "targets: %s\n", strings.Join(targets, ", "))
fmt.Fprintf(w, "chunk size: %d\n", cfg.ChunkSize)
fmt.Fprintf(w, "shard by: %s\n", cfg.ShardBy)
if cfg.ShardBy != "none" {
fmt.Fprintf(w, "shard rows: %d\n", cfg.ShardRows)
}
if a.probe.shard != nil {
fmt.Fprintf(w, "shard column: %s (%s)\n", a.probe.shard.Column, a.probe.shard.Type)
fmt.Fprintf(w, "shard time zone: %s\n", cfg.ShardTimeZone)
}
fmt.Fprintf(w, "format: %s\n", cfg.Format)
if comp.Name == "none" {
fmt.Fprintf(w, "compress: none\n")
} else {
fmt.Fprintf(w, "compress: %s (level %d)\n", comp.Name, comp.Level)
}
if cfg.InsertMode != "" {
fmt.Fprintf(w, "insert mode: %s\n", cfg.InsertMode)
}
fmt.Fprintf(w, "max lag: %s (check interval %s)\n", cfg.MaxLag, cfg.CheckInterval)
fmt.Fprintf(w, "retries: %d\n", cfg.Retries)
fmt.Fprintf(w, "safe auto inc: %t\n", cfg.SafeAutoIncrement)
fmt.Fprintf(w, "config hash: %s\n", a.cfgHash)
fmt.Fprintln(w, "\n== index probe ==")
fmt.Fprintf(w, "scan key: %s (%s)\n", a.probe.key.Name, strings.Join(a.probe.key.Columns, ", "))
if a.probe.path == "A" {
fmt.Fprintf(w, "scan path: A (keyset chunks using index %s)\n", a.probe.joint.Name)
} else if a.probe.path == "B" {
if a.probe.single != nil {
fmt.Fprintf(w, "scan path: B (day-at-a-time using index %s)\n", a.probe.single.Name)
} else {
fmt.Fprintln(w, "scan path: B (day-at-a-time, full table scan allowed)")
}
} else {
fmt.Fprintln(w, "scan path: primary key seek")
}
fmt.Fprintln(w, "\n== first SELECT ==")
for _, q := range a.dryRunSelects() {
fmt.Fprintf(w, " %s\n", q)
}
fmt.Fprintln(w, "\n== write statements ==")
if cfg.DestDSN != "" {
fmt.Fprintf(w, " %s\n", a.dryRunInsert())
}
if cfg.Purge {
p := purge.New(nil, a.srcCfg.DBName, cfg.Table, a.probe.key.Columns, cfg.Where, cfg.BulkDelete, cfg.SafeAutoIncrement, false)
for _, q := range p.DryRunSQL() {
fmt.Fprintf(w, " %s\n", q)
}
}
if cfg.HasFile() {
fmt.Fprintf(w, " file: %s/[...]<seq>.<pk_lo>-<pk_hi>.%s%s (written as .partial until sealed)\n",
cfg.FileDir, cfg.Format, comp.Ext)
}
fmt.Fprintln(w, "\nnote: file output is at-least-once; a crash and rerun may duplicate rows in shard files")
return nil
}
func (a *App) dryRunSelects() []string {
cols := sqlutil.Columns(a.probe.selectCols)
table := sqlutil.Qualified(a.srcCfg.DBName, a.cfg.Table)
where := a.cfg.Where
key := sqlutil.Columns(a.probe.key.Columns)
limit := fmt.Sprintf(" LIMIT %d", a.cfg.ChunkSize)
switch a.probe.path {
case "A":
shard := sqlutil.Ident(a.probe.shard.Column)
colsWithShard := sqlutil.Columns(append([]string{a.probe.shard.Column}, a.probe.key.Columns...))
return []string{
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s IS NULL ORDER BY %s%s", cols, table, where, shard, key, limit),
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s IS NOT NULL AND (%s) > (?, ...) ORDER BY %s%s", cols, table, where, shard, colsWithShard, colsWithShard, limit),
}
case "B":
shard := sqlutil.Ident(a.probe.shard.Column)
colsWithShard := sqlutil.Columns(append([]string{a.probe.shard.Column}, a.probe.key.Columns...))
return []string{
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s IS NULL ORDER BY %s", cols, table, where, shard, key),
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s >= ? AND %s < ? ORDER BY %s", cols, table, where, shard, shard, colsWithShard),
}
default:
return []string{fmt.Sprintf("SELECT %s FROM %s WHERE (%s) ORDER BY %s%s", cols, table, where, key, limit)}
}
}
func (a *App) dryRunInsert() string {
destCfg, err := a.cfg.DestConfig()
dbName := ""
if err == nil && destCfg != nil {
dbName = destCfg.DBName
}
verb := "INSERT INTO"
switch a.cfg.InsertMode {
case "ignore":
verb = "INSERT IGNORE INTO"
case "replace":
verb = "REPLACE INTO"
}
names := make([]string, len(a.probe.outCols))
for i, c := range a.probe.outCols {
names[i] = c.Name
}
tuple := sqlutil.ParamTuple(len(names))
return fmt.Sprintf("%s %s (%s) VALUES %s, %s, ...",
verb, sqlutil.Qualified(dbName, a.cfg.DestTable), sqlutil.Columns(names), tuple, tuple)
}
func redactDSN(dsn string) string {
c, err := mysql.ParseDSN(dsn)
if err != nil {
return "<invalid dsn>"
}
c.Passwd = ""
return c.FormatDSN()
}
+64
View File
@@ -0,0 +1,64 @@
package app
import (
"context"
"database/sql"
"fmt"
"time"
"archer/internal/estimate"
"archer/internal/progress"
"archer/internal/sqlutil"
)
// estimateTimeout bounds the one-off MIN/MAX query at startup.
const estimateTimeout = 2 * time.Second
// estimateSpec decides which cheap position metric this run can expose.
func (a *App) estimateSpec() estimate.Spec {
if a.cfg.IsDateShard() && a.probe.shard != nil {
return estimate.Spec{
Kind: "day",
Column: a.probe.shard.Column,
ShardKind: a.probe.shard.Kind,
Unit: a.probe.shard.Unit,
Loc: a.loc,
}
}
types := make([]string, len(a.probe.key.Columns))
for i, name := range a.probe.key.Columns {
c, _ := a.probe.table.Column(name)
types[i] = c.DataType
}
col, ok := estimate.KeyColumn(a.probe.key.Columns, types)
if !ok {
return estimate.Spec{}
}
return estimate.Spec{Kind: "key", Column: col}
}
// queryEstimate runs the bounded range query on a dedicated connection and
// silently returns nil when the metric does not apply or the query fails.
func (a *App) queryEstimate(ctx context.Context) *progress.Estimate {
spec := a.estimateSpec()
if spec.Kind == "" {
return nil
}
conn, err := a.srcDB.Conn(ctx)
if err != nil {
return nil
}
defer conn.Close()
query := func(qctx context.Context, column string) (sql.NullString, sql.NullString, error) {
col := sqlutil.Ident(column)
q := fmt.Sprintf("SELECT MIN(%s), MAX(%s) FROM %s WHERE (%s)",
col, col, sqlutil.Qualified(a.srcCfg.DBName, a.cfg.Table), a.cfg.Where)
if spec.Kind == "day" {
q += " AND " + col + " IS NOT NULL"
}
var minRaw, maxRaw sql.NullString
err := conn.QueryRowContext(qctx, q).Scan(&minRaw, &maxRaw)
return minRaw, maxRaw, err
}
return estimate.Bounded(ctx, query, spec, estimateTimeout)
}
+918
View File
@@ -0,0 +1,918 @@
package app
import (
"bytes"
"compress/gzip"
"context"
"database/sql"
"encoding/csv"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"testing"
"github.com/go-sql-driver/mysql"
"github.com/klauspost/compress/zstd"
"github.com/ulikunitz/xz"
"archer/internal/cli"
)
func testDSN(t *testing.T) string {
d := os.Getenv("ARCHER_TEST_DSN")
if d == "" {
t.Skip("ARCHER_TEST_DSN is not set; skipping MySQL integration test")
}
return d
}
func TestMain(m *testing.M) {
if dsn := os.Getenv("ARCHER_TEST_DSN"); dsn != "" {
cfg, err := mysql.ParseDSN(dsn)
if err == nil && cfg.DBName != "" {
dbName := cfg.DBName
cfg.DBName = ""
db, err := sql.Open("mysql", cfg.FormatDSN())
if err == nil {
_, _ = db.Exec("CREATE DATABASE IF NOT EXISTS `" + dbName + "`")
_, _ = db.Exec("SET GLOBAL local_infile=ON")
_, _ = db.Exec("SET GLOBAL sql_mode=''")
db.Close()
}
}
}
os.Exit(m.Run())
}
func openRoot(t *testing.T, dsn string) *sql.DB {
t.Helper()
cfg, err := mysql.ParseDSN(dsn)
if err != nil {
t.Fatal(err)
}
db, err := sql.Open("mysql", cfg.FormatDSN())
if err != nil {
t.Fatal(err)
}
if err := db.Ping(); err != nil {
t.Fatal(err)
}
return db
}
func execSQL(t *testing.T, db *sql.DB, q string, args ...any) {
t.Helper()
if _, err := db.Exec(q, args...); err != nil {
t.Fatalf("exec %q: %v", q, err)
}
}
var stdoutMu sync.Mutex
var stderrMu sync.Mutex
func captureStdout(fn func() error) (string, error) {
stdoutMu.Lock()
defer stdoutMu.Unlock()
old := os.Stdout
r, w, err := os.Pipe()
if err != nil {
return "", err
}
os.Stdout = w
var buf bytes.Buffer
done := make(chan struct{})
go func() {
_, _ = io.Copy(&buf, r)
close(done)
}()
runErr := fn()
_ = w.Close()
os.Stdout = old
<-done
return buf.String(), runErr
}
func captureStderr(fn func() error) (string, error) {
stderrMu.Lock()
defer stderrMu.Unlock()
old := os.Stderr
r, w, err := os.Pipe()
if err != nil {
return "", err
}
os.Stderr = w
var buf bytes.Buffer
done := make(chan struct{})
go func() {
_, _ = io.Copy(&buf, r)
close(done)
}()
runErr := fn()
_ = w.Close()
os.Stderr = old
<-done
return buf.String(), runErr
}
func runCLI(t *testing.T, ctx context.Context, args ...string) (string, error) {
t.Helper()
cfg, err := cli.Parse(args, io.Discard)
if err != nil {
return "", err
}
return captureStdout(func() error { return Run(ctx, cfg) })
}
func runCLIStderr(t *testing.T, ctx context.Context, args ...string) (string, error) {
t.Helper()
cfg, err := cli.Parse(args, io.Discard)
if err != nil {
return "", err
}
return captureStderr(func() error { return Run(ctx, cfg) })
}
func baseCLI(dsn, table, out string, extra ...string) []string {
args := []string{
"--source", dsn,
"--table", table,
"--where", "1=1",
"--file", out,
}
return append(args, extra...)
}
func listFiles(t *testing.T, dir, pattern string) []string {
t.Helper()
matches, err := filepath.Glob(filepath.Join(dir, pattern))
if err != nil {
t.Fatal(err)
}
sort.Strings(matches)
return matches
}
func dumpIDs(t *testing.T, path string) []int {
t.Helper()
b, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var ids []int
for _, line := range strings.Split(strings.TrimRight(string(b), "\n"), "\n") {
if line == "" {
continue
}
id, err := strconv.Atoi(strings.SplitN(line, "\t", 2)[0])
if err != nil {
t.Fatalf("%s: bad line %q", path, line)
}
ids = append(ids, id)
}
return ids
}
func TestIntegrationCountShards(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_count")
execSQL(t, db, "CREATE TABLE it_count (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
var b strings.Builder
b.WriteString("INSERT INTO it_count (id, v) VALUES ")
for i := 1; i <= 25; i++ {
if i > 1 {
b.WriteString(",")
}
fmt.Fprintf(&b, "(%d,'v%d')", i, i)
}
execSQL(t, db, b.String())
out := t.TempDir()
_, err := runCLI(t, context.Background(), baseCLI(dsn, "it_count", out,
"--shard-by", "count", "--shard-rows", "10", "--chunk-size", "7", "--format", "dump")...)
if err != nil {
t.Fatal(err)
}
files := listFiles(t, out, "*.dump")
if len(files) != 3 {
t.Fatalf("got %d shard files: %v", len(files), files)
}
wantNames := []string{"000001.1-10.dump", "000002.11-20.dump", "000003.21-25.dump"}
var got []int
for i, f := range files {
if filepath.Base(f) != wantNames[i] {
t.Fatalf("file %d = %s want %s", i, filepath.Base(f), wantNames[i])
}
got = append(got, dumpIDs(t, f)...)
}
if len(got) != 25 {
t.Fatalf("got %d rows", len(got))
}
for i, id := range got {
if id != i+1 {
t.Fatalf("row %d = %d, shards are not contiguous", i, id)
}
}
entries := readManifest(t, out)
if len(entries) != 3 {
t.Fatalf("manifest entries %d", len(entries))
}
for i, e := range entries {
if e.Seq != int64(i+1) || e.Rows != int64(len(dumpIDs(t, filepath.Join(out, e.Path)))) {
t.Fatalf("entry %d %+v", i, e)
}
if e.Bytes == 0 || e.SHA == "" {
t.Fatalf("entry %d missing bytes/hash", i)
}
}
if _, err := os.Stat(filepath.Join(out, "state.json")); err != nil {
t.Fatal(err)
}
var complete bool
st, err := os.ReadFile(filepath.Join(out, "state.json"))
if err != nil {
t.Fatal(err)
}
complete = bytes.Contains(st, []byte(`"complete": true`))
if !complete {
t.Fatalf("state not complete: %s", st)
}
}
type manifestEntry struct {
Seq int64 `json:"seq"`
Path string `json:"path"`
Date string `json:"date"`
Rows int64 `json:"rows"`
Bytes int64 `json:"bytes"`
SHA string `json:"sha256"`
PKLo string `json:"pk_lo"`
PKHi string `json:"pk_hi"`
}
func readManifest(t *testing.T, dir string) []manifestEntry {
t.Helper()
b, err := os.ReadFile(filepath.Join(dir, "manifest.jsonl"))
if err != nil {
t.Fatal(err)
}
var out []manifestEntry
for _, line := range strings.Split(strings.TrimSpace(string(b)), "\n") {
if line == "" {
continue
}
var e manifestEntry
if err := json.Unmarshal([]byte(line), &e); err != nil {
t.Fatal(err)
}
out = append(out, e)
}
return out
}
func TestIntegrationDatePathsAgree(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
for _, spec := range []struct{ name, index string }{
{"it_date_a", "KEY da (created_at, id)"},
{"it_date_b", "KEY db (created_at)"},
} {
execSQL(t, db, "DROP TABLE IF EXISTS "+spec.name)
execSQL(t, db, "CREATE TABLE "+spec.name+` (
id BIGINT NOT NULL,
v VARCHAR(32),
created_at DATETIME NULL,
PRIMARY KEY (id),
`+spec.index+`)`)
execSQL(t, db, `INSERT INTO `+spec.name+` (id, v, created_at) VALUES
(1,'a','2025-01-01 10:00:00'),
(2,'b','2025-01-01 12:00:00'),
(3,'c','2025-01-02 01:00:00'),
(4,'d',NULL),
(5,'e','2025-01-01 11:00:00'),
(6,'f','2025-01-03 00:00:00')`)
}
outA := t.TempDir()
outB := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_a", outA,
"--shard-by", "date", "--shard-column", "created_at", "--format", "dump", "--chunk-size", "2")...); err != nil {
t.Fatal(err)
}
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_b", outB,
"--shard-by", "date", "--shard-column", "created_at", "--format", "dump", "--chunk-size", "2")...); err != nil {
t.Fatal(err)
}
a := readTree(t, outA)
b := readTree(t, outB)
if len(a) != len(b) {
t.Fatalf("file counts differ: %d vs %d (%v / %v)", len(a), len(b), a, b)
}
for k, va := range a {
if b[k] != va {
t.Fatalf("%s:\nA=%q\nB=%q", k, va, b[k])
}
}
if _, ok := a["date=_null/000001.4-4.dump"]; !ok {
t.Fatalf("NULL row not routed to _null: %v", a)
}
if len(a) != 4 {
t.Fatalf("expected 4 shards (null + 3 days), got %v", a)
}
}
func readTree(t *testing.T, dir string) map[string]string {
t.Helper()
out := map[string]string{}
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
if err != nil || d.IsDir() {
return err
}
if !strings.HasSuffix(d.Name(), ".dump") {
return nil
}
rel, _ := filepath.Rel(dir, path)
b, err := os.ReadFile(path)
if err != nil {
return err
}
out[filepath.ToSlash(rel)] = string(b)
return nil
})
if err != nil {
t.Fatal(err)
}
return out
}
func TestIntegrationZeroDateAndNull(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_zero")
execSQL(t, db, `CREATE TABLE it_zero (
id BIGINT NOT NULL,
d DATE NULL,
PRIMARY KEY (id),
KEY dz (d, id))`)
execSQL(t, db, `INSERT INTO it_zero (id, d) VALUES
(1, NULL), (2, '0000-00-00'), (3, '2024-01-01'), (4, '2024-01-01')`)
out := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_zero", out,
"--shard-by", "date", "--shard-column", "d")...); err == nil {
t.Fatal("expected zero-date error")
} else if !strings.Contains(err.Error(), "zero date") {
t.Fatalf("unexpected error: %v", err)
}
out = t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_zero", out,
"--shard-by", "date", "--shard-column", "d", "--zero-date", "bucket")...); err != nil {
t.Fatal(err)
}
tree := readTree(t, out)
for _, want := range []string{
"date=_null/000001.1-1.dump",
"date=0000-00-00/000001.2-2.dump",
"date=2024-01-01/000001.3-4.dump",
} {
if _, ok := tree[want]; !ok {
t.Fatalf("missing %s in %v", want, tree)
}
}
}
func TestIntegrationEpochUnits(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_epoch")
execSQL(t, db, `CREATE TABLE it_epoch (
id BIGINT NOT NULL,
ts BIGINT NOT NULL,
PRIMARY KEY (id),
KEY e (ts, id))`)
execSQL(t, db, `INSERT INTO it_epoch (id, ts) VALUES
(1, 1717200000), (2, 1717200001)`)
for _, tc := range []struct {
unit string
vals string
}{
{"epoch-seconds", "(3, 1717200000), (4, 1717200001)"},
{"epoch-milliseconds", "(3, 1717200000000), (4, 1717200001000)"},
{"epoch-microseconds", "(3, 1717200000000000), (4, 1717200001000000)"},
} {
table := "it_epoch_" + strings.Split(tc.unit, "-")[1]
execSQL(t, db, "DROP TABLE IF EXISTS "+table)
execSQL(t, db, `CREATE TABLE `+table+` (
id BIGINT NOT NULL, ts BIGINT NOT NULL,
PRIMARY KEY (id), KEY e (ts, id))`)
execSQL(t, db, "INSERT INTO "+table+" (id, ts) VALUES "+tc.vals)
out := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, table, out,
"--shard-by", "date", "--shard-column", "ts", "--shard-column-type", tc.unit,
"--shard-time-zone", "UTC")...); err != nil {
t.Fatalf("%s: %v", tc.unit, err)
}
tree := readTree(t, out)
if len(tree) != 1 {
t.Fatalf("%s: expected one date bucket, got %v", tc.unit, tree)
}
for k := range tree {
if !strings.Contains(k, "date=2024-06-01") {
t.Fatalf("%s: wrong bucket %s", tc.unit, k)
}
}
}
}
func TestIntegrationNoIndexFailFast(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_noindex")
execSQL(t, db, `CREATE TABLE it_noindex (
id BIGINT NOT NULL, d DATETIME NULL, PRIMARY KEY (id))`)
execSQL(t, db, `INSERT INTO it_noindex (id, d) VALUES (1,'2024-01-01 00:00:00')`)
out := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_noindex", out,
"--shard-by", "date", "--shard-column", "d")...); err == nil {
t.Fatal("expected no-index error")
} else if !strings.Contains(err.Error(), "no usable index") {
t.Fatalf("unexpected error: %v", err)
}
out = t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_noindex", out,
"--shard-by", "date", "--shard-column", "d", "--allow-full-scan")...); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(out, "date=2024-01-01")); err != nil {
t.Fatal(err)
}
}
func TestIntegrationDryRunCreatesNothing(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_dry")
execSQL(t, db, "CREATE TABLE it_dry (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "INSERT INTO it_dry (id, v) VALUES (1,'a')")
out := filepath.Join(t.TempDir(), "never-created")
stdout, err := runCLI(t, context.Background(), baseCLI(dsn, "it_dry", out, "--dry-run")...)
if err != nil {
t.Fatal(err)
}
if _, statErr := os.Stat(out); !os.IsNotExist(statErr) {
t.Fatal("dry-run created the output directory")
}
if !strings.Contains(stdout, "== first SELECT ==") || !strings.Contains(stdout, "config hash") {
t.Fatalf("dry-run output incomplete:\n%s", stdout)
}
}
func TestIntegrationResumeAfterInterrupt(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_resume")
execSQL(t, db, "CREATE TABLE it_resume (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
var b strings.Builder
b.WriteString("INSERT INTO it_resume (id, v) VALUES ")
const total = 20000
for i := 1; i <= total; i++ {
if i > 1 {
b.WriteString(",")
}
fmt.Fprintf(&b, "(%d,'v')", i)
}
execSQL(t, db, b.String())
out := t.TempDir()
args := baseCLI(dsn, "it_resume", out, "--shard-by", "count", "--shard-rows", "137",
"--chunk-size", "50", "--progress", "100")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
stderrMu.Lock()
old := os.Stderr
r, w, _ := os.Pipe()
os.Stderr = w
cfg, err := cli.Parse(args, io.Discard)
if err != nil {
os.Stderr = old
stderrMu.Unlock()
t.Fatal(err)
}
firstErr := make(chan error, 1)
go func() { firstErr <- Run(ctx, cfg) }()
go func() {
buf := make([]byte, 4096)
for {
n, err := r.Read(buf)
if n > 0 && strings.Contains(string(buf[:n]), "progress:") {
cancel()
}
if err != nil {
return
}
}
}()
err = <-firstErr
_ = w.Close()
os.Stderr = old
stderrMu.Unlock()
if err == nil {
t.Fatal("first run should have been interrupted")
}
if _, err := runCLI(t, context.Background(), args...); err != nil {
t.Fatal(err)
}
seen := map[int]bool{}
for _, f := range listFiles(t, out, "*.dump") {
for _, id := range dumpIDs(t, f) {
seen[id] = true
}
}
for i := 1; i <= total; i++ {
if !seen[i] {
t.Fatalf("row %d missing after resume", i)
}
}
st, err := os.ReadFile(filepath.Join(out, "state.json"))
if err != nil {
t.Fatal(err)
}
if !bytes.Contains(st, []byte(`"complete": true`)) {
t.Fatalf("state not complete: %s", st)
}
var saved struct {
RowsDone int64 `json:"rows_done"`
}
if err := json.Unmarshal(st, &saved); err != nil {
t.Fatal(err)
}
if saved.RowsDone != total {
t.Fatalf("rows_done = %d after resume, want cumulative %d", saved.RowsDone, total)
}
if _, err := filepath.Glob(filepath.Join(out, "*.partial")); err != nil {
t.Fatal(err)
}
}
func TestIntegrationProgressModes(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_progress")
execSQL(t, db, "CREATE TABLE it_progress (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "INSERT INTO it_progress (id, v) VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e')")
for _, extra := range [][]string{
nil,
{"--progress", "off"},
{"--progress", "0"},
} {
out := t.TempDir()
args := baseCLI(dsn, "it_progress", out, extra...)
errOut, err := runCLIStderr(t, context.Background(), args...)
if err != nil {
t.Fatalf("%v: %v", extra, err)
}
if strings.Contains(errOut, "progress:") || strings.Contains(errOut, "done rows=") {
t.Fatalf("%v: unexpected progress output %q", extra, errOut)
}
}
out := t.TempDir()
errOut, err := runCLIStderr(t, context.Background(), baseCLI(dsn, "it_progress", out, "--progress", "on")...)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(errOut, "done rows=5") {
t.Fatalf("forced progress missing done summary: %q", errOut)
}
if strings.Contains(errOut, "\r") {
t.Fatalf("redirected progress must not use carriage returns: %q", errOut)
}
out = t.TempDir()
errOut, err = runCLIStderr(t, context.Background(), baseCLI(dsn, "it_progress", out, "--progress", "1")...)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(errOut, "progress: rows=5") {
t.Fatalf("row-threshold progress line missing: %q", errOut)
}
}
func TestIntegrationEstimateMetrics(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_estimate")
execSQL(t, db, `CREATE TABLE it_estimate (
id BIGINT NOT NULL,
v VARCHAR(32),
created_at DATETIME NULL,
PRIMARY KEY (id),
KEY de (created_at, id))`)
execSQL(t, db, `INSERT INTO it_estimate (id, v, created_at) VALUES
(1,'a','2025-01-01 01:00:00'),
(2,'b','2025-01-01 02:00:00'),
(3,'c','2025-01-02 01:00:00'),
(4,'d','2025-01-03 01:00:00'),
(5,'e','2025-01-03 02:00:00')`)
out := t.TempDir()
errOut, err := runCLIStderr(t, context.Background(), baseCLI(dsn, "it_estimate", out, "--progress", "1")...)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(errOut, "estimate=key") {
t.Fatalf("primary key estimate missing: %q", errOut)
}
out = t.TempDir()
errOut, err = runCLIStderr(t, context.Background(),
baseCLI(dsn, "it_estimate", out, "--progress", "1", "--shard-by", "date", "--shard-column", "created_at")...)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(errOut, "estimate=day") {
t.Fatalf("day estimate missing: %q", errOut)
}
}
func TestIntegrationImportDump(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_import_src")
execSQL(t, db, "CREATE TABLE it_import_src (id BIGINT NOT NULL, v VARCHAR(64), PRIMARY KEY (id))")
execSQL(t, db, `INSERT INTO it_import_src (id, v) VALUES
(1,'plain'), (2,'tab\there'), (3,'line\nbreak'), (4,'back\\slash')`)
for _, comp := range []string{"none", "gzip", "zstd", "xz"} {
out := t.TempDir()
args := baseCLI(dsn, "it_import_src", out, "--format", "dump", "--compress", comp)
if _, err := runCLI(t, context.Background(), args...); err != nil {
t.Fatalf("%s: %v", comp, err)
}
files := listFiles(t, out, "*.dump*")
if len(files) != 1 {
t.Fatalf("%s: files %v", comp, files)
}
loadPath := files[0]
if comp != "none" {
loadPath = filepath.Join(t.TempDir(), "decompressed.dump")
if err := decompressFile(files[0], loadPath, comp); err != nil {
t.Fatal(err)
}
}
execSQL(t, db, "DROP TABLE IF EXISTS it_import_dst")
execSQL(t, db, "CREATE TABLE it_import_dst (id BIGINT NOT NULL, v VARCHAR(64), PRIMARY KEY (id))")
loadLocal(t, dsn, loadPath, "it_import_dst")
var n int
if err := db.QueryRow("SELECT COUNT(*) FROM it_import_dst").Scan(&n); err != nil {
t.Fatal(err)
}
if n != 4 {
t.Fatalf("%s: loaded %d rows", comp, n)
}
var v string
if err := db.QueryRow("SELECT v FROM it_import_dst WHERE id=3").Scan(&v); err != nil {
t.Fatal(err)
}
if v != "line\nbreak" {
t.Fatalf("%s: round trip mismatch %q", comp, v)
}
}
}
func decompressFile(src, dst, comp string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
var r io.Reader
switch comp {
case "gzip":
gr, err := gzip.NewReader(in)
if err != nil {
return err
}
defer gr.Close()
r = gr
case "zstd":
zr, err := zstd.NewReader(in)
if err != nil {
return err
}
defer zr.Close()
r = zr
case "xz":
xr, err := xz.NewReader(in)
if err != nil {
return err
}
r = xr
default:
r = in
}
out, err := os.Create(dst)
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, r)
return err
}
func loadLocal(t *testing.T, dsn, path, table string) {
t.Helper()
cfg, err := mysql.ParseDSN(dsn)
if err != nil {
t.Fatal(err)
}
cfg.AllowAllFiles = true
db, err := sql.Open("mysql", cfg.FormatDSN())
if err != nil {
t.Fatal(err)
}
defer db.Close()
p := filepath.ToSlash(path)
p = strings.ReplaceAll(p, `'`, `''`)
q := fmt.Sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE %s", p, table)
if _, err := db.Exec(q); err != nil {
t.Fatalf("load data: %v", err)
}
}
func TestIntegrationCSVAndJSONL(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_fmt")
execSQL(t, db, `CREATE TABLE it_fmt (
id BIGINT NOT NULL, name VARCHAR(64), price DECIMAL(10,2), created DATETIME,
data BLOB, PRIMARY KEY (id))`)
execSQL(t, db, `INSERT INTO it_fmt (id, name, price, created, data) VALUES
(1,'a,\"b\"',12.30,'2024-01-02 03:04:05',X'6869'), (2,'',NULL,NULL,NULL)`)
// CSV
out := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_fmt", out, "--format", "csv")...); err != nil {
t.Fatal(err)
}
files := listFiles(t, out, "*.csv")
if len(files) != 1 {
t.Fatalf("csv files %v", files)
}
f, err := os.Open(files[0])
if err != nil {
t.Fatal(err)
}
recs, err := csv.NewReader(f).ReadAll()
f.Close()
if err != nil {
t.Fatal(err)
}
if len(recs) != 3 {
t.Fatalf("csv records %d", len(recs))
}
if recs[0][0] != "id" || recs[0][1] != "name" {
t.Fatalf("header %v", recs[0])
}
if recs[1][1] != `a,"b"` || recs[1][2] != "12.30" {
t.Fatalf("row %v", recs[1])
}
if recs[2][2] != `\N` {
t.Fatalf("NULL decimal %q", recs[2][2])
}
// JSONL
out = t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_fmt", out, "--format", "jsonl")...); err != nil {
t.Fatal(err)
}
files = listFiles(t, out, "*.jsonl")
var shardFiles []string
for _, f := range files {
if filepath.Base(f) != "manifest.jsonl" {
shardFiles = append(shardFiles, f)
}
}
files = shardFiles
if len(files) != 1 {
t.Fatalf("jsonl files %v", files)
}
b, err := os.ReadFile(files[0])
if err != nil {
t.Fatal(err)
}
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
if len(lines) != 2 {
t.Fatalf("jsonl lines %d", len(lines))
}
var obj map[string]any
if err := json.Unmarshal([]byte(lines[0]), &obj); err != nil {
t.Fatal(err)
}
if obj["price"] != "12.30" {
t.Fatalf("price %v", obj["price"])
}
if obj["created"] != "2024-01-02T03:04:05Z" {
t.Fatalf("created %v", obj["created"])
}
if obj["data"] != "aGk=" {
t.Fatalf("data %v", obj["data"])
}
var second map[string]any
if err := json.Unmarshal([]byte(lines[1]), &second); err != nil {
t.Fatal(err)
}
if second["price"] != nil || second["created"] != nil {
t.Fatalf("null mapping %v", second)
}
}
func TestIntegrationPurgeAndDest(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_purge_src")
execSQL(t, db, "DROP TABLE IF EXISTS it_purge_dst")
execSQL(t, db, "CREATE TABLE it_purge_src (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "CREATE TABLE it_purge_dst (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "INSERT INTO it_purge_src (id, v) VALUES (1,'a'),(2,'b'),(3,'c')")
out := t.TempDir()
args := []string{
"--source", dsn, "--table", "it_purge_src", "--where", "1=1",
"--file", out, "--dest", dsn, "--dest-table", "it_purge_dst",
"--purge", "--chunk-size", "2",
}
if _, err := runCLI(t, context.Background(), args...); err != nil {
t.Fatal(err)
}
var srcCount, dstCount int
db.QueryRow("SELECT COUNT(*) FROM it_purge_src").Scan(&srcCount)
db.QueryRow("SELECT COUNT(*) FROM it_purge_dst").Scan(&dstCount)
if srcCount != 0 || dstCount != 3 {
t.Fatalf("src=%d dst=%d", srcCount, dstCount)
}
if len(listFiles(t, out, "*.dump")) != 1 {
t.Fatal("expected one shard")
}
}
func TestIntegrationCheckpointConfigMismatch(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_mismatch")
execSQL(t, db, "CREATE TABLE it_mismatch (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "INSERT INTO it_mismatch (id, v) VALUES (1,'a'),(2,'b')")
out := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_mismatch", out)...); err != nil {
t.Fatal(err)
}
// Remove the complete flag so resume logic runs.
stPath := filepath.Join(out, "state.json")
b, _ := os.ReadFile(stPath)
b = bytes.Replace(b, []byte(`"complete": true`), []byte(`"complete": false`), 1)
if err := os.WriteFile(stPath, b, 0o644); err != nil {
t.Fatal(err)
}
args := baseCLI(dsn, "it_mismatch", out)
for i := range args {
if args[i] == "1=1" {
args[i] = "id > 0"
}
}
if _, err := runCLI(t, context.Background(), args...); err == nil {
t.Fatal("expected config mismatch error")
} else if !strings.Contains(err.Error(), "different configuration") {
t.Fatalf("unexpected error: %v", err)
}
args = append(args, "--force-resume")
if _, err := runCLI(t, context.Background(), args...); err != nil {
t.Fatalf("force resume: %v", err)
}
}
+129
View File
@@ -0,0 +1,129 @@
package app
import (
"context"
"database/sql"
"fmt"
"strings"
"time"
"archer/internal/archererr"
"archer/internal/cli"
"archer/internal/row"
"archer/internal/schema"
)
// probeResult is everything learned about the source table needed to plan the
// scan and the encoders.
type probeResult struct {
table *schema.Table
key schema.KeyInfo
shard *schema.ShardSpec
path string // pk | A | B
joint *schema.Index
single *schema.Index
selectCols []string
selectMeta []row.Column
keyIdx []int
shardIdx int
outCols []row.Column
outIdx []int
}
func probeTable(ctx context.Context, db *sql.DB, cfg *cli.Config, dbName string, loc *time.Location) (*probeResult, error) {
table, err := schema.Load(ctx, db, dbName, cfg.Table)
if err != nil {
return nil, archererr.Wrap(archererr.KindValidation, err)
}
key, err := table.ChooseKey(cfg.Index)
if err != nil {
return nil, archererr.Wrap(archererr.KindValidation, err)
}
outCols, err := table.OutputColumns(cfg.Columns)
if err != nil {
return nil, archererr.Wrap(archererr.KindValidation, err)
}
p := &probeResult{table: table, key: key, path: "pk", shardIdx: -1, outCols: outCols}
if cfg.IsDateShard() {
spec, err := schema.ResolveShardColumn(ctx, db, table, cfg.ShardColumn, cfg.ShardColumnType, cfg.Where, loc)
if err != nil {
return nil, archererr.Wrap(archererr.KindValidation, err)
}
p.shard = spec
prefix := append([]string{spec.Column}, key.Columns...)
p.joint = table.FindIndexPrefix(prefix)
p.single = table.FindIndexPrefix([]string{spec.Column})
switch {
case p.joint != nil:
p.path = "A"
case p.single != nil:
p.path = "B"
case cfg.AllowFullScan:
p.path = "B"
default:
return nil, archererr.Newf(archererr.KindValidation,
"no usable index on %s for date sharding\n joint: create an index on (%s) for keyset chunks\n or: create an index on (%s) for day-at-a-time streaming\n or: pass --allow-full-scan to accept a full table scan\n%s",
table.Name, strings.Join(prefix, ", "), spec.Column, describeIndexes(table))
}
}
needed := map[string]bool{}
for _, c := range outCols {
needed[strings.ToLower(c.Name)] = true
}
for _, c := range key.Columns {
needed[strings.ToLower(c)] = true
}
if p.shard != nil {
needed[strings.ToLower(p.shard.Column)] = true
}
for _, c := range table.Columns {
if needed[strings.ToLower(c.Name)] {
p.selectCols = append(p.selectCols, c.Name)
p.selectMeta = append(p.selectMeta, row.Column{Name: c.Name, Type: c.DataType})
}
}
pos := map[string]int{}
for i, name := range p.selectCols {
pos[name] = i
}
p.keyIdx = make([]int, len(key.Columns))
for i, c := range key.Columns {
idx, ok := pos[c]
if !ok {
return nil, archererr.Newf(archererr.KindValidation, "internal: key column %s missing from select list", c)
}
p.keyIdx[i] = idx
}
if p.shard != nil {
idx, ok := pos[p.shard.Column]
if !ok {
return nil, archererr.Newf(archererr.KindValidation, "internal: shard column %s missing from select list", p.shard.Column)
}
p.shardIdx = idx
}
p.outIdx = make([]int, len(outCols))
for i, c := range outCols {
p.outIdx[i] = pos[c.Name]
}
return p, nil
}
func describeIndexes(t *schema.Table) string {
if len(t.Indexes) == 0 {
return " indexes: (none)"
}
var b strings.Builder
b.WriteString(" indexes:\n")
for _, ix := range t.Indexes {
kind := "index"
if ix.Name == "PRIMARY" {
kind = "primary key"
} else if ix.Unique {
kind = "unique index"
}
fmt.Fprintf(&b, " - %s (%s) %s\n", ix.Name, strings.Join(ix.Columns, ", "), kind)
}
return strings.TrimRight(b.String(), "\n")
}
+43
View File
@@ -0,0 +1,43 @@
package app
import (
"bytes"
"os"
"testing"
"archer/internal/checkpoint"
"archer/internal/cli"
)
func TestStateConfigStoresRedactedDSN(t *testing.T) {
const source = "u:s3cr3t@tcp(h:3306)/db?charset=utf8mb4"
cfg, err := cli.Parse([]string{"--source", source, "--table", "t", "--where", "id > 0", "--file", "out"}, &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
hash, fields, err := cfg.Fingerprint()
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
if err := checkpoint.Save(dir, &checkpoint.State{ConfigHash: hash, Config: fields}); err != nil {
t.Fatal(err)
}
st, err := checkpoint.Load(dir)
if err != nil {
t.Fatal(err)
}
want := redactDSN(source)
if st.Config["source"] != want {
t.Fatalf("config.source = %q, want %q", st.Config["source"], want)
}
raw, err := os.ReadFile(checkpoint.Path(dir))
if err != nil {
t.Fatal(err)
}
if bytes.Contains(raw, []byte("s3cr3t")) {
t.Fatal("state.json must not contain the password")
}
}