Initial commit: archer, a Go-native MySQL/MariaDB table archiver

Renamed from arcdb: module path, cmd/archer, internal/archererr,
docs, build scripts and ARCHER_* environment variables.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-06 22:12:17 +08:00
commit bc4a6b504f
99 files changed
+12752

No files matched your search

+705
View File
@@ -0,0 +1,705 @@
// Package app wires the CLI, schema probe, scanners, shard writer and
// database targets into one run.
package app
import (
"context"
"crypto/rand"
"database/sql"
"encoding/hex"
"fmt"
"io"
"os"
"strconv"
"strings"
"time"
"github.com/go-sql-driver/mysql"
"archer/internal/archererr"
"archer/internal/checkpoint"
"archer/internal/cli"
"archer/internal/lag"
"archer/internal/progress"
"archer/internal/purge"
"archer/internal/row"
"archer/internal/scan"
"archer/internal/shard"
"archer/internal/sink"
)
// Stats collects counters and phase timings for --statistics.
type Stats struct {
Started time.Time
RowsRead int64
RowsCommitted int64
RowsInserted int64
RowsDeleted int64
ScanDur time.Duration
FileDur time.Duration
DBDur time.Duration
CheckpointDur time.Duration
Shards int64
Bytes int64
}
// App holds a run's state.
type App struct {
cfg *cli.Config
srcDB *sql.DB
destDB *sql.DB
srcCfg *mysql.Config
destCfg *mysql.Config
loc *time.Location
probe *probeResult
cfgHash string
cfgFields map[string]string
scanner scan.Scanner
fileSink *sink.File
writer *shard.Writer
manifest *shard.Manifest
destSink *sink.DB
purger *purge.Purger
lagChecker *lag.Checker
sameInstance bool
resume *scan.Resume
alreadyComplete bool
st *checkpoint.State
stats Stats
reporter *progress.Reporter
estimate *progress.Estimate
baseRows int64
}
// Run executes one archiving run.
func Run(ctx context.Context, cfg *cli.Config) error {
if err := cfg.Validate(); err != nil {
return err
}
a := &App{cfg: cfg, stats: Stats{Started: time.Now()}}
defer a.close()
if !cfg.DryRun {
a.startReporter()
defer a.reporter.Stop()
}
if err := a.setupSource(ctx); err != nil {
return err
}
if cfg.DryRun {
return a.printDryRun(os.Stdout)
}
if err := a.prepare(ctx); err != nil {
return err
}
if a.alreadyComplete {
fmt.Fprintf(os.Stdout, "archer: output in %s is already complete; nothing to do\n", cfg.FileDir)
return nil
}
return a.execute(ctx)
}
func (a *App) startReporter() {
tty := progress.IsTerminal(os.Stderr)
enabled := a.cfg.Progress.Enabled(tty)
a.reporter = progress.New(progress.Options{
Out: os.Stderr,
Enabled: enabled,
Live: enabled && tty,
Interval: a.cfg.ProgressInterval,
RowEvery: a.cfg.Progress.Rows,
})
a.reporter.Start()
}
func (a *App) phase(name, reason string) {
if a.reporter != nil {
a.reporter.Phase(name, reason)
}
}
func (a *App) message(format string, args ...any) {
if a.reporter != nil {
a.reporter.Message(format, args...)
return
}
fmt.Fprintf(os.Stderr, format+"\n", args...)
}
func (a *App) setupSource(ctx context.Context) error {
a.phase("probe", "")
loc, err := time.LoadLocation(a.cfg.ShardTimeZone)
if err != nil {
return archererr.Newf(archererr.KindConfig, "invalid --shard-time-zone: %v", err)
}
a.loc = loc
srcCfg, err := a.cfg.SourceConfig()
if err != nil {
return archererr.Wrap(archererr.KindConfig, err)
}
if srcCfg.DBName == "" {
return archererr.Newf(archererr.KindConfig, "--source must include a database name")
}
a.srcCfg = srcCfg
a.srcDB, err = openDB(srcCfg)
if err != nil {
return err
}
a.probe, err = probeTable(ctx, a.srcDB, a.cfg, srcCfg.DBName, loc)
if err != nil {
return err
}
a.cfgHash, a.cfgFields, err = a.cfg.Fingerprint()
if err != nil {
return archererr.Wrap(archererr.KindConfig, err)
}
return nil
}
func (a *App) prepare(ctx context.Context) error {
a.phase("resume", "")
if err := a.openDest(ctx); err != nil {
return err
}
if a.cfg.Purge {
autoInc := false
if len(a.probe.key.Columns) == 1 {
if c, ok := a.probe.table.Column(a.probe.key.Columns[0]); ok {
autoInc = c.IsAutoIncrement()
}
}
a.purger = purge.New(a.srcDB, a.srcCfg.DBName, a.cfg.Table, a.probe.key.Columns,
a.cfg.Where, a.cfg.BulkDelete, a.cfg.SafeAutoIncrement, autoInc)
}
if a.cfg.CheckReplicaLag != "" {
c, err := lag.Open(a.cfg.CheckReplicaLag, a.cfg.MaxLag, a.cfg.CheckInterval)
if err != nil {
return archererr.Wrap(archererr.KindConnection, err)
}
c.OnWait = func(reason string) {
a.phase("lag-wait", reason)
}
a.lagChecker = c
}
if a.cfg.HasFile() {
if err := a.prepareFile(ctx); err != nil {
return err
}
}
if a.alreadyComplete {
return nil
}
a.phase("estimate", "")
a.estimate = a.queryEstimate(ctx)
if a.reporter != nil {
a.reporter.SetEstimate(a.estimate)
}
return a.openScanner(ctx)
}
func (a *App) openDest(ctx context.Context) error {
if a.cfg.DestDSN == "" {
return nil
}
destCfg, err := a.cfg.DestConfig()
if err != nil {
return archererr.Wrap(archererr.KindConfig, err)
}
if destCfg.DBName == "" {
return archererr.Newf(archererr.KindConfig, "--dest must include a database name")
}
a.destCfg = destCfg
a.sameInstance = cli.SameInstance(a.srcCfg, destCfg)
a.destDB, err = openDB(destCfg)
if err != nil {
return err
}
names := make([]string, len(a.probe.outCols))
for i, c := range a.probe.outCols {
names[i] = c.Name
}
s, err := sink.NewDB(ctx, a.destDB, destCfg.DBName, a.cfg.DestTable, names, a.cfg.InsertMode, a.probe.outIdx)
if err != nil {
return archererr.Wrap(archererr.KindValidation, err)
}
a.destSink = s
return nil
}
func (a *App) prepareFile(ctx context.Context) error {
if err := os.MkdirAll(a.cfg.FileDir, 0o755); err != nil {
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("create output directory: %w", err))
}
comp, err := shard.NewCompressor(a.cfg.Compress, a.cfg.CompressLevel)
if err != nil {
return archererr.Wrap(archererr.KindConfig, err)
}
man, err := shard.OpenManifest(a.cfg.FileDir)
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
a.manifest = man
maxSeq, err := man.Adopt(a.cfg.FileDir, a.cfg.Format, comp)
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
st, err := checkpoint.Load(a.cfg.FileDir)
if err != nil {
return archererr.Wrap(archererr.KindValidation, err)
}
startBucket, startSeq := "", int64(0)
if st != nil {
if st.ConfigHash != a.cfgHash {
if !a.cfg.ForceResume {
diffs := cli.DiffFields(st.Config, a.cfgFields)
if len(diffs) == 0 {
diffs = []string{"fingerprint changed but no field differences were recorded"}
}
return archererr.Newf(archererr.KindValidation,
"checkpoint in %s was created with a different configuration:\n %s\nre-run with --force-resume to ignore this",
a.cfg.FileDir, strings.Join(diffs, "\n "))
}
a.message("warning: --force-resume ignores the changed configuration fingerprint")
}
if st.Complete {
a.alreadyComplete = true
a.st = st
return nil
}
a.st = st
a.baseRows = st.RowsDone
a.resume = &scan.Resume{Phase: st.Cursor.Phase, Key: st.Cursor.Key, DayStart: st.Cursor.DayStart}
startBucket, startSeq = st.LastBucket, st.LastShardSeq
} else {
a.st = &checkpoint.State{RunID: newRunID(), ShardBy: a.cfg.ShardBy}
}
if a.reporter != nil {
a.reporter.SetBase(a.baseRows)
}
if maxSeq[startBucket] > startSeq {
startSeq = maxSeq[startBucket]
}
var keep string
var resumeMeta *shard.PartialMeta
if a.cfg.Purge {
if m, err := shard.LoadPartialMeta(a.cfg.FileDir, startBucket, startSeq+1); err != nil {
return archererr.Wrap(archererr.KindValidation, err)
} else if m != nil {
resumeMeta = m
keep = shard.PartialPath(a.cfg.FileDir, startBucket, startSeq+1)
}
}
if err := shard.CleanupPartials(a.cfg.FileDir, keep); err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
w, err := shard.Open(shard.Options{
Dir: a.cfg.FileDir,
Mode: a.cfg.ShardBy,
ShardRows: int64(a.cfg.ShardRows),
Format: a.cfg.Format,
Compressor: comp,
Columns: a.probe.selectMeta,
OutIdx: a.probe.outIdx,
CSVNull: a.cfg.CSVNull,
StartBucket: startBucket,
StartSeq: startSeq,
}, man)
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
if resumeMeta != nil {
if err := w.Resume(resumeMeta); err != nil {
return archererr.Wrap(archererr.KindValidation, err)
}
}
a.writer = w
a.fileSink = &sink.File{W: w}
return nil
}
func (a *App) openScanner(ctx context.Context) error {
opts := scan.Options{
DB: a.srcDB,
DBName: a.srcCfg.DBName,
Table: a.cfg.Table,
Where: a.cfg.Where,
SelectCols: a.probe.selectCols,
KeyCols: a.probe.key.Columns,
KeyIdx: a.probe.keyIdx,
ChunkSize: a.cfg.ChunkSize,
Shard: a.probe.shard,
ShardIdx: a.probe.shardIdx,
ShardLoc: a.loc,
ZeroDate: a.cfg.ZeroDate,
Resume: a.resume,
}
switch a.probe.path {
case "A":
a.scanner = scan.NewDateA(opts)
case "B":
s, err := scan.NewDateB(ctx, opts)
if err != nil {
return archererr.Wrap(archererr.KindConnection, err)
}
a.scanner = s
default:
a.scanner = scan.NewPK(opts)
}
return nil
}
func (a *App) execute(ctx context.Context) error {
if a.cfg.HasFile() {
a.message("note: file output is at-least-once; a crash and rerun may duplicate rows in shard files")
}
if a.cfg.Purge {
a.message("warning: --purge deletes archived rows from the source table")
}
if a.cfg.BulkDelete {
a.message("warning: --bulk-delete removes all rows in the scanned key range that match --where, including concurrent inserts")
}
for {
if err := ctx.Err(); err != nil {
return a.interrupt(err)
}
if a.lagChecker != nil {
if err := a.lagChecker.Wait(ctx); err != nil {
if ctx.Err() != nil {
return a.interrupt(err)
}
return archererr.Wrap(archererr.KindConnection, err)
}
}
a.phase("scan", "")
start := time.Now()
chunk, err := a.scanner.Next(ctx)
a.stats.ScanDur += time.Since(start)
if err == io.EOF {
break
}
if err != nil {
if ctx.Err() != nil {
return a.interrupt(err)
}
return archererr.Wrap(archererr.KindOther, fmt.Errorf("scan: %w", err))
}
if err := a.processChunk(ctx, chunk); err != nil {
if ctx.Err() != nil {
return a.interrupt(err)
}
return err
}
a.stats.RowsRead += int64(chunk.Len())
a.publishProgress(chunk)
}
return a.finish()
}
func (a *App) processChunk(ctx context.Context, chunk *row.Chunk) error {
fileStart := time.Now()
if a.fileSink != nil {
a.phase("file", "")
if err := a.fileSink.WriteChunk(ctx, chunk); err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
if a.cfg.Purge {
if err := a.fileSink.Checkpoint(); err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
}
}
a.stats.FileDur += time.Since(fileStart)
dbStart := time.Now()
if a.destSink != nil || a.purger != nil {
a.phase("db", "")
}
if err := a.runDB(ctx, chunk); err != nil {
return err
}
a.stats.DBDur += time.Since(dbStart)
if a.fileSink != nil && !a.cfg.Purge {
if err := a.fileSink.Flush(ctx); err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
}
a.stats.RowsCommitted += int64(chunk.Len())
if a.cfg.HasFile() {
a.phase("checkpoint", "")
}
cpStart := time.Now()
if err := a.saveState(false); err != nil {
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("save checkpoint: %w", err))
}
a.stats.CheckpointDur += time.Since(cpStart)
return nil
}
func (a *App) runDB(ctx context.Context, chunk *row.Chunk) error {
attempts := a.cfg.Retries + 1
if a.destSink != nil && a.purger != nil && a.sameInstance {
err := retry(ctx, attempts, func() error {
tx, err := a.destDB.BeginTx(ctx, nil)
if err != nil {
return err
}
n, err := a.destSink.InsertChunk(ctx, tx, chunk)
if err != nil {
_ = tx.Rollback()
return err
}
d, err := a.purger.DeleteChunk(ctx, tx, chunk)
if err != nil {
_ = tx.Rollback()
return err
}
if err := tx.Commit(); err != nil {
return err
}
a.stats.RowsInserted += n
a.stats.RowsDeleted += d
return nil
})
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
return nil
}
if a.destSink != nil {
err := retry(ctx, attempts, func() error {
tx, err := a.destDB.BeginTx(ctx, nil)
if err != nil {
return err
}
n, err := a.destSink.InsertChunk(ctx, tx, chunk)
if err != nil {
_ = tx.Rollback()
return err
}
if err := tx.Commit(); err != nil {
return err
}
a.stats.RowsInserted += n
return nil
})
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
}
if a.purger != nil {
err := retry(ctx, attempts, func() error {
tx, err := a.srcDB.BeginTx(ctx, nil)
if err != nil {
return err
}
d, err := a.purger.DeleteChunk(ctx, tx, chunk)
if err != nil {
_ = tx.Rollback()
return err
}
if err := tx.Commit(); err != nil {
return err
}
a.stats.RowsDeleted += d
return nil
})
if err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
}
return nil
}
func retry(ctx context.Context, attempts int, fn func() error) error {
var err error
for i := 0; i < attempts; i++ {
if i > 0 {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Duration(i) * time.Second):
}
}
if err = fn(); err == nil {
return nil
}
}
return err
}
func (a *App) saveState(complete bool) error {
if a.cfg.FileDir == "" || a.st == nil || a.alreadyComplete {
return nil
}
st := a.st
st.ConfigHash = a.cfgHash
st.Config = a.cfgFields
st.ShardBy = a.cfg.ShardBy
st.RowsDone = a.baseRows + a.stats.RowsCommitted
if a.scanner != nil {
cs := a.scanner.ResumeState()
st.Cursor = checkpoint.Cursor{Phase: cs.Phase, Key: cs.Key, DayStart: cs.DayStart}
}
if a.writer != nil {
st.LastBucket, st.LastShardSeq = a.writer.StatePosition()
}
st.Complete = complete
return checkpoint.Save(a.cfg.FileDir, st)
}
func (a *App) finish() error {
a.phase("finalize", "")
if a.writer != nil {
if err := a.writer.Close(); err != nil {
return archererr.Wrap(archererr.KindWrite, err)
}
var shards, bytes int64
_, shards, bytes = a.writer.Stats()
a.stats.Shards, a.stats.Bytes = shards, bytes
}
if err := a.saveState(true); err != nil {
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("save checkpoint: %w", err))
}
a.printStats()
if a.reporter != nil {
if a.writer != nil {
a.reporter.SetShards(a.stats.Shards)
a.reporter.SetBytes(a.stats.Bytes)
}
a.reporter.Done()
}
return nil
}
func (a *App) interrupt(cause error) error {
if a.reporter != nil {
a.reporter.Interrupted()
}
if a.writer != nil {
if err := a.writer.SealCurrent(); err != nil {
a.message("warning: could not seal current shard: %v", err)
}
}
if err := a.saveState(false); err != nil {
a.message("warning: could not save checkpoint: %v", err)
}
return archererr.Wrap(archererr.KindInterrupted, cause)
}
func (a *App) publishProgress(chunk *row.Chunk) {
if a.reporter == nil {
return
}
if a.estimate != nil {
switch a.estimate.Kind {
case "day":
if len(chunk.Buckets) > 0 {
b := chunk.Buckets[len(chunk.Buckets)-1]
if b != "" && b != "_null" && !strings.HasPrefix(b, "0000-") {
if t, err := time.ParseInLocation("2006-01-02", b, a.loc); err == nil {
a.reporter.SetPosition(float64(t.Unix()))
}
}
}
case "key":
if len(chunk.LastKey) == 1 && !chunk.LastKey[0].Null {
if n, err := strconv.ParseInt(chunk.LastKey[0].Val, 10, 64); err == nil {
a.reporter.SetPosition(float64(n))
}
}
}
}
if a.writer != nil {
a.reporter.SetShard(a.writer.CurrentShard())
a.reporter.SetBytes(a.writer.WrittenBytes())
}
a.reporter.AddRows(int64(chunk.Len()))
}
func (a *App) printStats() {
if !a.cfg.Statistics {
return
}
fmt.Fprintln(os.Stdout, "this run:")
fmt.Fprintf(os.Stdout, " rows read: %d\n", a.stats.RowsRead)
fmt.Fprintf(os.Stdout, " rows committed: %d\n", a.stats.RowsCommitted)
fmt.Fprintf(os.Stdout, " rows inserted: %d\n", a.stats.RowsInserted)
fmt.Fprintf(os.Stdout, " rows deleted: %d\n", a.stats.RowsDeleted)
if a.writer != nil {
fmt.Fprintf(os.Stdout, " shards: %d\n", a.stats.Shards)
fmt.Fprintf(os.Stdout, " bytes written: %d\n", a.stats.Bytes)
}
fmt.Fprintf(os.Stdout, " elapsed: %s (scan %s, file %s, db %s, checkpoint %s)\n",
time.Since(a.stats.Started).Round(time.Millisecond),
a.stats.ScanDur.Round(time.Millisecond), a.stats.FileDur.Round(time.Millisecond),
a.stats.DBDur.Round(time.Millisecond), a.stats.CheckpointDur.Round(time.Millisecond))
fmt.Fprintln(os.Stdout, "cumulative (including prior runs):")
fmt.Fprintf(os.Stdout, " rows read: %d\n", a.baseRows+a.stats.RowsRead)
fmt.Fprintf(os.Stdout, " rows committed: %d\n", a.baseRows+a.stats.RowsCommitted)
if a.writer != nil && a.manifest != nil {
var rows, bytes int64
entries := a.manifest.Entries()
for _, e := range entries {
rows += e.Rows
bytes += e.Bytes
}
fmt.Fprintf(os.Stdout, " shards: %d\n", len(entries))
fmt.Fprintf(os.Stdout, " rows written: %d\n", rows)
fmt.Fprintf(os.Stdout, " bytes written: %d\n", bytes)
}
}
func (a *App) close() {
if a.scanner != nil {
_ = a.scanner.Close()
}
if a.writer != nil {
_ = a.writer.Abandon()
} else if a.manifest != nil {
_ = a.manifest.Close()
}
if a.lagChecker != nil {
_ = a.lagChecker.Close()
}
if a.destDB != nil {
_ = a.destDB.Close()
}
if a.srcDB != nil {
_ = a.srcDB.Close()
}
}
func openDB(cfg *mysql.Config) (*sql.DB, error) {
db, err := sql.Open("mysql", cfg.FormatDSN())
if err != nil {
return nil, archererr.Wrap(archererr.KindConnection, err)
}
db.SetMaxOpenConns(8)
db.SetMaxIdleConns(4)
db.SetConnMaxLifetime(time.Hour)
if err := db.Ping(); err != nil {
db.Close()
return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("connect %s: %w", cfg.Addr, err))
}
return db, nil
}
func newRunID() string {
b := make([]byte, 8)
if _, err := rand.Read(b); err != nil {
return time.Now().UTC().Format("20060102T150405Z")
}
return time.Now().UTC().Format("20060102T150405Z") + "-" + hex.EncodeToString(b)
}
+162
View File
@@ -0,0 +1,162 @@
package app
import (
"fmt"
"io"
"strings"
"github.com/go-sql-driver/mysql"
"archer/internal/purge"
"archer/internal/shard"
"archer/internal/sqlutil"
)
func (a *App) printDryRun(w io.Writer) error {
cfg := a.cfg
comp, err := shard.NewCompressor(cfg.Compress, cfg.CompressLevel)
if err != nil {
return err
}
fmt.Fprintln(w, "== resolved configuration ==")
fmt.Fprintf(w, "source: %s\n", redactDSN(cfg.SourceDSN))
if cfg.DestDSN != "" {
fmt.Fprintf(w, "dest: %s\n", redactDSN(cfg.DestDSN))
}
fmt.Fprintf(w, "table: %s\n", cfg.Table)
if cfg.DestDSN != "" {
fmt.Fprintf(w, "dest table: %s\n", cfg.DestTable)
}
fmt.Fprintf(w, "where: %s\n", cfg.Where)
if len(cfg.Columns) > 0 {
fmt.Fprintf(w, "columns: %s\n", strings.Join(cfg.Columns, ", "))
} else {
fmt.Fprintf(w, "columns: (all)\n")
}
targets := []string{}
if cfg.HasFile() {
targets = append(targets, "file="+cfg.FileDir)
}
if cfg.DestDSN != "" {
targets = append(targets, "dest")
}
if cfg.Purge {
targets = append(targets, "purge")
}
fmt.Fprintf(w, "targets: %s\n", strings.Join(targets, ", "))
fmt.Fprintf(w, "chunk size: %d\n", cfg.ChunkSize)
fmt.Fprintf(w, "shard by: %s\n", cfg.ShardBy)
if cfg.ShardBy != "none" {
fmt.Fprintf(w, "shard rows: %d\n", cfg.ShardRows)
}
if a.probe.shard != nil {
fmt.Fprintf(w, "shard column: %s (%s)\n", a.probe.shard.Column, a.probe.shard.Type)
fmt.Fprintf(w, "shard time zone: %s\n", cfg.ShardTimeZone)
}
fmt.Fprintf(w, "format: %s\n", cfg.Format)
if comp.Name == "none" {
fmt.Fprintf(w, "compress: none\n")
} else {
fmt.Fprintf(w, "compress: %s (level %d)\n", comp.Name, comp.Level)
}
if cfg.InsertMode != "" {
fmt.Fprintf(w, "insert mode: %s\n", cfg.InsertMode)
}
fmt.Fprintf(w, "max lag: %s (check interval %s)\n", cfg.MaxLag, cfg.CheckInterval)
fmt.Fprintf(w, "retries: %d\n", cfg.Retries)
fmt.Fprintf(w, "safe auto inc: %t\n", cfg.SafeAutoIncrement)
fmt.Fprintf(w, "config hash: %s\n", a.cfgHash)
fmt.Fprintln(w, "\n== index probe ==")
fmt.Fprintf(w, "scan key: %s (%s)\n", a.probe.key.Name, strings.Join(a.probe.key.Columns, ", "))
if a.probe.path == "A" {
fmt.Fprintf(w, "scan path: A (keyset chunks using index %s)\n", a.probe.joint.Name)
} else if a.probe.path == "B" {
if a.probe.single != nil {
fmt.Fprintf(w, "scan path: B (day-at-a-time using index %s)\n", a.probe.single.Name)
} else {
fmt.Fprintln(w, "scan path: B (day-at-a-time, full table scan allowed)")
}
} else {
fmt.Fprintln(w, "scan path: primary key seek")
}
fmt.Fprintln(w, "\n== first SELECT ==")
for _, q := range a.dryRunSelects() {
fmt.Fprintf(w, " %s\n", q)
}
fmt.Fprintln(w, "\n== write statements ==")
if cfg.DestDSN != "" {
fmt.Fprintf(w, " %s\n", a.dryRunInsert())
}
if cfg.Purge {
p := purge.New(nil, a.srcCfg.DBName, cfg.Table, a.probe.key.Columns, cfg.Where, cfg.BulkDelete, cfg.SafeAutoIncrement, false)
for _, q := range p.DryRunSQL() {
fmt.Fprintf(w, " %s\n", q)
}
}
if cfg.HasFile() {
fmt.Fprintf(w, " file: %s/[...]<seq>.<pk_lo>-<pk_hi>.%s%s (written as .partial until sealed)\n",
cfg.FileDir, cfg.Format, comp.Ext)
}
fmt.Fprintln(w, "\nnote: file output is at-least-once; a crash and rerun may duplicate rows in shard files")
return nil
}
func (a *App) dryRunSelects() []string {
cols := sqlutil.Columns(a.probe.selectCols)
table := sqlutil.Qualified(a.srcCfg.DBName, a.cfg.Table)
where := a.cfg.Where
key := sqlutil.Columns(a.probe.key.Columns)
limit := fmt.Sprintf(" LIMIT %d", a.cfg.ChunkSize)
switch a.probe.path {
case "A":
shard := sqlutil.Ident(a.probe.shard.Column)
colsWithShard := sqlutil.Columns(append([]string{a.probe.shard.Column}, a.probe.key.Columns...))
return []string{
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s IS NULL ORDER BY %s%s", cols, table, where, shard, key, limit),
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s IS NOT NULL AND (%s) > (?, ...) ORDER BY %s%s", cols, table, where, shard, colsWithShard, colsWithShard, limit),
}
case "B":
shard := sqlutil.Ident(a.probe.shard.Column)
colsWithShard := sqlutil.Columns(append([]string{a.probe.shard.Column}, a.probe.key.Columns...))
return []string{
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s IS NULL ORDER BY %s", cols, table, where, shard, key),
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s >= ? AND %s < ? ORDER BY %s", cols, table, where, shard, shard, colsWithShard),
}
default:
return []string{fmt.Sprintf("SELECT %s FROM %s WHERE (%s) ORDER BY %s%s", cols, table, where, key, limit)}
}
}
func (a *App) dryRunInsert() string {
destCfg, err := a.cfg.DestConfig()
dbName := ""
if err == nil && destCfg != nil {
dbName = destCfg.DBName
}
verb := "INSERT INTO"
switch a.cfg.InsertMode {
case "ignore":
verb = "INSERT IGNORE INTO"
case "replace":
verb = "REPLACE INTO"
}
names := make([]string, len(a.probe.outCols))
for i, c := range a.probe.outCols {
names[i] = c.Name
}
tuple := sqlutil.ParamTuple(len(names))
return fmt.Sprintf("%s %s (%s) VALUES %s, %s, ...",
verb, sqlutil.Qualified(dbName, a.cfg.DestTable), sqlutil.Columns(names), tuple, tuple)
}
func redactDSN(dsn string) string {
c, err := mysql.ParseDSN(dsn)
if err != nil {
return "<invalid dsn>"
}
c.Passwd = ""
return c.FormatDSN()
}
+64
View File
@@ -0,0 +1,64 @@
package app
import (
"context"
"database/sql"
"fmt"
"time"
"archer/internal/estimate"
"archer/internal/progress"
"archer/internal/sqlutil"
)
// estimateTimeout bounds the one-off MIN/MAX query at startup.
const estimateTimeout = 2 * time.Second
// estimateSpec decides which cheap position metric this run can expose.
func (a *App) estimateSpec() estimate.Spec {
if a.cfg.IsDateShard() && a.probe.shard != nil {
return estimate.Spec{
Kind: "day",
Column: a.probe.shard.Column,
ShardKind: a.probe.shard.Kind,
Unit: a.probe.shard.Unit,
Loc: a.loc,
}
}
types := make([]string, len(a.probe.key.Columns))
for i, name := range a.probe.key.Columns {
c, _ := a.probe.table.Column(name)
types[i] = c.DataType
}
col, ok := estimate.KeyColumn(a.probe.key.Columns, types)
if !ok {
return estimate.Spec{}
}
return estimate.Spec{Kind: "key", Column: col}
}
// queryEstimate runs the bounded range query on a dedicated connection and
// silently returns nil when the metric does not apply or the query fails.
func (a *App) queryEstimate(ctx context.Context) *progress.Estimate {
spec := a.estimateSpec()
if spec.Kind == "" {
return nil
}
conn, err := a.srcDB.Conn(ctx)
if err != nil {
return nil
}
defer conn.Close()
query := func(qctx context.Context, column string) (sql.NullString, sql.NullString, error) {
col := sqlutil.Ident(column)
q := fmt.Sprintf("SELECT MIN(%s), MAX(%s) FROM %s WHERE (%s)",
col, col, sqlutil.Qualified(a.srcCfg.DBName, a.cfg.Table), a.cfg.Where)
if spec.Kind == "day" {
q += " AND " + col + " IS NOT NULL"
}
var minRaw, maxRaw sql.NullString
err := conn.QueryRowContext(qctx, q).Scan(&minRaw, &maxRaw)
return minRaw, maxRaw, err
}
return estimate.Bounded(ctx, query, spec, estimateTimeout)
}
+918
View File
@@ -0,0 +1,918 @@
package app
import (
"bytes"
"compress/gzip"
"context"
"database/sql"
"encoding/csv"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"testing"
"github.com/go-sql-driver/mysql"
"github.com/klauspost/compress/zstd"
"github.com/ulikunitz/xz"
"archer/internal/cli"
)
func testDSN(t *testing.T) string {
d := os.Getenv("ARCHER_TEST_DSN")
if d == "" {
t.Skip("ARCHER_TEST_DSN is not set; skipping MySQL integration test")
}
return d
}
func TestMain(m *testing.M) {
if dsn := os.Getenv("ARCHER_TEST_DSN"); dsn != "" {
cfg, err := mysql.ParseDSN(dsn)
if err == nil && cfg.DBName != "" {
dbName := cfg.DBName
cfg.DBName = ""
db, err := sql.Open("mysql", cfg.FormatDSN())
if err == nil {
_, _ = db.Exec("CREATE DATABASE IF NOT EXISTS `" + dbName + "`")
_, _ = db.Exec("SET GLOBAL local_infile=ON")
_, _ = db.Exec("SET GLOBAL sql_mode=''")
db.Close()
}
}
}
os.Exit(m.Run())
}
func openRoot(t *testing.T, dsn string) *sql.DB {
t.Helper()
cfg, err := mysql.ParseDSN(dsn)
if err != nil {
t.Fatal(err)
}
db, err := sql.Open("mysql", cfg.FormatDSN())
if err != nil {
t.Fatal(err)
}
if err := db.Ping(); err != nil {
t.Fatal(err)
}
return db
}
func execSQL(t *testing.T, db *sql.DB, q string, args ...any) {
t.Helper()
if _, err := db.Exec(q, args...); err != nil {
t.Fatalf("exec %q: %v", q, err)
}
}
var stdoutMu sync.Mutex
var stderrMu sync.Mutex
func captureStdout(fn func() error) (string, error) {
stdoutMu.Lock()
defer stdoutMu.Unlock()
old := os.Stdout
r, w, err := os.Pipe()
if err != nil {
return "", err
}
os.Stdout = w
var buf bytes.Buffer
done := make(chan struct{})
go func() {
_, _ = io.Copy(&buf, r)
close(done)
}()
runErr := fn()
_ = w.Close()
os.Stdout = old
<-done
return buf.String(), runErr
}
func captureStderr(fn func() error) (string, error) {
stderrMu.Lock()
defer stderrMu.Unlock()
old := os.Stderr
r, w, err := os.Pipe()
if err != nil {
return "", err
}
os.Stderr = w
var buf bytes.Buffer
done := make(chan struct{})
go func() {
_, _ = io.Copy(&buf, r)
close(done)
}()
runErr := fn()
_ = w.Close()
os.Stderr = old
<-done
return buf.String(), runErr
}
func runCLI(t *testing.T, ctx context.Context, args ...string) (string, error) {
t.Helper()
cfg, err := cli.Parse(args, io.Discard)
if err != nil {
return "", err
}
return captureStdout(func() error { return Run(ctx, cfg) })
}
func runCLIStderr(t *testing.T, ctx context.Context, args ...string) (string, error) {
t.Helper()
cfg, err := cli.Parse(args, io.Discard)
if err != nil {
return "", err
}
return captureStderr(func() error { return Run(ctx, cfg) })
}
func baseCLI(dsn, table, out string, extra ...string) []string {
args := []string{
"--source", dsn,
"--table", table,
"--where", "1=1",
"--file", out,
}
return append(args, extra...)
}
func listFiles(t *testing.T, dir, pattern string) []string {
t.Helper()
matches, err := filepath.Glob(filepath.Join(dir, pattern))
if err != nil {
t.Fatal(err)
}
sort.Strings(matches)
return matches
}
func dumpIDs(t *testing.T, path string) []int {
t.Helper()
b, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var ids []int
for _, line := range strings.Split(strings.TrimRight(string(b), "\n"), "\n") {
if line == "" {
continue
}
id, err := strconv.Atoi(strings.SplitN(line, "\t", 2)[0])
if err != nil {
t.Fatalf("%s: bad line %q", path, line)
}
ids = append(ids, id)
}
return ids
}
func TestIntegrationCountShards(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_count")
execSQL(t, db, "CREATE TABLE it_count (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
var b strings.Builder
b.WriteString("INSERT INTO it_count (id, v) VALUES ")
for i := 1; i <= 25; i++ {
if i > 1 {
b.WriteString(",")
}
fmt.Fprintf(&b, "(%d,'v%d')", i, i)
}
execSQL(t, db, b.String())
out := t.TempDir()
_, err := runCLI(t, context.Background(), baseCLI(dsn, "it_count", out,
"--shard-by", "count", "--shard-rows", "10", "--chunk-size", "7", "--format", "dump")...)
if err != nil {
t.Fatal(err)
}
files := listFiles(t, out, "*.dump")
if len(files) != 3 {
t.Fatalf("got %d shard files: %v", len(files), files)
}
wantNames := []string{"000001.1-10.dump", "000002.11-20.dump", "000003.21-25.dump"}
var got []int
for i, f := range files {
if filepath.Base(f) != wantNames[i] {
t.Fatalf("file %d = %s want %s", i, filepath.Base(f), wantNames[i])
}
got = append(got, dumpIDs(t, f)...)
}
if len(got) != 25 {
t.Fatalf("got %d rows", len(got))
}
for i, id := range got {
if id != i+1 {
t.Fatalf("row %d = %d, shards are not contiguous", i, id)
}
}
entries := readManifest(t, out)
if len(entries) != 3 {
t.Fatalf("manifest entries %d", len(entries))
}
for i, e := range entries {
if e.Seq != int64(i+1) || e.Rows != int64(len(dumpIDs(t, filepath.Join(out, e.Path)))) {
t.Fatalf("entry %d %+v", i, e)
}
if e.Bytes == 0 || e.SHA == "" {
t.Fatalf("entry %d missing bytes/hash", i)
}
}
if _, err := os.Stat(filepath.Join(out, "state.json")); err != nil {
t.Fatal(err)
}
var complete bool
st, err := os.ReadFile(filepath.Join(out, "state.json"))
if err != nil {
t.Fatal(err)
}
complete = bytes.Contains(st, []byte(`"complete": true`))
if !complete {
t.Fatalf("state not complete: %s", st)
}
}
type manifestEntry struct {
Seq int64 `json:"seq"`
Path string `json:"path"`
Date string `json:"date"`
Rows int64 `json:"rows"`
Bytes int64 `json:"bytes"`
SHA string `json:"sha256"`
PKLo string `json:"pk_lo"`
PKHi string `json:"pk_hi"`
}
func readManifest(t *testing.T, dir string) []manifestEntry {
t.Helper()
b, err := os.ReadFile(filepath.Join(dir, "manifest.jsonl"))
if err != nil {
t.Fatal(err)
}
var out []manifestEntry
for _, line := range strings.Split(strings.TrimSpace(string(b)), "\n") {
if line == "" {
continue
}
var e manifestEntry
if err := json.Unmarshal([]byte(line), &e); err != nil {
t.Fatal(err)
}
out = append(out, e)
}
return out
}
func TestIntegrationDatePathsAgree(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
for _, spec := range []struct{ name, index string }{
{"it_date_a", "KEY da (created_at, id)"},
{"it_date_b", "KEY db (created_at)"},
} {
execSQL(t, db, "DROP TABLE IF EXISTS "+spec.name)
execSQL(t, db, "CREATE TABLE "+spec.name+` (
id BIGINT NOT NULL,
v VARCHAR(32),
created_at DATETIME NULL,
PRIMARY KEY (id),
`+spec.index+`)`)
execSQL(t, db, `INSERT INTO `+spec.name+` (id, v, created_at) VALUES
(1,'a','2025-01-01 10:00:00'),
(2,'b','2025-01-01 12:00:00'),
(3,'c','2025-01-02 01:00:00'),
(4,'d',NULL),
(5,'e','2025-01-01 11:00:00'),
(6,'f','2025-01-03 00:00:00')`)
}
outA := t.TempDir()
outB := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_a", outA,
"--shard-by", "date", "--shard-column", "created_at", "--format", "dump", "--chunk-size", "2")...); err != nil {
t.Fatal(err)
}
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_b", outB,
"--shard-by", "date", "--shard-column", "created_at", "--format", "dump", "--chunk-size", "2")...); err != nil {
t.Fatal(err)
}
a := readTree(t, outA)
b := readTree(t, outB)
if len(a) != len(b) {
t.Fatalf("file counts differ: %d vs %d (%v / %v)", len(a), len(b), a, b)
}
for k, va := range a {
if b[k] != va {
t.Fatalf("%s:\nA=%q\nB=%q", k, va, b[k])
}
}
if _, ok := a["date=_null/000001.4-4.dump"]; !ok {
t.Fatalf("NULL row not routed to _null: %v", a)
}
if len(a) != 4 {
t.Fatalf("expected 4 shards (null + 3 days), got %v", a)
}
}
func readTree(t *testing.T, dir string) map[string]string {
t.Helper()
out := map[string]string{}
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
if err != nil || d.IsDir() {
return err
}
if !strings.HasSuffix(d.Name(), ".dump") {
return nil
}
rel, _ := filepath.Rel(dir, path)
b, err := os.ReadFile(path)
if err != nil {
return err
}
out[filepath.ToSlash(rel)] = string(b)
return nil
})
if err != nil {
t.Fatal(err)
}
return out
}
func TestIntegrationZeroDateAndNull(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_zero")
execSQL(t, db, `CREATE TABLE it_zero (
id BIGINT NOT NULL,
d DATE NULL,
PRIMARY KEY (id),
KEY dz (d, id))`)
execSQL(t, db, `INSERT INTO it_zero (id, d) VALUES
(1, NULL), (2, '0000-00-00'), (3, '2024-01-01'), (4, '2024-01-01')`)
out := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_zero", out,
"--shard-by", "date", "--shard-column", "d")...); err == nil {
t.Fatal("expected zero-date error")
} else if !strings.Contains(err.Error(), "zero date") {
t.Fatalf("unexpected error: %v", err)
}
out = t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_zero", out,
"--shard-by", "date", "--shard-column", "d", "--zero-date", "bucket")...); err != nil {
t.Fatal(err)
}
tree := readTree(t, out)
for _, want := range []string{
"date=_null/000001.1-1.dump",
"date=0000-00-00/000001.2-2.dump",
"date=2024-01-01/000001.3-4.dump",
} {
if _, ok := tree[want]; !ok {
t.Fatalf("missing %s in %v", want, tree)
}
}
}
func TestIntegrationEpochUnits(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_epoch")
execSQL(t, db, `CREATE TABLE it_epoch (
id BIGINT NOT NULL,
ts BIGINT NOT NULL,
PRIMARY KEY (id),
KEY e (ts, id))`)
execSQL(t, db, `INSERT INTO it_epoch (id, ts) VALUES
(1, 1717200000), (2, 1717200001)`)
for _, tc := range []struct {
unit string
vals string
}{
{"epoch-seconds", "(3, 1717200000), (4, 1717200001)"},
{"epoch-milliseconds", "(3, 1717200000000), (4, 1717200001000)"},
{"epoch-microseconds", "(3, 1717200000000000), (4, 1717200001000000)"},
} {
table := "it_epoch_" + strings.Split(tc.unit, "-")[1]
execSQL(t, db, "DROP TABLE IF EXISTS "+table)
execSQL(t, db, `CREATE TABLE `+table+` (
id BIGINT NOT NULL, ts BIGINT NOT NULL,
PRIMARY KEY (id), KEY e (ts, id))`)
execSQL(t, db, "INSERT INTO "+table+" (id, ts) VALUES "+tc.vals)
out := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, table, out,
"--shard-by", "date", "--shard-column", "ts", "--shard-column-type", tc.unit,
"--shard-time-zone", "UTC")...); err != nil {
t.Fatalf("%s: %v", tc.unit, err)
}
tree := readTree(t, out)
if len(tree) != 1 {
t.Fatalf("%s: expected one date bucket, got %v", tc.unit, tree)
}
for k := range tree {
if !strings.Contains(k, "date=2024-06-01") {
t.Fatalf("%s: wrong bucket %s", tc.unit, k)
}
}
}
}
func TestIntegrationNoIndexFailFast(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_noindex")
execSQL(t, db, `CREATE TABLE it_noindex (
id BIGINT NOT NULL, d DATETIME NULL, PRIMARY KEY (id))`)
execSQL(t, db, `INSERT INTO it_noindex (id, d) VALUES (1,'2024-01-01 00:00:00')`)
out := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_noindex", out,
"--shard-by", "date", "--shard-column", "d")...); err == nil {
t.Fatal("expected no-index error")
} else if !strings.Contains(err.Error(), "no usable index") {
t.Fatalf("unexpected error: %v", err)
}
out = t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_noindex", out,
"--shard-by", "date", "--shard-column", "d", "--allow-full-scan")...); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(out, "date=2024-01-01")); err != nil {
t.Fatal(err)
}
}
func TestIntegrationDryRunCreatesNothing(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_dry")
execSQL(t, db, "CREATE TABLE it_dry (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "INSERT INTO it_dry (id, v) VALUES (1,'a')")
out := filepath.Join(t.TempDir(), "never-created")
stdout, err := runCLI(t, context.Background(), baseCLI(dsn, "it_dry", out, "--dry-run")...)
if err != nil {
t.Fatal(err)
}
if _, statErr := os.Stat(out); !os.IsNotExist(statErr) {
t.Fatal("dry-run created the output directory")
}
if !strings.Contains(stdout, "== first SELECT ==") || !strings.Contains(stdout, "config hash") {
t.Fatalf("dry-run output incomplete:\n%s", stdout)
}
}
func TestIntegrationResumeAfterInterrupt(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_resume")
execSQL(t, db, "CREATE TABLE it_resume (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
var b strings.Builder
b.WriteString("INSERT INTO it_resume (id, v) VALUES ")
const total = 20000
for i := 1; i <= total; i++ {
if i > 1 {
b.WriteString(",")
}
fmt.Fprintf(&b, "(%d,'v')", i)
}
execSQL(t, db, b.String())
out := t.TempDir()
args := baseCLI(dsn, "it_resume", out, "--shard-by", "count", "--shard-rows", "137",
"--chunk-size", "50", "--progress", "100")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
stderrMu.Lock()
old := os.Stderr
r, w, _ := os.Pipe()
os.Stderr = w
cfg, err := cli.Parse(args, io.Discard)
if err != nil {
os.Stderr = old
stderrMu.Unlock()
t.Fatal(err)
}
firstErr := make(chan error, 1)
go func() { firstErr <- Run(ctx, cfg) }()
go func() {
buf := make([]byte, 4096)
for {
n, err := r.Read(buf)
if n > 0 && strings.Contains(string(buf[:n]), "progress:") {
cancel()
}
if err != nil {
return
}
}
}()
err = <-firstErr
_ = w.Close()
os.Stderr = old
stderrMu.Unlock()
if err == nil {
t.Fatal("first run should have been interrupted")
}
if _, err := runCLI(t, context.Background(), args...); err != nil {
t.Fatal(err)
}
seen := map[int]bool{}
for _, f := range listFiles(t, out, "*.dump") {
for _, id := range dumpIDs(t, f) {
seen[id] = true
}
}
for i := 1; i <= total; i++ {
if !seen[i] {
t.Fatalf("row %d missing after resume", i)
}
}
st, err := os.ReadFile(filepath.Join(out, "state.json"))
if err != nil {
t.Fatal(err)
}
if !bytes.Contains(st, []byte(`"complete": true`)) {
t.Fatalf("state not complete: %s", st)
}
var saved struct {
RowsDone int64 `json:"rows_done"`
}
if err := json.Unmarshal(st, &saved); err != nil {
t.Fatal(err)
}
if saved.RowsDone != total {
t.Fatalf("rows_done = %d after resume, want cumulative %d", saved.RowsDone, total)
}
if _, err := filepath.Glob(filepath.Join(out, "*.partial")); err != nil {
t.Fatal(err)
}
}
func TestIntegrationProgressModes(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_progress")
execSQL(t, db, "CREATE TABLE it_progress (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "INSERT INTO it_progress (id, v) VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e')")
for _, extra := range [][]string{
nil,
{"--progress", "off"},
{"--progress", "0"},
} {
out := t.TempDir()
args := baseCLI(dsn, "it_progress", out, extra...)
errOut, err := runCLIStderr(t, context.Background(), args...)
if err != nil {
t.Fatalf("%v: %v", extra, err)
}
if strings.Contains(errOut, "progress:") || strings.Contains(errOut, "done rows=") {
t.Fatalf("%v: unexpected progress output %q", extra, errOut)
}
}
out := t.TempDir()
errOut, err := runCLIStderr(t, context.Background(), baseCLI(dsn, "it_progress", out, "--progress", "on")...)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(errOut, "done rows=5") {
t.Fatalf("forced progress missing done summary: %q", errOut)
}
if strings.Contains(errOut, "\r") {
t.Fatalf("redirected progress must not use carriage returns: %q", errOut)
}
out = t.TempDir()
errOut, err = runCLIStderr(t, context.Background(), baseCLI(dsn, "it_progress", out, "--progress", "1")...)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(errOut, "progress: rows=5") {
t.Fatalf("row-threshold progress line missing: %q", errOut)
}
}
func TestIntegrationEstimateMetrics(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_estimate")
execSQL(t, db, `CREATE TABLE it_estimate (
id BIGINT NOT NULL,
v VARCHAR(32),
created_at DATETIME NULL,
PRIMARY KEY (id),
KEY de (created_at, id))`)
execSQL(t, db, `INSERT INTO it_estimate (id, v, created_at) VALUES
(1,'a','2025-01-01 01:00:00'),
(2,'b','2025-01-01 02:00:00'),
(3,'c','2025-01-02 01:00:00'),
(4,'d','2025-01-03 01:00:00'),
(5,'e','2025-01-03 02:00:00')`)
out := t.TempDir()
errOut, err := runCLIStderr(t, context.Background(), baseCLI(dsn, "it_estimate", out, "--progress", "1")...)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(errOut, "estimate=key") {
t.Fatalf("primary key estimate missing: %q", errOut)
}
out = t.TempDir()
errOut, err = runCLIStderr(t, context.Background(),
baseCLI(dsn, "it_estimate", out, "--progress", "1", "--shard-by", "date", "--shard-column", "created_at")...)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(errOut, "estimate=day") {
t.Fatalf("day estimate missing: %q", errOut)
}
}
func TestIntegrationImportDump(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_import_src")
execSQL(t, db, "CREATE TABLE it_import_src (id BIGINT NOT NULL, v VARCHAR(64), PRIMARY KEY (id))")
execSQL(t, db, `INSERT INTO it_import_src (id, v) VALUES
(1,'plain'), (2,'tab\there'), (3,'line\nbreak'), (4,'back\\slash')`)
for _, comp := range []string{"none", "gzip", "zstd", "xz"} {
out := t.TempDir()
args := baseCLI(dsn, "it_import_src", out, "--format", "dump", "--compress", comp)
if _, err := runCLI(t, context.Background(), args...); err != nil {
t.Fatalf("%s: %v", comp, err)
}
files := listFiles(t, out, "*.dump*")
if len(files) != 1 {
t.Fatalf("%s: files %v", comp, files)
}
loadPath := files[0]
if comp != "none" {
loadPath = filepath.Join(t.TempDir(), "decompressed.dump")
if err := decompressFile(files[0], loadPath, comp); err != nil {
t.Fatal(err)
}
}
execSQL(t, db, "DROP TABLE IF EXISTS it_import_dst")
execSQL(t, db, "CREATE TABLE it_import_dst (id BIGINT NOT NULL, v VARCHAR(64), PRIMARY KEY (id))")
loadLocal(t, dsn, loadPath, "it_import_dst")
var n int
if err := db.QueryRow("SELECT COUNT(*) FROM it_import_dst").Scan(&n); err != nil {
t.Fatal(err)
}
if n != 4 {
t.Fatalf("%s: loaded %d rows", comp, n)
}
var v string
if err := db.QueryRow("SELECT v FROM it_import_dst WHERE id=3").Scan(&v); err != nil {
t.Fatal(err)
}
if v != "line\nbreak" {
t.Fatalf("%s: round trip mismatch %q", comp, v)
}
}
}
func decompressFile(src, dst, comp string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
var r io.Reader
switch comp {
case "gzip":
gr, err := gzip.NewReader(in)
if err != nil {
return err
}
defer gr.Close()
r = gr
case "zstd":
zr, err := zstd.NewReader(in)
if err != nil {
return err
}
defer zr.Close()
r = zr
case "xz":
xr, err := xz.NewReader(in)
if err != nil {
return err
}
r = xr
default:
r = in
}
out, err := os.Create(dst)
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, r)
return err
}
func loadLocal(t *testing.T, dsn, path, table string) {
t.Helper()
cfg, err := mysql.ParseDSN(dsn)
if err != nil {
t.Fatal(err)
}
cfg.AllowAllFiles = true
db, err := sql.Open("mysql", cfg.FormatDSN())
if err != nil {
t.Fatal(err)
}
defer db.Close()
p := filepath.ToSlash(path)
p = strings.ReplaceAll(p, `'`, `''`)
q := fmt.Sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE %s", p, table)
if _, err := db.Exec(q); err != nil {
t.Fatalf("load data: %v", err)
}
}
func TestIntegrationCSVAndJSONL(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_fmt")
execSQL(t, db, `CREATE TABLE it_fmt (
id BIGINT NOT NULL, name VARCHAR(64), price DECIMAL(10,2), created DATETIME,
data BLOB, PRIMARY KEY (id))`)
execSQL(t, db, `INSERT INTO it_fmt (id, name, price, created, data) VALUES
(1,'a,\"b\"',12.30,'2024-01-02 03:04:05',X'6869'), (2,'',NULL,NULL,NULL)`)
// CSV
out := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_fmt", out, "--format", "csv")...); err != nil {
t.Fatal(err)
}
files := listFiles(t, out, "*.csv")
if len(files) != 1 {
t.Fatalf("csv files %v", files)
}
f, err := os.Open(files[0])
if err != nil {
t.Fatal(err)
}
recs, err := csv.NewReader(f).ReadAll()
f.Close()
if err != nil {
t.Fatal(err)
}
if len(recs) != 3 {
t.Fatalf("csv records %d", len(recs))
}
if recs[0][0] != "id" || recs[0][1] != "name" {
t.Fatalf("header %v", recs[0])
}
if recs[1][1] != `a,"b"` || recs[1][2] != "12.30" {
t.Fatalf("row %v", recs[1])
}
if recs[2][2] != `\N` {
t.Fatalf("NULL decimal %q", recs[2][2])
}
// JSONL
out = t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_fmt", out, "--format", "jsonl")...); err != nil {
t.Fatal(err)
}
files = listFiles(t, out, "*.jsonl")
var shardFiles []string
for _, f := range files {
if filepath.Base(f) != "manifest.jsonl" {
shardFiles = append(shardFiles, f)
}
}
files = shardFiles
if len(files) != 1 {
t.Fatalf("jsonl files %v", files)
}
b, err := os.ReadFile(files[0])
if err != nil {
t.Fatal(err)
}
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
if len(lines) != 2 {
t.Fatalf("jsonl lines %d", len(lines))
}
var obj map[string]any
if err := json.Unmarshal([]byte(lines[0]), &obj); err != nil {
t.Fatal(err)
}
if obj["price"] != "12.30" {
t.Fatalf("price %v", obj["price"])
}
if obj["created"] != "2024-01-02T03:04:05Z" {
t.Fatalf("created %v", obj["created"])
}
if obj["data"] != "aGk=" {
t.Fatalf("data %v", obj["data"])
}
var second map[string]any
if err := json.Unmarshal([]byte(lines[1]), &second); err != nil {
t.Fatal(err)
}
if second["price"] != nil || second["created"] != nil {
t.Fatalf("null mapping %v", second)
}
}
func TestIntegrationPurgeAndDest(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_purge_src")
execSQL(t, db, "DROP TABLE IF EXISTS it_purge_dst")
execSQL(t, db, "CREATE TABLE it_purge_src (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "CREATE TABLE it_purge_dst (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "INSERT INTO it_purge_src (id, v) VALUES (1,'a'),(2,'b'),(3,'c')")
out := t.TempDir()
args := []string{
"--source", dsn, "--table", "it_purge_src", "--where", "1=1",
"--file", out, "--dest", dsn, "--dest-table", "it_purge_dst",
"--purge", "--chunk-size", "2",
}
if _, err := runCLI(t, context.Background(), args...); err != nil {
t.Fatal(err)
}
var srcCount, dstCount int
db.QueryRow("SELECT COUNT(*) FROM it_purge_src").Scan(&srcCount)
db.QueryRow("SELECT COUNT(*) FROM it_purge_dst").Scan(&dstCount)
if srcCount != 0 || dstCount != 3 {
t.Fatalf("src=%d dst=%d", srcCount, dstCount)
}
if len(listFiles(t, out, "*.dump")) != 1 {
t.Fatal("expected one shard")
}
}
func TestIntegrationCheckpointConfigMismatch(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_mismatch")
execSQL(t, db, "CREATE TABLE it_mismatch (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "INSERT INTO it_mismatch (id, v) VALUES (1,'a'),(2,'b')")
out := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_mismatch", out)...); err != nil {
t.Fatal(err)
}
// Remove the complete flag so resume logic runs.
stPath := filepath.Join(out, "state.json")
b, _ := os.ReadFile(stPath)
b = bytes.Replace(b, []byte(`"complete": true`), []byte(`"complete": false`), 1)
if err := os.WriteFile(stPath, b, 0o644); err != nil {
t.Fatal(err)
}
args := baseCLI(dsn, "it_mismatch", out)
for i := range args {
if args[i] == "1=1" {
args[i] = "id > 0"
}
}
if _, err := runCLI(t, context.Background(), args...); err == nil {
t.Fatal("expected config mismatch error")
} else if !strings.Contains(err.Error(), "different configuration") {
t.Fatalf("unexpected error: %v", err)
}
args = append(args, "--force-resume")
if _, err := runCLI(t, context.Background(), args...); err != nil {
t.Fatalf("force resume: %v", err)
}
}
+129
View File
@@ -0,0 +1,129 @@
package app
import (
"context"
"database/sql"
"fmt"
"strings"
"time"
"archer/internal/archererr"
"archer/internal/cli"
"archer/internal/row"
"archer/internal/schema"
)
// probeResult is everything learned about the source table needed to plan the
// scan and the encoders.
type probeResult struct {
table *schema.Table
key schema.KeyInfo
shard *schema.ShardSpec
path string // pk | A | B
joint *schema.Index
single *schema.Index
selectCols []string
selectMeta []row.Column
keyIdx []int
shardIdx int
outCols []row.Column
outIdx []int
}
func probeTable(ctx context.Context, db *sql.DB, cfg *cli.Config, dbName string, loc *time.Location) (*probeResult, error) {
table, err := schema.Load(ctx, db, dbName, cfg.Table)
if err != nil {
return nil, archererr.Wrap(archererr.KindValidation, err)
}
key, err := table.ChooseKey(cfg.Index)
if err != nil {
return nil, archererr.Wrap(archererr.KindValidation, err)
}
outCols, err := table.OutputColumns(cfg.Columns)
if err != nil {
return nil, archererr.Wrap(archererr.KindValidation, err)
}
p := &probeResult{table: table, key: key, path: "pk", shardIdx: -1, outCols: outCols}
if cfg.IsDateShard() {
spec, err := schema.ResolveShardColumn(ctx, db, table, cfg.ShardColumn, cfg.ShardColumnType, cfg.Where, loc)
if err != nil {
return nil, archererr.Wrap(archererr.KindValidation, err)
}
p.shard = spec
prefix := append([]string{spec.Column}, key.Columns...)
p.joint = table.FindIndexPrefix(prefix)
p.single = table.FindIndexPrefix([]string{spec.Column})
switch {
case p.joint != nil:
p.path = "A"
case p.single != nil:
p.path = "B"
case cfg.AllowFullScan:
p.path = "B"
default:
return nil, archererr.Newf(archererr.KindValidation,
"no usable index on %s for date sharding\n joint: create an index on (%s) for keyset chunks\n or: create an index on (%s) for day-at-a-time streaming\n or: pass --allow-full-scan to accept a full table scan\n%s",
table.Name, strings.Join(prefix, ", "), spec.Column, describeIndexes(table))
}
}
needed := map[string]bool{}
for _, c := range outCols {
needed[strings.ToLower(c.Name)] = true
}
for _, c := range key.Columns {
needed[strings.ToLower(c)] = true
}
if p.shard != nil {
needed[strings.ToLower(p.shard.Column)] = true
}
for _, c := range table.Columns {
if needed[strings.ToLower(c.Name)] {
p.selectCols = append(p.selectCols, c.Name)
p.selectMeta = append(p.selectMeta, row.Column{Name: c.Name, Type: c.DataType})
}
}
pos := map[string]int{}
for i, name := range p.selectCols {
pos[name] = i
}
p.keyIdx = make([]int, len(key.Columns))
for i, c := range key.Columns {
idx, ok := pos[c]
if !ok {
return nil, archererr.Newf(archererr.KindValidation, "internal: key column %s missing from select list", c)
}
p.keyIdx[i] = idx
}
if p.shard != nil {
idx, ok := pos[p.shard.Column]
if !ok {
return nil, archererr.Newf(archererr.KindValidation, "internal: shard column %s missing from select list", p.shard.Column)
}
p.shardIdx = idx
}
p.outIdx = make([]int, len(outCols))
for i, c := range outCols {
p.outIdx[i] = pos[c.Name]
}
return p, nil
}
func describeIndexes(t *schema.Table) string {
if len(t.Indexes) == 0 {
return " indexes: (none)"
}
var b strings.Builder
b.WriteString(" indexes:\n")
for _, ix := range t.Indexes {
kind := "index"
if ix.Name == "PRIMARY" {
kind = "primary key"
} else if ix.Unique {
kind = "unique index"
}
fmt.Fprintf(&b, " - %s (%s) %s\n", ix.Name, strings.Join(ix.Columns, ", "), kind)
}
return strings.TrimRight(b.String(), "\n")
}
+43
View File
@@ -0,0 +1,43 @@
package app
import (
"bytes"
"os"
"testing"
"archer/internal/checkpoint"
"archer/internal/cli"
)
func TestStateConfigStoresRedactedDSN(t *testing.T) {
const source = "u:s3cr3t@tcp(h:3306)/db?charset=utf8mb4"
cfg, err := cli.Parse([]string{"--source", source, "--table", "t", "--where", "id > 0", "--file", "out"}, &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
hash, fields, err := cfg.Fingerprint()
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
if err := checkpoint.Save(dir, &checkpoint.State{ConfigHash: hash, Config: fields}); err != nil {
t.Fatal(err)
}
st, err := checkpoint.Load(dir)
if err != nil {
t.Fatal(err)
}
want := redactDSN(source)
if st.Config["source"] != want {
t.Fatalf("config.source = %q, want %q", st.Config["source"], want)
}
raw, err := os.ReadFile(checkpoint.Path(dir))
if err != nil {
t.Fatal(err)
}
if bytes.Contains(raw, []byte("s3cr3t")) {
t.Fatal("state.json must not contain the password")
}
}
+72
View File
@@ -0,0 +1,72 @@
// Package archererr classifies runtime errors so the process can pick an exit
// code and so callers can distinguish interrupted runs.
package archererr
import (
"errors"
"fmt"
)
type Kind int
const (
KindOther Kind = iota
KindConfig
KindConnection
KindWrite
KindValidation
KindInterrupted
)
// Error wraps an error with a classification.
type Error struct {
Kind Kind
Err error
}
func (e *Error) Error() string { return e.Err.Error() }
func (e *Error) Unwrap() error { return e.Err }
func Wrap(kind Kind, err error) error {
if err == nil {
return nil
}
var existing *Error
if errors.As(err, &existing) {
return err
}
return &Error{Kind: kind, Err: err}
}
func Newf(kind Kind, format string, args ...any) error {
return &Error{Kind: kind, Err: fmt.Errorf(format, args...)}
}
func KindOf(err error) Kind {
var e *Error
if errors.As(err, &e) {
return e.Kind
}
return KindOther
}
// ExitCode maps a classification to a process exit code.
func ExitCode(err error) int {
if err == nil {
return 0
}
switch KindOf(err) {
case KindConfig:
return 2
case KindConnection:
return 3
case KindWrite:
return 4
case KindValidation:
return 5
case KindInterrupted:
return 130
default:
return 1
}
}
+56
View File
@@ -0,0 +1,56 @@
// Package atomicfile 提供 Windows 上更稳健的原子文件替换。
//
// 直接 os.Rename 在 Windows 上走 MoveFileEx(MOVEFILE_REPLACE_EXISTING),
// 只要目标文件此刻被任何进程以「不共享删除」方式打开(杀毒软件实时扫描、
// Windows Search 索引器、编辑器预览等都会这样),就会返回
// ERROR_ACCESS_DENIED,而且这种占用通常是毫秒级的瞬时状态。
// 因此这里对 rename 做有限次指数退避重试;非 Windows 平台无此问题,
// 但同样行为无害(第一次就成功)。
package atomicfile
import (
"errors"
"fmt"
"os"
"runtime"
"time"
)
// renameRetries 是 rename 的最大尝试次数(含首次)。
const renameRetries = 12
// Replace 原子地用 tmp 覆盖 final。tmp 与 final 必须在同一文件系统内。
//
// 失败时保留 tmp 供调用者排查,并返回最后一次 rename 的原始错误。
func Replace(tmp, final string) error {
var err error
delay := time.Millisecond
for i := 0; i < renameRetries; i++ {
if err = os.Rename(tmp, final); err == nil {
return nil
}
// 非 Windows 上的失败基本都是真错误(跨设备、权限、目录不存在),
// 重试无意义,立即返回。
if runtime.GOOS != "windows" || !isTransient(err) {
return err
}
time.Sleep(delay)
if delay < 20*time.Millisecond {
delay *= 2
}
}
return fmt.Errorf("%w (after %d attempts; the file is likely held open by "+
"antivirus or the search indexer)", err, renameRetries)
}
// isTransient 判断 rename 失败是否属于「目标被短暂占用」。
//
// 只识别 Windows 上真实的瞬时错误码。os.Rename 已经把 Win32 错误包成
// *os.LinkError,其 Err 是 syscall.Errno,可直接与 syscall 常量比较。
func isTransient(err error) bool {
var le *os.LinkError
if !errors.As(err, &le) {
return false
}
return isTransientErrno(le.Err)
}
+7
View File
@@ -0,0 +1,7 @@
//go:build !windows
package atomicfile
// 非 Windows 平台没有「目标被短暂占用导致 rename 失败」这一语义,
// 因此不做重试判断。
func isTransientErrno(err error) bool { return false }
+29
View File
@@ -0,0 +1,29 @@
//go:build windows
package atomicfile
import (
"errors"
"syscall"
)
// Windows 上值得重试的错误码。
const (
errAccessDenied syscall.Errno = 5 // ERROR_ACCESS_DENIED
errSharingViol syscall.Errno = 32 // ERROR_SHARING_VIOLATION
errLockViolation syscall.Errno = 33 // ERROR_LOCK_VIOLATION
errBusy syscall.Errno = 170 // ERROR_BUSY
errUserMapped syscall.Errno = 1224 // ERROR_USER_MAPPED_FILE
)
func isTransientErrno(err error) bool {
var errno syscall.Errno
if !errors.As(err, &errno) {
return false
}
switch errno {
case errAccessDenied, errSharingViol, errLockViolation, errBusy, errUserMapped:
return true
}
return false
}
+98
View File
@@ -0,0 +1,98 @@
// Package checkpoint persists and restores scan progress in state.json.
package checkpoint
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"time"
"archer/internal/atomicfile"
"archer/internal/row"
)
const Version = 1
// Cursor identifies where the scanner should resume. Phase is one of
// "pk", "null", "value" or "day"; the meaning of Key depends on it.
type Cursor struct {
Phase string `json:"phase,omitempty"`
Key row.Key `json:"key,omitempty"`
DayStart string `json:"day_start,omitempty"`
}
// State is the entire persisted checkpoint.
type State struct {
Version int `json:"version"`
RunID string `json:"run_id"`
ConfigHash string `json:"config_hash"`
Config map[string]string `json:"config,omitempty"`
ShardBy string `json:"shard_by"`
Cursor Cursor `json:"cursor"`
LastBucket string `json:"last_bucket,omitempty"`
LastShardSeq int64 `json:"last_shard_seq"`
RowsDone int64 `json:"rows_done"`
Complete bool `json:"complete"`
UpdatedAt string `json:"updated_at"`
}
func Path(dir string) string { return filepath.Join(dir, "state.json") }
// Load reads state.json. A missing file returns (nil, nil).
func Load(dir string) (*State, error) {
b, err := os.ReadFile(Path(dir))
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("read state: %w", err)
}
var st State
if err := json.Unmarshal(b, &st); err != nil {
return nil, fmt.Errorf("parse state.json: %w", err)
}
if st.Version != Version {
return nil, fmt.Errorf("unsupported state.json version %d", st.Version)
}
return &st, nil
}
// Save atomically replaces state.json using a temporary file, fsync and
// rename.
func Save(dir string, st *State) error {
st.Version = Version
st.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
b, err := json.MarshalIndent(st, "", " ")
if err != nil {
return fmt.Errorf("encode state: %w", err)
}
b = append(b, '\n')
final := Path(dir)
tmp := final + ".tmp"
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
if err != nil {
return fmt.Errorf("create state temp: %w", err)
}
if _, err := f.Write(b); err != nil {
f.Close()
os.Remove(tmp)
return fmt.Errorf("write state temp: %w", err)
}
if err := f.Sync(); err != nil {
f.Close()
os.Remove(tmp)
return fmt.Errorf("sync state temp: %w", err)
}
if err := f.Close(); err != nil {
os.Remove(tmp)
return fmt.Errorf("close state temp: %w", err)
}
if err := atomicfile.Replace(tmp, final); err != nil {
os.Remove(tmp)
return fmt.Errorf("replace state.json: %w", err)
}
return nil
}
+43
View File
@@ -0,0 +1,43 @@
package checkpoint
import (
"os"
"testing"
"archer/internal/row"
)
func TestSaveLoadRoundTrip(t *testing.T) {
dir := t.TempDir()
st := &State{
RunID: "run-1",
ConfigHash: "abc",
ShardBy: "date",
Cursor: Cursor{Phase: "value", Key: row.KeyOf("2025-01-01 00:00:00", "42")},
LastBucket: "2025-01-01",
RowsDone: 99,
}
if err := Save(dir, st); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(Path(dir) + ".tmp"); !os.IsNotExist(err) {
t.Fatal("temp state file should not remain")
}
got, err := Load(dir)
if err != nil {
t.Fatal(err)
}
if got == nil || got.RunID != "run-1" || got.RowsDone != 99 || got.LastBucket != "2025-01-01" {
t.Fatalf("got %+v", got)
}
if got.Cursor.Phase != "value" || len(got.Cursor.Key) != 2 || got.Cursor.Key[1].Val != "42" {
t.Fatalf("cursor %+v", got.Cursor)
}
}
func TestLoadMissing(t *testing.T) {
st, err := Load(t.TempDir())
if err != nil || st != nil {
t.Fatalf("got %v %v", st, err)
}
}
+352
View File
@@ -0,0 +1,352 @@
package cli
import (
"bytes"
"strings"
"testing"
"time"
"github.com/go-sql-driver/mysql"
)
func baseArgs() []string {
return []string{
"--source", "u:p@tcp(127.0.0.1:3306)/db",
"--table", "t",
"--where", "id > 0",
"--file", "out",
}
}
func TestDefaults(t *testing.T) {
c, err := Parse(baseArgs(), &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
if c.ChunkSize != 2000 || c.ShardBy != "none" || c.Format != "dump" || c.Compress != "none" {
t.Fatalf("bad defaults: %+v", c)
}
if c.MaxLag.String() != "1s" || c.CheckInterval.String() != "1s" || c.Retries != 1 {
t.Fatalf("bad defaults: %+v", c)
}
if !c.SafeAutoIncrement {
t.Fatal("safe auto increment must default true")
}
if err := c.Validate(); err != nil {
t.Fatal(err)
}
}
func TestParseLimitAlias(t *testing.T) {
args := append(baseArgs(), "--limit", "500")
c, err := Parse(args, &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
if c.ChunkSize != 500 {
t.Fatalf("got %d", c.ChunkSize)
}
}
func TestParseInsertModesMutuallyExclusive(t *testing.T) {
args := append(baseArgs(), "--insert-ignore", "--replace")
if _, err := Parse(args, &bytes.Buffer{}); err == nil {
t.Fatal("expected error")
}
}
func TestValidateRequired(t *testing.T) {
c, _ := Parse([]string{"--source", "u:p@tcp(h:3306)/db", "--table", "t", "--file", "out"}, &bytes.Buffer{})
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--where") {
t.Fatalf("got %v", err)
}
}
func TestValidateNeedsTarget(t *testing.T) {
c, _ := Parse([]string{"--source", "u:p@tcp(h:3306)/db", "--table", "t", "--where", "1=1"}, &bytes.Buffer{})
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "output target") {
t.Fatalf("got %v", err)
}
}
func TestValidateShardRules(t *testing.T) {
args := append(baseArgs(), "--shard-by", "count", "--shard-column", "created_at", "--shard-rows", "10")
c, err := Parse(args, &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--shard-column") {
t.Fatalf("got %v", err)
}
args = append(baseArgs(), "--shard-by", "count")
c, _ = Parse(args, &bytes.Buffer{})
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--shard-rows") {
t.Fatalf("got %v", err)
}
args = append(baseArgs(), "--shard-by", "date", "--shard-column", "c", "--bulk-delete", "--purge")
c, _ = Parse(args, &bytes.Buffer{})
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "bulk-delete") {
t.Fatalf("got %v", err)
}
}
func TestValidateBulkDeleteNeedsPurge(t *testing.T) {
args := append(baseArgs(), "--bulk-delete")
c, _ := Parse(args, &bytes.Buffer{})
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "requires --purge") {
t.Fatalf("got %v", err)
}
}
func TestFingerprintExcludesPassword(t *testing.T) {
a, _ := Parse([]string{"--source", "u:one@tcp(h:3306)/db", "--table", "t", "--where", "1=1", "--file", "out"}, &bytes.Buffer{})
b, _ := Parse([]string{"--source", "u:two@tcp(h:3306)/db", "--table", "t", "--where", "1=1", "--file", "out"}, &bytes.Buffer{})
ha, fa, err := a.Fingerprint()
if err != nil {
t.Fatal(err)
}
hb, fb, err := b.Fingerprint()
if err != nil {
t.Fatal(err)
}
if ha != hb {
t.Fatal("password must not affect the fingerprint")
}
if fa["source"] != fb["source"] {
t.Fatalf("password must not affect the source field: %q vs %q", fa["source"], fb["source"])
}
}
func fingerprintFor(t *testing.T, dsn string) (string, map[string]string) {
t.Helper()
c, err := Parse([]string{"--source", dsn, "--table", "t", "--where", "id > 0", "--file", "out"}, &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
h, f, err := c.Fingerprint()
if err != nil {
t.Fatal(err)
}
return h, f
}
func TestFingerprintTracksConnectionCharset(t *testing.T) {
hBase, _ := fingerprintFor(t, "u:p@tcp(h:3306)/db")
hCharset, fCharset := fingerprintFor(t, "u:p@tcp(h:3306)/db?charset=utf8mb4")
if hBase == hCharset {
t.Fatal("charset must affect the fingerprint")
}
if !strings.Contains(fCharset["source"], "charset=utf8mb4") {
t.Fatalf("source field missing charset: %q", fCharset["source"])
}
hCollation, _ := fingerprintFor(t, "u:p@tcp(h:3306)/db?charset=utf8mb4&collation=utf8mb4_bin")
if hCharset == hCollation {
t.Fatal("collation must affect the fingerprint")
}
}
func TestDsnFingerprintRedactsPassword(t *testing.T) {
c, err := mysql.ParseDSN("u:s3cr3t@tcp(h:3306)/db")
if err != nil {
t.Fatal(err)
}
got := dsnFingerprint(c)
if got != "u@tcp(h:3306)/db" {
t.Fatalf("got %q", got)
}
if strings.Contains(got, "s3cr3t") {
t.Fatal("password leaked into the fingerprint")
}
if c.Passwd != "s3cr3t" {
t.Fatal("dsnFingerprint must not modify the input config")
}
if dsnFingerprint(nil) != "" {
t.Fatal("nil config must yield an empty fragment")
}
}
func TestFingerprintDestEmptyAndNoPassword(t *testing.T) {
h, f := fingerprintFor(t, "u:s3cr3t@tcp(h:3306)/db")
if f["dest"] != "" {
t.Fatalf("dest must be empty without --dest: %q", f["dest"])
}
if f["source"] != "u@tcp(h:3306)/db" {
t.Fatalf("source = %q", f["source"])
}
if strings.Contains(h, "s3cr3t") || strings.Contains(f["source"], "s3cr3t") {
t.Fatal("password leaked into the fingerprint")
}
}
func TestFingerprintDetectsTimezoneChange(t *testing.T) {
a, _ := Parse(append(baseArgs(), "--shard-by", "date", "--shard-column", "c", "--shard-time-zone", "UTC"), &bytes.Buffer{})
b, _ := Parse(append(baseArgs(), "--shard-by", "date", "--shard-column", "c", "--shard-time-zone", "Asia/Shanghai"), &bytes.Buffer{})
ha, fa, _ := a.Fingerprint()
hb, _, _ := b.Fingerprint()
if ha == hb {
t.Fatal("time zone must affect the fingerprint")
}
_ = fa
}
func TestDiffFields(t *testing.T) {
old := map[string]string{"a": "1", "b": "2"}
cur := map[string]string{"a": "1", "b": "3", "c": "4"}
d := DiffFields(old, cur)
if len(d) != 2 {
t.Fatalf("got %v", d)
}
}
func TestValidateCompressLevel(t *testing.T) {
for _, level := range []string{"0", "10"} {
c, err := Parse(append(baseArgs(), "--compress", "gzip", "--compress-level", level), &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "between 1 and 9") {
t.Fatalf("level %s: got %v", level, err)
}
}
c, _ := Parse(append(baseArgs(), "--compress", "none", "--compress-level", "5"), &bytes.Buffer{})
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "only valid when compression is enabled") {
t.Fatalf("none + level 5: got %v", err)
}
c, _ = Parse(append(baseArgs(), "--compress", "none", "--compress-level", "3"), &bytes.Buffer{})
if err := c.Validate(); err != nil {
t.Fatalf("none + default level must be accepted: %v", err)
}
for _, name := range []string{"gzip", "zstd", "xz"} {
c, _ = Parse(append(baseArgs(), "--compress", name, "--compress-level", "9"), &bytes.Buffer{})
if err := c.Validate(); err != nil {
t.Fatalf("%s level 9: %v", name, err)
}
}
}
func TestProgressModes(t *testing.T) {
cases := []struct {
arg string
mode ProgressMode
rows int64
valid bool
}{
{"", ProgressAuto, 0, true},
{"auto", ProgressAuto, 0, true},
{"on", ProgressOn, 0, true},
{"off", ProgressOff, 0, true},
{"0", ProgressOff, 0, true},
{"100000", ProgressRows, 100000, true},
{"1", ProgressRows, 1, true},
{"abc", 0, 0, false},
{"-1", 0, 0, false},
}
for _, tc := range cases {
args := baseArgs()
if tc.arg != "" {
args = append(args, "--progress", tc.arg)
}
c, err := Parse(args, &bytes.Buffer{})
if err != nil {
t.Fatalf("%q: parse: %v", tc.arg, err)
}
err = c.Validate()
if !tc.valid {
if err == nil {
t.Fatalf("%q: expected a configuration error", tc.arg)
}
continue
}
if err != nil {
t.Fatalf("%q: validate: %v", tc.arg, err)
}
if c.Progress.Mode != tc.mode || c.Progress.Rows != tc.rows {
t.Fatalf("%q: got mode=%d rows=%d", tc.arg, c.Progress.Mode, c.Progress.Rows)
}
}
c, _ := Parse(baseArgs(), &bytes.Buffer{})
if c.Progress.Mode != ProgressAuto {
t.Fatalf("default progress mode = %d, want auto", c.Progress.Mode)
}
if c.ProgressInterval != time.Second {
t.Fatalf("default progress interval = %s", c.ProgressInterval)
}
}
func TestProgressEnabledDependsOnTTY(t *testing.T) {
on, _ := Parse(append(baseArgs(), "--progress", "on"), &bytes.Buffer{})
off, _ := Parse(append(baseArgs(), "--progress", "off"), &bytes.Buffer{})
auto, _ := Parse(baseArgs(), &bytes.Buffer{})
for _, c := range []*Config{on, off, auto} {
if err := c.Validate(); err != nil {
t.Fatal(err)
}
}
if !on.Progress.Enabled(false) || !on.Progress.Enabled(true) {
t.Fatal("on must always be enabled")
}
if off.Progress.Enabled(true) || off.Progress.Enabled(false) {
t.Fatal("off must never be enabled")
}
if !auto.Progress.Enabled(true) {
t.Fatal("auto must be enabled on a terminal")
}
if auto.Progress.Enabled(false) {
t.Fatal("auto must be silent without a terminal")
}
}
func TestProgressIntervalValidation(t *testing.T) {
c, err := Parse(append(baseArgs(), "--progress-interval", "0s"), &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--progress-interval") {
t.Fatalf("got %v", err)
}
}
func TestFingerprintCompressLevel(t *testing.T) {
implicit, _ := Parse(append(baseArgs(), "--compress", "gzip"), &bytes.Buffer{})
explicit, _ := Parse(append(baseArgs(), "--compress", "gzip", "--compress-level", "3"), &bytes.Buffer{})
hImplicit, fImplicit, err := implicit.Fingerprint()
if err != nil {
t.Fatal(err)
}
hExplicit, _, err := explicit.Fingerprint()
if err != nil {
t.Fatal(err)
}
if hImplicit != hExplicit {
t.Fatal("the default level must not change the fingerprint")
}
if _, ok := fImplicit["compress_level"]; ok {
t.Fatal("the default level must not be fingerprinted")
}
five, _ := Parse(append(baseArgs(), "--compress", "gzip", "--compress-level", "5"), &bytes.Buffer{})
hFive, fFive, err := five.Fingerprint()
if err != nil {
t.Fatal(err)
}
if hFive == hImplicit {
t.Fatal("a non-default level must change the fingerprint")
}
if fFive["compress_level"] != "5" {
t.Fatalf("compress_level = %q", fFive["compress_level"])
}
if diffs := DiffFields(fImplicit, fFive); len(diffs) != 1 || !strings.Contains(diffs[0], "compress_level") {
t.Fatalf("raising the level: %v", diffs)
}
if diffs := DiffFields(fFive, fImplicit); len(diffs) != 1 || !strings.Contains(diffs[0], "compress_level") {
t.Fatalf("lowering the level back to default: %v", diffs)
}
}
+112
View File
@@ -0,0 +1,112 @@
// Package cli parses and validates archer command-line configuration.
package cli
import (
"fmt"
"time"
"github.com/go-sql-driver/mysql"
)
// Config is the fully resolved runtime configuration.
type Config struct {
SourceDSN string
DestDSN string
Table string
DestTable string
Where string
Columns []string
FileDir string
Purge bool
BulkDelete bool
InsertMode string // "", "ignore", "replace"
ChunkSize int
ShardBy string // none|count|date
ShardRows int
ShardColumn string
ShardColumnType string
ShardTimeZone string
ZeroDate string
Format string
Compress string
CompressLevel int
CSVNull string
MaxLag time.Duration
CheckInterval time.Duration
CheckReplicaLag string
Retries int
SafeAutoIncrement bool
Progress ProgressValue
ProgressInterval time.Duration
Statistics bool
DryRun bool
AllowFullScan bool
ForceResume bool
Index string
Version bool
}
// Defaults returns a Config with documented default values applied.
func Defaults() *Config {
return &Config{
ChunkSize: 2000,
ShardBy: "none",
ShardRows: 0,
ShardTimeZone: "UTC",
Format: "dump",
Compress: "none",
CompressLevel: 3,
CSVNull: `\N`,
MaxLag: time.Second,
CheckInterval: time.Second,
Retries: 1,
SafeAutoIncrement: true,
Progress: ProgressValue{Mode: ProgressAuto},
ProgressInterval: time.Second,
}
}
// SourceConfig parses and normalizes the source DSN, forcing parseTime off so
// values arrive as raw bytes.
func (c *Config) SourceConfig() (*mysql.Config, error) {
return normalize(c.SourceDSN)
}
// DestConfig parses and normalizes the dest DSN.
func (c *Config) DestConfig() (*mysql.Config, error) {
if c.DestDSN == "" {
return nil, nil
}
return normalize(c.DestDSN)
}
func normalize(dsn string) (*mysql.Config, error) {
cfg, err := mysql.ParseDSN(dsn)
if err != nil {
return nil, fmt.Errorf("invalid DSN: %w", err)
}
cfg.ParseTime = false
if cfg.Loc == nil {
cfg.Loc = time.UTC
}
return cfg, nil
}
// SameInstance reports whether two normalized DSNs point at the same MySQL
// server, which determines whether dest writes and purges share a transaction.
func SameInstance(src, dst *mysql.Config) bool {
if src == nil || dst == nil {
return false
}
return src.Net == dst.Net && src.Addr == dst.Addr
}
// ShardMode helpers.
func (c *Config) IsDateShard() bool { return c.ShardBy == "date" }
func (c *Config) IsCountShard() bool { return c.ShardBy == "count" }
func (c *Config) HasFile() bool { return c.FileDir != "" }
+102
View File
@@ -0,0 +1,102 @@
package cli
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strconv"
"strings"
"github.com/go-sql-driver/mysql"
)
// Fingerprint returns a stable hash of every configuration input that can
// change scan or shard semantics, plus the normalized fields used to build it.
// The source and dest fields are standard MySQL DSNs that include every
// connection parameter the driver parsed (charset, collation, timeout, ...),
// so a resume notices any of them changing. The password is the one exclusion:
// rotating it must not break resume.
func (c *Config) Fingerprint() (string, map[string]string, error) {
src, err := c.SourceConfig()
if err != nil {
return "", nil, fmt.Errorf("source DSN: %w", err)
}
dst, err := c.DestConfig()
if err != nil {
return "", nil, fmt.Errorf("dest DSN: %w", err)
}
fields := map[string]string{
"source": dsnFingerprint(src),
"dest": dsnFingerprint(dst),
"table": c.Table,
"dest_table": c.DestTable,
"where": c.Where,
"columns": strings.Join(c.Columns, ","),
"purge": strconv.FormatBool(c.Purge),
"bulk_delete": strconv.FormatBool(c.BulkDelete),
"insert_mode": c.InsertMode,
"shard_by": c.ShardBy,
"shard_rows": strconv.Itoa(c.ShardRows),
"shard_column": c.ShardColumn,
"shard_column_type": c.ShardColumnType,
"shard_time_zone": c.ShardTimeZone,
"zero_date": c.ZeroDate,
"format": c.Format,
"compress": c.Compress,
"csv_null": c.CSVNull,
"safe_auto_increment": strconv.FormatBool(c.SafeAutoIncrement),
}
// The level only changes the output when compression is on, and omitting
// the default keeps fingerprints of tasks started before it existed
// byte-identical so they can resume.
if c.Compress != "" && c.Compress != "none" && c.CompressLevel != 3 {
fields["compress_level"] = strconv.Itoa(c.CompressLevel)
}
keys := make([]string, 0, len(fields))
for k := range fields {
keys = append(keys, k)
}
sort.Strings(keys)
var b strings.Builder
for _, k := range keys {
b.WriteString(k)
b.WriteByte('=')
b.WriteString(fields[k])
b.WriteByte('\n')
}
sum := sha256.Sum256([]byte(b.String()))
return hex.EncodeToString(sum[:]), fields, nil
}
// dsnFingerprint renders c as a standard MySQL DSN covering every parameter
// the driver parsed, except the password. Copying by value keeps the driver's
// unexported fields (charsets, timeTruncate, ...) intact, which is the only way
// FormatDSN can serialize them.
func dsnFingerprint(c *mysql.Config) string {
if c == nil {
return ""
}
redacted := *c
redacted.Passwd = ""
return redacted.FormatDSN()
}
// DiffFields lists keys whose values differ between two fingerprints.
func DiffFields(old, cur map[string]string) []string {
seen := map[string]bool{}
var out []string
for k, v := range cur {
seen[k] = true
if old[k] != v {
out = append(out, fmt.Sprintf("%s: %q -> %q", k, old[k], v))
}
}
for k, v := range old {
if !seen[k] {
out = append(out, fmt.Sprintf("%s: %q -> (absent)", k, v))
}
}
sort.Strings(out)
return out
}
+102
View File
@@ -0,0 +1,102 @@
package cli
import (
"flag"
"fmt"
"io"
"strings"
)
// Parse parses args (without the program name) into a Config. It does not
// validate combinations; call Validate for that.
func Parse(args []string, errOut io.Writer) (*Config, error) {
c := Defaults()
var (
columnsArg string
limitArg = -1
insertIgnoreArg bool
replaceArg bool
)
fs := flag.NewFlagSet("archer", flag.ContinueOnError)
fs.SetOutput(errOut)
fs.StringVar(&c.SourceDSN, "source", "", "source MySQL DSN (required)")
fs.StringVar(&c.DestDSN, "dest", "", "destination MySQL DSN")
fs.StringVar(&c.Table, "table", "", "source table (required)")
fs.StringVar(&c.DestTable, "dest-table", "", "destination table (defaults to --table)")
fs.StringVar(&c.Where, "where", "", "row filter for the source table (required)")
fs.StringVar(&columnsArg, "columns", "", "comma separated columns to export (default: all)")
fs.StringVar(&c.FileDir, "file", "", "output directory for shard files")
fs.BoolVar(&c.Purge, "purge", false, "delete archived rows from the source")
fs.BoolVar(&c.BulkDelete, "bulk-delete", false, "delete by key range instead of IN list (requires --purge)")
fs.BoolVar(&insertIgnoreArg, "insert-ignore", false, "use INSERT IGNORE for --dest")
fs.BoolVar(&replaceArg, "replace", false, "use REPLACE for --dest")
fs.IntVar(&c.ChunkSize, "chunk-size", c.ChunkSize, "rows per SELECT / transaction / checkpoint")
fs.IntVar(&limitArg, "limit", -1, "deprecated alias for --chunk-size")
fs.StringVar(&c.ShardBy, "shard-by", c.ShardBy, "shard mode: none|count|date")
fs.IntVar(&c.ShardRows, "shard-rows", c.ShardRows, "rows per shard file (count/date modes)")
fs.StringVar(&c.ShardColumn, "shard-column", "", "date or epoch column used for date sharding")
fs.StringVar(&c.ShardColumnType, "shard-column-type", "", "epoch unit: epoch-seconds|epoch-milliseconds|epoch-microseconds")
fs.StringVar(&c.ShardTimeZone, "shard-time-zone", c.ShardTimeZone, "time zone for epoch shard values")
fs.StringVar(&c.ZeroDate, "zero-date", "", "zero date handling: bucket (default: error)")
fs.StringVar(&c.Format, "format", c.Format, "output format: dump|csv|jsonl")
fs.StringVar(&c.Compress, "compress", c.Compress, "compression: none|gzip|zstd|xz")
fs.IntVar(&c.CompressLevel, "compress-level", c.CompressLevel, "compression effort 1-9 (higher is smaller and slower)")
fs.StringVar(&c.CSVNull, "csv-null", c.CSVNull, "string used for NULL in csv output")
fs.DurationVar(&c.MaxLag, "max-lag", c.MaxLag, "maximum tolerated replica lag")
fs.DurationVar(&c.CheckInterval, "check-interval", c.CheckInterval, "wait between replica lag checks")
fs.StringVar(&c.CheckReplicaLag, "check-replica-lag", "", "DSN of a replica whose lag is checked before each fetch")
fs.IntVar(&c.Retries, "retries", c.Retries, "retries for cross-instance commits")
fs.BoolVar(&c.SafeAutoIncrement, "safe-auto-increment", c.SafeAutoIncrement, "never purge the maximum auto-increment row")
fs.Var(&c.Progress, "progress", "progress output: auto|on|off|N (default auto; N: also every N rows)")
fs.DurationVar(&c.ProgressInterval, "progress-interval", c.ProgressInterval, "minimum interval between progress updates")
fs.BoolVar(&c.Statistics, "statistics", false, "print a summary when finished")
fs.BoolVar(&c.DryRun, "dry-run", false, "print resolved config and SQL without writing")
fs.BoolVar(&c.AllowFullScan, "allow-full-scan", false, "allow date sharding without a usable index")
fs.BoolVar(&c.ForceResume, "force-resume", false, "resume even when the config fingerprint changed")
fs.StringVar(&c.Index, "index", "", "force a unique index for the scan key")
fs.BoolVar(&c.Version, "version", false, "print version and exit")
if err := fs.Parse(args); err != nil {
return nil, err
}
if c.Version {
return c, nil
}
set := map[string]bool{}
fs.Visit(func(f *flag.Flag) { set[f.Name] = true })
if set["limit"] {
if set["chunk-size"] {
return nil, fmt.Errorf("--limit is a deprecated alias of --chunk-size; pass only one")
}
c.ChunkSize = limitArg
}
if columnsArg != "" {
for _, part := range strings.Split(columnsArg, ",") {
part = strings.TrimSpace(part)
if part != "" {
c.Columns = append(c.Columns, part)
}
}
}
if insertIgnoreArg && replaceArg {
return nil, fmt.Errorf("--insert-ignore and --replace are mutually exclusive")
}
switch {
case insertIgnoreArg:
c.InsertMode = "ignore"
case replaceArg:
c.InsertMode = "replace"
}
if c.DestTable == "" {
c.DestTable = c.Table
}
return c, nil
}
+84
View File
@@ -0,0 +1,84 @@
package cli
import (
"fmt"
"strconv"
"strings"
)
// ProgressMode is the resolved meaning of --progress.
type ProgressMode int
const (
ProgressAuto ProgressMode = iota // live on a terminal, silent otherwise
ProgressOff
ProgressOn
ProgressRows // enabled, plus a row-count threshold
)
// ProgressValue is the flag.Value behind --progress. It keeps the raw
// argument so invalid values are reported as configuration errors by Validate
// rather than as bare flag parse errors.
type ProgressValue struct {
Mode ProgressMode
Rows int64
raw string
}
// Set records the raw argument; resolve happens in Validate.
func (p *ProgressValue) Set(s string) error {
p.raw = s
return nil
}
func (p *ProgressValue) String() string {
if p.raw != "" {
return p.raw
}
switch p.Mode {
case ProgressOn:
return "on"
case ProgressOff:
return "off"
case ProgressRows:
return strconv.FormatInt(p.Rows, 10)
default:
return "auto"
}
}
// resolve parses the recorded argument into Mode and Rows.
func (p *ProgressValue) resolve() error {
switch strings.ToLower(strings.TrimSpace(p.raw)) {
case "", "auto":
p.Mode, p.Rows = ProgressAuto, 0
case "on":
p.Mode, p.Rows = ProgressOn, 0
case "off":
p.Mode, p.Rows = ProgressOff, 0
default:
n, err := strconv.ParseInt(strings.TrimSpace(p.raw), 10, 64)
if err != nil || n < 0 {
return fmt.Errorf("--progress must be auto, on, off or a non-negative integer, got %q", p.raw)
}
if n == 0 {
p.Mode, p.Rows = ProgressOff, 0
} else {
p.Mode, p.Rows = ProgressRows, n
}
}
return nil
}
// Enabled reports whether progress output should be produced. tty describes
// whether stderr is a terminal.
func (p ProgressValue) Enabled(tty bool) bool {
switch p.Mode {
case ProgressOn, ProgressRows:
return true
case ProgressAuto:
return tty
default:
return false
}
}
+119
View File
@@ -0,0 +1,119 @@
package cli
import (
"strings"
"time"
"archer/internal/archererr"
)
// Validate checks required flags and the inter-parameter dependency rules.
// The shard column's existence and type are checked later, after metadata is
// loaded, so the error can list candidate columns.
func (c *Config) Validate() error {
if c.SourceDSN == "" {
return cfgErr("--source is required")
}
if c.Table == "" {
return cfgErr("--table is required")
}
if strings.TrimSpace(c.Where) == "" {
return cfgErr("--where is required")
}
if !c.HasFile() && c.DestDSN == "" && !c.Purge {
return cfgErr("at least one output target is required: --file, --dest or --purge")
}
if c.ChunkSize <= 0 {
return cfgErr("--chunk-size must be greater than 0")
}
if c.Retries < 0 {
return cfgErr("--retries must be 0 or greater")
}
if err := c.Progress.resolve(); err != nil {
return cfgErr("%v", err)
}
if c.ProgressInterval <= 0 {
return cfgErr("--progress-interval must be greater than 0")
}
switch c.ShardBy {
case "none", "count", "date":
default:
return cfgErr("--shard-by must be one of none, count, date")
}
switch c.ShardBy {
case "date":
if c.ShardRows < 0 {
return cfgErr("--shard-rows must be 0 or greater")
}
default:
if c.ShardColumn != "" {
return cfgErr("--shard-column is only valid with --shard-by date")
}
if c.ShardColumnType != "" {
return cfgErr("--shard-column-type is only valid with --shard-by date")
}
if c.ZeroDate != "" {
return cfgErr("--zero-date is only valid with --shard-by date")
}
}
if c.ShardBy == "none" && c.ShardRows != 0 {
return cfgErr("--shard-rows is only valid with --shard-by count or date")
}
if c.ShardBy == "count" && c.ShardRows == 0 {
return cfgErr("--shard-rows must be greater than 0 for --shard-by count")
}
if c.BulkDelete && !c.Purge {
return cfgErr("--bulk-delete requires --purge")
}
if c.BulkDelete && c.ShardBy == "date" {
return cfgErr("--bulk-delete cannot be used with --shard-by date; delete by IN list instead")
}
switch c.Format {
case "dump", "csv", "jsonl":
default:
return cfgErr("--format must be one of dump, csv, jsonl")
}
switch c.Compress {
case "none", "gzip", "zstd", "xz":
default:
return cfgErr("--compress must be one of none, gzip, zstd, xz")
}
if c.CompressLevel < 1 || c.CompressLevel > 9 {
return cfgErr("--compress-level must be between 1 and 9")
}
if c.Compress == "none" && c.CompressLevel != 3 {
return cfgErr("--compress-level is only valid when compression is enabled")
}
switch c.ZeroDate {
case "", "bucket":
default:
return cfgErr("--zero-date must be empty or bucket")
}
if _, err := time.LoadLocation(c.ShardTimeZone); err != nil {
return cfgErr("--shard-time-zone %q is not a valid location: %v", c.ShardTimeZone, err)
}
if _, err := c.SourceConfig(); err != nil {
return cfgErr("--source: %v", err)
}
if _, err := c.DestConfig(); err != nil {
return cfgErr("--dest: %v", err)
}
if c.CheckReplicaLag != "" {
if err := validateDSN(c.CheckReplicaLag); err != nil {
return cfgErr("--check-replica-lag: %v", err)
}
}
return nil
}
func validateDSN(dsn string) error {
_, err := normalize(dsn)
return err
}
func cfgErr(format string, args ...any) error {
return archererr.Newf(archererr.KindConfig, format, args...)
}
+122
View File
@@ -0,0 +1,122 @@
// Package estimate derives a cheap total-size estimate for progress display.
//
// Only bounded MIN/MAX queries are used; COUNT(*) is never attempted. A query
// that fails, times out, or whose metric does not apply silently yields no
// estimate, leaving the caller to render rows and rate only.
package estimate
import (
"context"
"database/sql"
"strconv"
"strings"
"time"
"archer/internal/progress"
"archer/internal/row"
"archer/internal/schema"
)
// Spec describes which metric a run can expose.
type Spec struct {
Kind string // "day", "key" or "" for none
Column string
ShardKind schema.ShardKind
Unit int64
Loc *time.Location
}
// RangeQuery returns MIN and MAX of a column as raw text. Implementations must
// honor ctx so the bounded timeout is effective.
type RangeQuery func(ctx context.Context, column string) (minRaw, maxRaw sql.NullString, err error)
// Bounded runs q under a timeout and converts the result into a display
// estimate. It returns nil instead of an error on every failure path.
func Bounded(ctx context.Context, q RangeQuery, spec Spec, timeout time.Duration) *progress.Estimate {
if q == nil || spec.Kind == "" || timeout <= 0 {
return nil
}
qctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
minRaw, maxRaw, err := q(qctx, spec.Column)
if err != nil || !minRaw.Valid || !maxRaw.Valid {
return nil
}
var lo, hi float64
var ok bool
switch spec.Kind {
case "day":
lo, hi, ok = dayRange(minRaw.String, maxRaw.String, spec.ShardKind, spec.Unit, spec.Loc)
case "key":
lo, hi, ok = keyRange(minRaw.String, maxRaw.String)
}
if !ok || hi <= lo {
return nil
}
return &progress.Estimate{Kind: spec.Kind, Lo: lo, Hi: hi}
}
// KeyColumn reports the scan key column when it is a single integer column,
// which is the only non-date shape a cheap key-range estimate applies to.
func KeyColumn(cols, types []string) (string, bool) {
if len(cols) != 1 || len(types) != 1 {
return "", false
}
if !row.IsInteger(types[0]) {
return "", false
}
return cols[0], true
}
func dayRange(minRaw, maxRaw string, kind schema.ShardKind, unit int64, loc *time.Location) (float64, float64, bool) {
if loc == nil {
loc = time.UTC
}
switch kind {
case schema.ShardDate, schema.ShardDateTime:
lo, ok1 := daySeconds(minRaw, loc)
hi, ok2 := daySeconds(maxRaw, loc)
return lo, hi, ok1 && ok2
case schema.ShardEpoch:
minV, err1 := strconv.ParseInt(strings.TrimSpace(minRaw), 10, 64)
maxV, err2 := strconv.ParseInt(strings.TrimSpace(maxRaw), 10, 64)
if err1 != nil || err2 != nil {
return 0, 0, false
}
lo := dayStart(schema.EpochTime(minV, unit, loc), loc).Unix()
hi := dayStart(schema.EpochTime(maxV, unit, loc), loc).Unix()
return float64(lo), float64(hi), true
}
return 0, 0, false
}
func daySeconds(raw string, loc *time.Location) (float64, bool) {
s := strings.TrimSpace(raw)
if len(s) < 10 {
return 0, false
}
day := s[:10]
if strings.HasPrefix(day, "0000-") {
return 0, false
}
t, err := time.ParseInLocation("2006-01-02", day, loc)
if err != nil {
return 0, false
}
return float64(t.Unix()), true
}
func keyRange(minRaw, maxRaw string) (float64, float64, bool) {
minV, err1 := strconv.ParseInt(strings.TrimSpace(minRaw), 10, 64)
maxV, err2 := strconv.ParseInt(strings.TrimSpace(maxRaw), 10, 64)
if err1 != nil || err2 != nil {
return 0, 0, false
}
return float64(minV), float64(maxV), true
}
func dayStart(t time.Time, loc *time.Location) time.Time {
return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, loc)
}
+113
View File
@@ -0,0 +1,113 @@
package estimate
import (
"context"
"database/sql"
"strconv"
"testing"
"time"
"archer/internal/schema"
)
func validRange(minRaw, maxRaw string) RangeQuery {
return func(context.Context, string) (sql.NullString, sql.NullString, error) {
return sql.NullString{String: minRaw, Valid: true}, sql.NullString{String: maxRaw, Valid: true}, nil
}
}
func TestKeyColumnApplicability(t *testing.T) {
cases := []struct {
cols, types []string
want bool
}{
{[]string{"id"}, []string{"bigint"}, true},
{[]string{"id"}, []string{"int"}, true},
{[]string{"a", "b"}, []string{"bigint", "bigint"}, false},
{[]string{"code"}, []string{"varchar"}, false},
{[]string{"payload"}, []string{"blob"}, false},
{[]string{"bin"}, []string{"varbinary"}, false},
{[]string{"ts"}, []string{"datetime"}, false},
}
for _, tc := range cases {
col, ok := KeyColumn(tc.cols, tc.types)
if ok != tc.want {
t.Fatalf("%v/%v: got %v want %v", tc.cols, tc.types, ok, tc.want)
}
if ok && col != tc.cols[0] {
t.Fatalf("%v: wrong column %q", tc.cols, col)
}
}
}
func TestBoundedDayRangeFromDates(t *testing.T) {
spec := Spec{Kind: "day", Column: "created_at", ShardKind: schema.ShardDate, Loc: time.UTC}
got := Bounded(context.Background(), validRange("2025-01-01", "2025-01-11"), spec, time.Second)
if got == nil || got.Kind != "day" {
t.Fatalf("got %+v", got)
}
if d := time.Duration(got.Hi-got.Lo) * time.Second; d != 10*24*time.Hour {
t.Fatalf("span = %s", d)
}
}
func TestBoundedDayRangeFromDatetimeAndEpoch(t *testing.T) {
dt := Spec{Kind: "day", Column: "c", ShardKind: schema.ShardDateTime, Loc: time.UTC}
if got := Bounded(context.Background(), validRange("2025-01-01 10:00:00", "2025-01-03 23:00:00"), dt, time.Second); got == nil {
t.Fatal("datetime range unavailable")
}
ep := Spec{Kind: "day", Column: "ts", ShardKind: schema.ShardEpoch, Unit: 1, Loc: time.UTC}
lo := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).Unix()
hi := time.Date(2025, 1, 4, 0, 0, 0, 0, time.UTC).Unix()
got := Bounded(context.Background(), validRange(strconv.FormatInt(lo, 10), strconv.FormatInt(hi, 10)), ep, time.Second)
if got == nil || got.Hi-got.Lo != 3*86400 {
t.Fatalf("epoch range = %+v", got)
}
}
func TestBoundedKeyRange(t *testing.T) {
spec := Spec{Kind: "key", Column: "id"}
got := Bounded(context.Background(), validRange("100", "1100"), spec, time.Second)
if got == nil || got.Kind != "key" || got.Lo != 100 || got.Hi != 1100 {
t.Fatalf("got %+v", got)
}
}
func TestBoundedSkipsUnavailableMetrics(t *testing.T) {
cases := []struct {
name string
q RangeQuery
spec Spec
}{
{"no metric", validRange("1", "10"), Spec{}},
{"query error", func(context.Context, string) (sql.NullString, sql.NullString, error) {
return sql.NullString{}, sql.NullString{}, context.DeadlineExceeded
}, Spec{Kind: "key", Column: "id"}},
{"null bounds", func(context.Context, string) (sql.NullString, sql.NullString, error) {
return sql.NullString{}, sql.NullString{}, nil
}, Spec{Kind: "key", Column: "id"}},
{"empty table", validRange("5", "5"), Spec{Kind: "key", Column: "id"}},
{"zero date", validRange("0000-00-00", "2025-01-01"), Spec{Kind: "day", ShardKind: schema.ShardDate, Loc: time.UTC}},
{"string key", validRange("a", "z"), Spec{Kind: "key", Column: "id"}},
}
for _, tc := range cases {
if got := Bounded(context.Background(), tc.q, tc.spec, time.Second); got != nil {
t.Fatalf("%s: expected nil, got %+v", tc.name, got)
}
}
}
func TestBoundedTimeoutDegrades(t *testing.T) {
blocked := func(ctx context.Context, _ string) (sql.NullString, sql.NullString, error) {
<-ctx.Done()
return sql.NullString{}, sql.NullString{}, ctx.Err()
}
start := time.Now()
got := Bounded(context.Background(), blocked, Spec{Kind: "key", Column: "id"}, 20*time.Millisecond)
if got != nil {
t.Fatalf("timed-out estimate must be nil, got %+v", got)
}
if elapsed := time.Since(start); elapsed > time.Second {
t.Fatalf("timeout took too long: %s", elapsed)
}
}
+144
View File
@@ -0,0 +1,144 @@
// Package lag checks replica delay before each fetch.
package lag
import (
"context"
"database/sql"
"errors"
"fmt"
"strconv"
"strings"
"time"
_ "github.com/go-sql-driver/mysql"
)
// Checker waits until a replica's delay is below Max and replication is
// running.
type Checker struct {
db *sql.DB
Max time.Duration
Interval time.Duration
OnWait func(reason string)
}
// Open connects to the replica DSN.
func Open(dsn string, max, interval time.Duration) (*Checker, error) {
db, err := sql.Open("mysql", dsn)
if err != nil {
return nil, err
}
db.SetMaxOpenConns(1)
if err := db.Ping(); err != nil {
db.Close()
return nil, fmt.Errorf("connect replica: %w", err)
}
return &Checker{db: db, Max: max, Interval: interval}, nil
}
func (c *Checker) Close() error { return c.db.Close() }
// Wait blocks until the replica lag is acceptable or ctx is cancelled.
func (c *Checker) Wait(ctx context.Context) error {
for {
reason, ok, err := c.status(ctx)
if err != nil {
return err
}
if ok {
return nil
}
if c.OnWait != nil {
c.OnWait(reason)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(c.Interval):
}
}
}
func (c *Checker) status(ctx context.Context) (string, bool, error) {
fields, err := c.showStatus(ctx, "SHOW REPLICA STATUS")
if err != nil || len(fields) == 0 {
fields, err = c.showStatus(ctx, "SHOW SLAVE STATUS")
if err != nil {
return "", false, err
}
}
if len(fields) == 0 {
return "replication is not configured", false, nil
}
sqlRunning := yes(fields["Replica_SQL_Running"], fields["Slave_SQL_Running"])
ioRunning := yes(fields["Replica_IO_Running"], fields["Slave_IO_Running"])
if !sqlRunning || !ioRunning {
return "replication threads are not running", false, nil
}
raw := firstNonEmpty(fields["Seconds_Behind_Source"], fields["Seconds_Behind_Master"])
if raw == "" || strings.EqualFold(raw, "NULL") {
return "replica lag is unknown", false, nil
}
secs, err := strconv.ParseFloat(raw, 64)
if err != nil {
return fmt.Sprintf("cannot parse replica lag %q", raw), false, nil
}
lag := time.Duration(secs * float64(time.Second))
if lag > c.Max {
return fmt.Sprintf("replica lag %s exceeds %s", lag, c.Max), false, nil
}
return "", true, nil
}
func (c *Checker) showStatus(ctx context.Context, q string) (map[string]string, error) {
rows, err := c.db.QueryContext(ctx, q)
if err != nil {
return nil, err
}
defer rows.Close()
cols, err := rows.Columns()
if err != nil {
return nil, err
}
if !rows.Next() {
if err := rows.Err(); err != nil && !errors.Is(err, sql.ErrNoRows) {
return nil, err
}
return nil, nil
}
vals := make([]sql.RawBytes, len(cols))
ptrs := make([]any, len(cols))
for i := range vals {
ptrs[i] = &vals[i]
}
if err := rows.Scan(ptrs...); err != nil {
return nil, err
}
out := make(map[string]string, len(cols))
for i, name := range cols {
if vals[i] == nil {
out[name] = "NULL"
} else {
out[name] = string(vals[i])
}
}
return out, nil
}
func yes(vals ...string) bool {
for _, v := range vals {
if strings.EqualFold(v, "Yes") {
return true
}
}
return false
}
func firstNonEmpty(vals ...string) string {
for _, v := range vals {
if v != "" {
return v
}
}
return ""
}
+707
View File
@@ -0,0 +1,707 @@
// Package progress renders human-facing run progress for long archives.
//
// A Reporter receives events from the single-threaded run loop and renders
// them from a background ticker, so blocking phases (a slow scan, fsync, lag
// waiting) still produce a heartbeat. Rendering never touches scanner or
// writer state directly: the run loop copies what it needs into the Reporter.
package progress
import (
"fmt"
"io"
"os"
"strconv"
"strings"
"sync"
"time"
)
// rateWindow is how far back a rate sample may sit to be considered current.
const rateWindow = 5 * time.Second
// Estimate describes the numeric range used for percentage and ETA.
// Kind is "day" (position is a day boundary in unix seconds) or "key"
// (position is a numeric scan key).
type Estimate struct {
Kind string
Lo, Hi float64
}
// Options configures a Reporter.
type Options struct {
Out io.Writer // destination, default os.Stderr
Enabled bool // whether progress lines are produced at all
Live bool // render in place with carriage returns
Interval time.Duration // minimum refresh interval, default 1s
RowEvery int64 // extra refresh every N rows; 0 disables
Width int // live line limit, default 100
Now func() time.Time
}
type sample struct {
t time.Time
rows int64
pos float64
hasPos bool
}
type data struct {
started time.Time
phase string
reason string
phaseT time.Time
base int64
read int64
pos float64
hasPos bool
originP float64
originT time.Time
hasOrig bool
estimate *Estimate
shard int64
hasShard bool
bytes int64
hasBytes bool
shards int64
hasShardN bool
samples []sample
rowMark int64
lastDraw time.Time
}
func (d *data) rows() int64 { return d.base + d.read }
func (d *data) clone() data {
c := *d
c.samples = append([]sample(nil), d.samples...)
return c
}
// Reporter accumulates run metrics and renders them.
type Reporter struct {
out io.Writer
now func() time.Time
enabled bool
live bool
interval time.Duration
rowEvery int64
width int
mu sync.Mutex
d data
started bool
running bool
stopCh chan struct{}
doneCh chan struct{}
wmu sync.Mutex
lastWidth int
}
// New builds a Reporter. A disabled Reporter still carries messages (warnings)
// to Out but emits no progress lines.
func New(o Options) *Reporter {
if o.Out == nil {
o.Out = os.Stderr
}
if o.Now == nil {
o.Now = time.Now
}
if o.Interval <= 0 {
o.Interval = time.Second
}
if o.Width <= 0 {
o.Width = 100
}
return &Reporter{
out: o.Out,
now: o.Now,
enabled: o.Enabled,
live: o.Live,
interval: o.Interval,
rowEvery: o.RowEvery,
width: o.Width,
stopCh: make(chan struct{}),
doneCh: make(chan struct{}),
}
}
// Start begins the refresh goroutine. It is a no-op for a disabled Reporter.
func (r *Reporter) Start() {
r.mu.Lock()
if r.started {
r.mu.Unlock()
return
}
r.started = true
r.initLocked(r.now())
if !r.enabled {
r.mu.Unlock()
return
}
r.running = true
r.mu.Unlock()
go r.loop()
}
func (r *Reporter) initLocked(now time.Time) {
r.d.started = now
r.d.phaseT = now
r.d.lastDraw = time.Time{}
if r.rowEvery > 0 {
r.d.rowMark = r.d.read / r.rowEvery
}
r.pushLocked(now)
}
func (r *Reporter) loop() {
t := time.NewTicker(r.interval)
defer t.Stop()
defer close(r.doneCh)
for {
select {
case <-r.stopCh:
return
case <-t.C:
r.maybeRender(false)
}
}
}
// SetBase installs the row count carried over from a previous run. Resetting
// the sample ring keeps the rate measured from this run's work rather than
// jumping by the inherited count on the first chunk.
func (r *Reporter) SetBase(n int64) {
r.mu.Lock()
r.d.base = n
r.d.samples = nil
r.pushLocked(r.now())
r.mu.Unlock()
}
// AddRows accounts for rows read in this run.
func (r *Reporter) AddRows(n int64) {
if n == 0 {
return
}
r.mu.Lock()
r.d.read += n
now := r.now()
r.pushLocked(now)
crossed := false
if r.rowEvery > 0 {
mark := r.d.read / r.rowEvery
if mark > r.d.rowMark {
r.d.rowMark = mark
crossed = true
}
}
r.mu.Unlock()
if crossed {
// The threshold is an extra line at a chunk boundary; draw it inline so
// it is not lost if the run finishes before the next tick.
r.maybeRender(true)
}
}
// SetPosition publishes the current numeric position (day seconds or key).
func (r *Reporter) SetPosition(pos float64) {
r.mu.Lock()
r.d.pos = pos
r.d.hasPos = true
if !r.d.hasOrig {
r.d.originP = pos
r.d.originT = r.now()
r.d.hasOrig = true
}
r.mu.Unlock()
}
// SetEstimate publishes or clears the range used for percentage and ETA.
func (r *Reporter) SetEstimate(e *Estimate) {
r.mu.Lock()
r.d.estimate = e
r.mu.Unlock()
}
// SetShard publishes the current shard sequence number.
func (r *Reporter) SetShard(seq int64) {
r.mu.Lock()
r.d.shard = seq
r.d.hasShard = true
r.mu.Unlock()
}
// SetBytes publishes bytes written so far.
func (r *Reporter) SetBytes(n int64) {
r.mu.Lock()
r.d.bytes = n
r.d.hasBytes = true
r.mu.Unlock()
}
// SetShards publishes the sealed shard count for the done summary.
func (r *Reporter) SetShards(n int64) {
r.mu.Lock()
r.d.shards = n
r.d.hasShardN = true
r.mu.Unlock()
}
// Phase records the current phase and an optional waiting reason. Repeating
// the same phase (for example lag-wait reported on every poll) keeps the phase
// clock running so its duration is visible.
func (r *Reporter) Phase(name, reason string) {
r.mu.Lock()
if r.d.phase != name {
r.d.phase = name
r.d.phaseT = r.now()
}
r.d.reason = reason
r.mu.Unlock()
}
// Message writes a complete diagnostic line, first clearing any live line.
func (r *Reporter) Message(format string, args ...any) {
line := strings.TrimRight(fmt.Sprintf(format, args...), "\n")
r.wmu.Lock()
defer r.wmu.Unlock()
if r.live {
r.clearLocked()
}
fmt.Fprintln(r.out, line)
}
// Stop ends refreshing and clears the live line without printing a summary.
func (r *Reporter) Stop() {
r.stopTicker()
r.wmu.Lock()
r.clearLocked()
r.wmu.Unlock()
}
// Done prints the final summary line on normal completion.
func (r *Reporter) Done() {
r.stopTicker()
now := r.now()
r.mu.Lock()
s := r.d.clone()
r.mu.Unlock()
r.wmu.Lock()
defer r.wmu.Unlock()
r.clearLocked()
if r.enabled {
fmt.Fprintln(r.out, buildDone(s, now))
}
}
// Interrupted prints the final snapshot line after a signal.
func (r *Reporter) Interrupted() {
r.stopTicker()
now := r.now()
r.mu.Lock()
s := r.d.clone()
r.mu.Unlock()
r.wmu.Lock()
defer r.wmu.Unlock()
r.clearLocked()
if r.enabled {
fmt.Fprintln(r.out, buildLine(s, now))
}
}
func (r *Reporter) stopTicker() {
r.mu.Lock()
if !r.running {
r.mu.Unlock()
return
}
r.running = false
close(r.stopCh)
done := r.doneCh
r.mu.Unlock()
<-done
}
func (r *Reporter) pushLocked(now time.Time) {
r.d.samples = append(r.d.samples, sample{
t: now,
rows: r.d.rows(),
pos: r.d.pos,
hasPos: r.d.hasPos,
})
cut := now.Add(-2 * rateWindow)
i := 0
for i < len(r.d.samples) && r.d.samples[i].t.Before(cut) {
i++
}
if i > 0 {
r.d.samples = append(r.d.samples[:0], r.d.samples[i:]...)
}
if len(r.d.samples) > 256 {
r.d.samples = r.d.samples[len(r.d.samples)-256:]
}
}
// maybeRender draws a frame. force bypasses the interval throttle so a row
// threshold can produce an extra line.
func (r *Reporter) maybeRender(force bool) {
if !r.enabled {
return
}
now := r.now()
r.mu.Lock()
if !force && !r.d.lastDraw.IsZero() && now.Sub(r.d.lastDraw) < r.interval {
r.mu.Unlock()
return
}
r.d.lastDraw = now
s := r.d.clone()
r.mu.Unlock()
var line string
if r.live {
line = r.buildLive(s, now)
} else {
line = buildLine(s, now)
}
r.wmu.Lock()
defer r.wmu.Unlock()
if r.live {
if line == "" {
line = " "
}
pad := 0
if r.lastWidth > len(line) {
pad = r.lastWidth - len(line)
}
fmt.Fprintf(r.out, "\r%s%s", line, strings.Repeat(" ", pad))
r.lastWidth = len(line)
return
}
fmt.Fprintln(r.out, line)
}
func (r *Reporter) clearLocked() {
if r.live && r.lastWidth > 0 {
fmt.Fprintf(r.out, "\r%s\r", strings.Repeat(" ", r.lastWidth))
r.lastWidth = 0
}
}
func (r *Reporter) buildLive(s data, now time.Time) string {
// Field priority when narrowing: drop bytes, then the shard number, then
// shrink the bar. The line never wraps.
type shape struct {
bar int
bytes bool
shard bool
}
shapes := []shape{
{20, true, true},
{20, false, true},
{20, false, false},
{12, false, false},
{8, false, false},
{0, false, false},
}
var last string
for _, sh := range shapes {
last = liveLine(s, now, sh.bar, sh.bytes, sh.shard)
if len(last) <= r.width {
return last
}
}
return truncate(last, r.width)
}
func liveLine(s data, now time.Time, barWidth int, showBytes, showShard bool) string {
var parts []string
pct, eta, hasEta, hasEst := calc(s, now)
if hasEst {
if barWidth > 0 {
filled := int(pct*float64(barWidth) + 0.5)
parts = append(parts, "["+strings.Repeat("#", filled)+strings.Repeat("-", barWidth-filled)+"]")
}
parts = append(parts, fmt.Sprintf("~%.0f%%", pct*100))
if s.estimate.Kind == "day" {
cur, total := dayProgress(s)
parts = append(parts, fmt.Sprintf("day %d/%d", cur, total))
}
}
parts = append(parts, humanRows(s.rows())+" rows")
if rr, ok := rate(s.samples, now, false); ok && rr > 0 {
parts = append(parts, humanRate(rr))
}
if s.phase != "" {
parts = append(parts, fmt.Sprintf("%s %s", s.phase, humanDuration(now.Sub(s.phaseT))))
}
if s.reason != "" {
parts = append(parts, "("+s.reason+")")
}
if showShard && s.hasShard {
parts = append(parts, fmt.Sprintf("shard %d", s.shard))
}
if showBytes && s.hasBytes {
parts = append(parts, humanBytes(s.bytes))
}
if hasEta {
parts = append(parts, "eta ~"+humanDuration(eta))
}
parts = append(parts, "elapsed "+humanDuration(now.Sub(s.started)))
return strings.Join(parts, " ")
}
func buildLine(s data, now time.Time) string {
var b strings.Builder
b.WriteString("progress: rows=")
b.WriteString(strconv.FormatInt(s.rows(), 10))
b.WriteString(" rate=")
if rr, ok := rate(s.samples, now, false); ok && rr > 0 {
b.WriteString(humanRate(rr))
} else {
b.WriteString(humanRate(cumulativeRows(s, now)))
}
b.WriteString(" phase=")
if s.phase == "" {
b.WriteString("start")
} else {
b.WriteString(s.phase)
}
b.WriteString(" phase_elapsed=")
b.WriteString(humanDuration(now.Sub(s.phaseT)))
b.WriteString(" elapsed=")
b.WriteString(humanDuration(now.Sub(s.started)))
if s.reason != "" {
b.WriteString(" reason=")
b.WriteString(strings.Join(strings.Fields(s.reason), "_"))
}
if s.hasShard {
fmt.Fprintf(&b, " shard=%d", s.shard)
}
if s.hasBytes {
fmt.Fprintf(&b, " bytes=%d", s.bytes)
}
pct, eta, hasEta, hasEst := calc(s, now)
if !hasEst {
b.WriteString(" estimate=none")
return b.String()
}
fmt.Fprintf(&b, " estimate=%s pct=~%.1f", s.estimate.Kind, pct*100)
if s.estimate.Kind == "day" {
cur, total := dayProgress(s)
fmt.Fprintf(&b, " day=%d/%d", cur, total)
}
if hasEta {
b.WriteString(" eta=~")
b.WriteString(humanDuration(eta))
}
return b.String()
}
func buildDone(s data, now time.Time) string {
var b strings.Builder
b.WriteString("done rows=")
b.WriteString(strconv.FormatInt(s.rows(), 10))
b.WriteString(" elapsed=")
b.WriteString(humanDuration(now.Sub(s.started)))
if s.hasShardN {
fmt.Fprintf(&b, " shards=%d", s.shards)
}
if s.hasBytes {
fmt.Fprintf(&b, " bytes=%d", s.bytes)
}
return b.String()
}
// calc returns percentage, ETA and whether an estimate is displayable.
func calc(s data, now time.Time) (pct float64, eta time.Duration, hasEta, ok bool) {
if s.estimate == nil || !s.hasPos {
return 0, 0, false, false
}
span := s.estimate.Hi - s.estimate.Lo
if span <= 0 {
return 0, 0, false, false
}
pct = (s.pos - s.estimate.Lo) / span
if pct < 0 {
pct = 0
}
if pct > 1 {
pct = 1
}
if pr, rok := positionRate(s, now); rok && pr > 0 {
remaining := s.estimate.Hi - s.pos
if remaining < 0 {
remaining = 0
}
secs := remaining / pr
if secs >= 0 && secs < 9e9 {
eta = time.Duration(secs * float64(time.Second))
hasEta = true
}
}
return pct, eta, hasEta, true
}
func dayProgress(s data) (int, int) {
const day = 86400.0
total := int((s.estimate.Hi-s.estimate.Lo)/day) + 1
cur := int((s.pos-s.estimate.Lo)/day) + 1
if cur < 1 {
cur = 1
}
if cur > total {
cur = total
}
return cur, total
}
func rate(samples []sample, now time.Time, usePos bool) (float64, bool) {
cut := now.Add(-rateWindow)
var first *sample
for i := range samples {
if usePos && !samples[i].hasPos {
continue
}
if !samples[i].t.Before(cut) {
first = &samples[i]
break
}
}
if first == nil {
for i := len(samples) - 1; i >= 0; i-- {
if !usePos || samples[i].hasPos {
first = &samples[i]
break
}
}
}
if first == nil || len(samples) == 0 {
return 0, false
}
last := &samples[len(samples)-1]
if usePos && !last.hasPos {
return 0, false
}
if first == last {
return 0, false
}
dt := last.t.Sub(first.t).Seconds()
if dt <= 0 {
return 0, false
}
var d float64
if usePos {
d = last.pos - first.pos
} else {
d = float64(last.rows - first.rows)
}
if d < 0 {
return 0, false
}
return d / dt, true
}
func positionRate(s data, now time.Time) (float64, bool) {
if v, ok := rate(s.samples, now, true); ok {
return v, true
}
if s.hasOrig && now.After(s.originT) {
return (s.pos - s.originP) / now.Sub(s.originT).Seconds(), true
}
return 0, false
}
func cumulativeRows(s data, now time.Time) float64 {
el := now.Sub(s.started).Seconds()
if el <= 0 {
return 0
}
return float64(s.read) / el
}
func humanRows(n int64) string {
switch {
case n >= 1_000_000_000:
return fmt.Sprintf("%.2fG", float64(n)/1e9)
case n >= 1_000_000:
return fmt.Sprintf("%.2fM", float64(n)/1e6)
case n >= 10_000:
return fmt.Sprintf("%.1fk", float64(n)/1e3)
default:
return strconv.FormatInt(n, 10)
}
}
func humanRate(v float64) string {
switch {
case v >= 1_000_000:
return fmt.Sprintf("%.2fM/s", v/1e6)
case v >= 10_000:
return fmt.Sprintf("%.1fk/s", v/1e3)
case v >= 1:
return fmt.Sprintf("%.0f/s", v)
default:
return "0/s"
}
}
func humanBytes(n int64) string {
const unit = 1024
if n < unit {
return fmt.Sprintf("%dB", n)
}
div, exp := int64(unit), 0
for v := n / unit; v >= unit; v /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f%ciB", float64(n)/float64(div), "KMGTPE"[exp])
}
func humanDuration(d time.Duration) string {
if d < 0 {
d = 0
}
if d < time.Second {
return fmt.Sprintf("%dms", d.Milliseconds())
}
secs := int(d.Seconds())
switch {
case secs < 60:
return fmt.Sprintf("%ds", secs)
case secs < 3600:
return fmt.Sprintf("%dm%02ds", secs/60, secs%60)
default:
return fmt.Sprintf("%dh%02dm", secs/3600, (secs%3600)/60)
}
}
func truncate(s string, max int) string {
if len(s) <= max {
return s
}
if max <= 0 {
return ""
}
runes := []rune(s)
if len(runes) <= max {
return s
}
return string(runes[:max])
}
// IsTerminal reports whether w is a character device (a terminal).
func IsTerminal(w io.Writer) bool {
f, ok := w.(*os.File)
if !ok {
return false
}
st, err := f.Stat()
if err != nil {
return false
}
return st.Mode()&os.ModeCharDevice != 0
}
+252
View File
@@ -0,0 +1,252 @@
package progress
import (
"bytes"
"strings"
"testing"
"time"
)
type clock struct{ t time.Time }
func (c *clock) now() time.Time { return c.t }
func (c *clock) add(d time.Duration) { c.t = c.t.Add(d) }
func newTestReporter(t *testing.T, live, enabled bool, opts ...func(*Options)) (*Reporter, *bytes.Buffer, *clock) {
t.Helper()
var buf bytes.Buffer
clk := &clock{t: time.Unix(1_700_000_000, 0)}
o := Options{Out: &buf, Enabled: enabled, Live: live, Interval: time.Second, Now: clk.now}
for _, fn := range opts {
fn(&o)
}
r := New(o)
r.mu.Lock()
r.started = true
r.initLocked(clk.t)
r.mu.Unlock()
return r, &buf, clk
}
func TestLiveLineHasRequiredFields(t *testing.T) {
r, buf, clk := newTestReporter(t, true, true)
r.SetEstimate(&Estimate{Kind: "day", Lo: 0, Hi: 10 * 86400})
r.Phase("scan", "")
r.SetShard(12)
r.SetPosition(3 * 86400)
r.AddRows(1000)
clk.add(time.Second)
r.SetPosition(4 * 86400)
r.AddRows(1000)
r.maybeRender(true)
out := buf.String()
if !strings.HasPrefix(out, "\r") {
t.Fatalf("live line must start with carriage return: %q", out)
}
if strings.Contains(out, "\n") {
t.Fatalf("live line must not emit a newline: %q", out)
}
for _, want := range []string{"rows", "~40%", "day 5/11", "shard 12", "scan", "eta ~", "elapsed"} {
if !strings.Contains(out, want) {
t.Fatalf("live line %q missing %q", out, want)
}
}
}
func TestLiveLineOmitsEstimateWhenUnavailable(t *testing.T) {
r, buf, _ := newTestReporter(t, true, true)
r.Phase("scan", "")
r.AddRows(10)
r.maybeRender(true)
out := buf.String()
if strings.Contains(out, "%") || strings.Contains(out, "eta") {
t.Fatalf("unexpected estimate in %q", out)
}
if !strings.Contains(out, "rows") {
t.Fatalf("missing rows in %q", out)
}
}
func TestLineRenderingIsStableKeyValue(t *testing.T) {
r, buf, clk := newTestReporter(t, false, true)
r.SetEstimate(&Estimate{Kind: "key", Lo: 0, Hi: 1000})
r.Phase("db", "")
r.SetPosition(100)
r.AddRows(500)
clk.add(time.Second)
r.SetPosition(500)
r.AddRows(500)
r.maybeRender(true)
out := buf.String()
if strings.Contains(out, "\r") {
t.Fatalf("line mode must not use carriage returns: %q", out)
}
if !strings.HasSuffix(out, "\n") {
t.Fatalf("line mode must end with a newline: %q", out)
}
for _, want := range []string{
"progress: rows=1000", "rate=", "phase=db", "phase_elapsed=", "elapsed=",
"estimate=key", "pct=~50.0", "eta=~",
} {
if !strings.Contains(out, want) {
t.Fatalf("line %q missing %q", out, want)
}
}
}
func TestLineRenderingMarksEstimateNone(t *testing.T) {
r, buf, _ := newTestReporter(t, false, true)
r.Phase("scan", "")
r.AddRows(100)
r.maybeRender(true)
if out := buf.String(); !strings.Contains(out, "estimate=none") {
t.Fatalf("want estimate=none, got %q", out)
}
}
func TestIntervalThrottle(t *testing.T) {
r, buf, clk := newTestReporter(t, false, true)
r.maybeRender(true)
if n := strings.Count(buf.String(), "\n"); n != 1 {
t.Fatalf("first frame not drawn: %q", buf.String())
}
clk.add(200 * time.Millisecond)
r.maybeRender(false)
if n := strings.Count(buf.String(), "\n"); n != 1 {
t.Fatalf("throttled frame drawn early: %q", buf.String())
}
clk.add(time.Second)
r.maybeRender(false)
if n := strings.Count(buf.String(), "\n"); n != 2 {
t.Fatalf("frame after interval not drawn: %q", buf.String())
}
}
func TestRowThresholdRenders(t *testing.T) {
r, buf, _ := newTestReporter(t, false, true, func(o *Options) { o.RowEvery = 100 })
r.AddRows(100)
if n := strings.Count(buf.String(), "\n"); n != 1 {
t.Fatalf("crossing the first threshold did not render: %q", buf.String())
}
r.AddRows(50)
if n := strings.Count(buf.String(), "\n"); n != 1 {
t.Fatalf("sub-threshold rows must not render: %q", buf.String())
}
r.AddRows(50)
if n := strings.Count(buf.String(), "\n"); n != 2 {
t.Fatalf("crossing the next threshold did not render: %q", buf.String())
}
}
func TestMessageClearsLiveLine(t *testing.T) {
r, buf, _ := newTestReporter(t, true, true)
r.AddRows(1000)
r.maybeRender(true)
width := r.lastWidth
buf.Reset()
r.Message("warning: %s", "something")
out := buf.String()
if !strings.Contains(out, "\r"+strings.Repeat(" ", width)+"\r") {
t.Fatalf("message did not clear the live line: %q", out)
}
if !strings.Contains(out, "warning: something\n") {
t.Fatalf("message missing: %q", out)
}
if strings.Contains(out, "rows=") {
t.Fatalf("live line leaked into message: %q", out)
}
}
func TestShortLineIsPadded(t *testing.T) {
r, buf, clk := newTestReporter(t, true, true)
r.SetEstimate(&Estimate{Kind: "day", Lo: 0, Hi: 100 * 86400})
r.SetPosition(50 * 86400)
r.AddRows(1000)
r.maybeRender(true)
first := r.lastWidth
buf.Reset()
r.SetEstimate(nil)
clk.add(2 * time.Second)
r.AddRows(1000)
r.maybeRender(true)
second := buf.String()
if r.lastWidth >= first {
t.Fatalf("second frame should be shorter: %d -> %d (%q)", first, r.lastWidth, second)
}
if !strings.HasSuffix(second, strings.Repeat(" ", first-r.lastWidth)) {
t.Fatalf("short frame not padded to previous width: %q", second)
}
}
func TestDoneSummary(t *testing.T) {
r, buf, clk := newTestReporter(t, false, true)
r.SetShard(3)
r.SetShards(3)
r.SetBytes(4096)
r.AddRows(1500)
clk.add(3 * time.Second)
r.Done()
out := buf.String()
for _, want := range []string{"done rows=1500", "elapsed=3s", "shards=3", "bytes=4096"} {
if !strings.Contains(out, want) {
t.Fatalf("done line %q missing %q", out, want)
}
}
}
func TestInterruptedSnapshot(t *testing.T) {
r, buf, _ := newTestReporter(t, true, true)
r.Phase("file", "")
r.AddRows(42)
r.maybeRender(true)
buf.Reset()
r.Interrupted()
out := buf.String()
if !strings.Contains(out, "progress: rows=42") {
t.Fatalf("final snapshot missing: %q", out)
}
if !strings.Contains(out, "phase=file") {
t.Fatalf("final snapshot missing phase: %q", out)
}
}
func TestDisabledReporterIsSilentButMessagesWork(t *testing.T) {
r, buf, _ := newTestReporter(t, false, false)
r.AddRows(10)
r.maybeRender(true)
r.Done()
if buf.Len() != 0 {
t.Fatalf("disabled reporter emitted progress: %q", buf.String())
}
r.Message("warning: still visible")
if !strings.Contains(buf.String(), "warning: still visible") {
t.Fatalf("messages must survive disabled progress: %q", buf.String())
}
}
func TestRepeatedPhaseKeepsClock(t *testing.T) {
r, buf, clk := newTestReporter(t, false, true)
r.Phase("lag-wait", "reason a")
clk.add(3 * time.Second)
r.Phase("lag-wait", "reason b")
r.maybeRender(true)
out := buf.String()
if !strings.Contains(out, "phase=lag-wait") || !strings.Contains(out, "phase_elapsed=3s") {
t.Fatalf("repeated phase reset its clock: %q", out)
}
}
func TestLagWaitShowsReason(t *testing.T) {
r, buf, _ := newTestReporter(t, false, true)
r.Phase("lag-wait", "replica lag 3s exceeds 1s")
r.AddRows(10)
r.maybeRender(true)
out := buf.String()
if !strings.Contains(out, "phase=lag-wait") || !strings.Contains(out, "reason=replica_lag_3s_exceeds_1s") {
t.Fatalf("lag reason missing: %q", out)
}
}
+187
View File
@@ -0,0 +1,187 @@
// Package purge deletes archived rows from the source table, either by an
// explicit key IN list or by key range.
package purge
import (
"context"
"database/sql"
"fmt"
"strconv"
"archer/internal/row"
"archer/internal/sqlutil"
)
// Execer is satisfied by *sql.DB, *sql.Tx and *sql.Conn.
type Execer interface {
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
}
const maxParams = 1000
// Purger deletes rows chunk by chunk.
type Purger struct {
DB *sql.DB
DBName string
Table string
KeyCols []string
Where string
Bulk bool
SafeAutoInc bool
AutoIncKey bool
KeyIdx []int
}
// New builds a Purger. keyIdx is unused for deletion but kept for symmetry
// with the scan configuration.
func New(db *sql.DB, dbName, table string, keyCols []string, where string, bulk, safe, autoInc bool) *Purger {
return &Purger{
DB: db,
DBName: dbName,
Table: table,
KeyCols: keyCols,
Where: where,
Bulk: bulk,
SafeAutoInc: safe,
AutoIncKey: autoInc,
}
}
// DeleteChunk removes the rows read in this chunk using exec. It returns the
// number of affected rows.
func (p *Purger) DeleteChunk(ctx context.Context, exec Execer, c *row.Chunk) (int64, error) {
if len(c.Keys) == 0 {
return 0, nil
}
keys := c.Keys
var maxKey *row.KeyPart
if p.SafeAutoInc && p.AutoIncKey && len(p.KeyCols) == 1 {
v, err := p.currentMax(ctx)
if err != nil {
return 0, err
}
maxKey = v
}
if p.Bulk && maxKey == nil {
return p.deleteRange(ctx, exec, c)
}
return p.deleteInList(ctx, exec, keys, maxKey)
}
func (p *Purger) table() string {
return sqlutil.Qualified(p.DBName, p.Table)
}
func (p *Purger) currentMax(ctx context.Context) (*row.KeyPart, error) {
q := fmt.Sprintf("SELECT MAX(%s) FROM %s", sqlutil.Ident(p.KeyCols[0]), p.table())
var v sql.NullString
if err := p.DB.QueryRowContext(ctx, q).Scan(&v); err != nil {
return nil, fmt.Errorf("read max auto-increment key: %w", err)
}
if !v.Valid {
return nil, nil
}
return &row.KeyPart{Val: v.String}, nil
}
func (p *Purger) deleteRange(ctx context.Context, exec Execer, c *row.Chunk) (int64, error) {
lo, hi := c.FirstKey, c.LastKey
if lo.IsZero() || hi.IsZero() {
return 0, nil
}
var (
q string
args []any
)
if len(p.KeyCols) == 1 {
q = fmt.Sprintf("DELETE FROM %s WHERE %s BETWEEN ? AND ? AND (%s)",
p.table(), sqlutil.Ident(p.KeyCols[0]), p.Where)
args = []any{lo[0].DriverValue(), hi[0].DriverValue()}
} else {
q = fmt.Sprintf("DELETE FROM %s WHERE %s >= %s AND %s <= %s AND (%s)",
p.table(), sqlutil.RowTuple(p.KeyCols), sqlutil.ParamTuple(len(p.KeyCols)),
sqlutil.RowTuple(p.KeyCols), sqlutil.ParamTuple(len(p.KeyCols)), p.Where)
args = append(args, lo.Args()...)
args = append(args, hi.Args()...)
}
res, err := exec.ExecContext(ctx, q, args...)
if err != nil {
return 0, fmt.Errorf("bulk delete: %w", err)
}
return res.RowsAffected()
}
func (p *Purger) deleteInList(ctx context.Context, exec Execer, keys []row.Key, maxKey *row.KeyPart) (int64, error) {
batch := maxParams / len(p.KeyCols)
if batch < 1 {
batch = 1
}
var total int64
for start := 0; start < len(keys); start += batch {
end := start + batch
if end > len(keys) {
end = len(keys)
}
var args [][]any
for _, k := range keys[start:end] {
if maxKey != nil && keyEquals(k[len(k)-1], *maxKey) {
continue
}
args = append(args, k.Args())
}
if len(args) == 0 {
continue
}
q := fmt.Sprintf("DELETE FROM %s WHERE %s AND (%s)",
p.table(), sqlutil.InList(p.KeyCols, len(args)), p.Where)
flat := make([]any, 0, len(args)*len(p.KeyCols))
for _, a := range args {
flat = append(flat, a...)
}
res, err := exec.ExecContext(ctx, q, flat...)
if err != nil {
return total, fmt.Errorf("delete chunk: %w", err)
}
n, err := res.RowsAffected()
if err != nil {
return total, err
}
total += n
}
return total, nil
}
func keyEquals(a, b row.KeyPart) bool {
if a.Null || b.Null {
return a.Null && b.Null
}
if a.Val == b.Val {
return true
}
ai, err1 := strconv.ParseInt(a.Val, 10, 64)
bi, err2 := strconv.ParseInt(b.Val, 10, 64)
return err1 == nil && err2 == nil && ai == bi
}
// DryRunSQL returns representative delete statements.
func (p *Purger) DryRunSQL() []string {
in := fmt.Sprintf("DELETE FROM %s WHERE %s IN (?, ?) AND (%s)",
p.table(), sqlutil.RowTuple(p.KeyCols), p.Where)
if len(p.KeyCols) == 1 {
in = fmt.Sprintf("DELETE FROM %s WHERE %s IN (?, ?) AND (%s)",
p.table(), sqlutil.Ident(p.KeyCols[0]), p.Where)
}
out := []string{in}
if p.Bulk {
if len(p.KeyCols) == 1 {
out = append(out, fmt.Sprintf("DELETE FROM %s WHERE %s BETWEEN ? AND ? AND (%s) -- --bulk-delete",
p.table(), sqlutil.Ident(p.KeyCols[0]), p.Where))
} else {
out = append(out, fmt.Sprintf("DELETE FROM %s WHERE %s >= %s AND %s <= %s AND (%s) -- --bulk-delete",
p.table(), sqlutil.RowTuple(p.KeyCols), sqlutil.ParamTuple(len(p.KeyCols)),
sqlutil.RowTuple(p.KeyCols), sqlutil.ParamTuple(len(p.KeyCols)), p.Where))
}
}
return out
}
+68
View File
@@ -0,0 +1,68 @@
package row
import "bytes"
// csvEncoder emits RFC4180 CSV with a column-name header and CRLF line
// endings. NULL is written as the configured token and empty strings as empty
// fields.
type csvEncoder struct {
cols []Column
idx []int
null string
}
func (e *csvEncoder) Header() []byte {
var b bytes.Buffer
for n, i := range e.idx {
if n > 0 {
b.WriteByte(',')
}
writeCSVField(&b, []byte(e.cols[i].Name))
}
b.WriteString("\r\n")
return b.Bytes()
}
func (e *csvEncoder) Encode(dst []byte, r Row) ([]byte, error) {
for n, i := range e.idx {
if n > 0 {
dst = append(dst, ',')
}
v := r.Values[i]
if v.IsNull() {
dst = append(dst, e.null...)
continue
}
dst = appendCSVField(dst, v.Raw)
}
return append(dst, '\r', '\n'), nil
}
func needsCSVQuote(b []byte) bool {
for _, c := range b {
switch c {
case ',', '"', '\r', '\n':
return true
}
}
return false
}
func appendCSVField(dst, field []byte) []byte {
if !needsCSVQuote(field) {
return append(dst, field...)
}
dst = append(dst, '"')
for _, c := range field {
if c == '"' {
dst = append(dst, '"', '"')
continue
}
dst = append(dst, c)
}
return append(dst, '"')
}
func writeCSVField(b *bytes.Buffer, field []byte) {
b.Write(appendCSVField(nil, field))
}
+44
View File
@@ -0,0 +1,44 @@
package row
// dumpEncoder emits MySQL LOAD DATA compatible TSV. Fields are separated by
// TAB, rows by LF, NULL is \N and the escape table matches
// SELECT ... INTO OUTFILE with the default FIELDS ESCAPED BY '\\'.
type dumpEncoder struct {
idx []int
}
func (e *dumpEncoder) Header() []byte { return nil }
func (e *dumpEncoder) Encode(dst []byte, r Row) ([]byte, error) {
for n, i := range e.idx {
if n > 0 {
dst = append(dst, '\t')
}
v := r.Values[i]
if v.IsNull() {
dst = append(dst, '\\', 'N')
continue
}
for _, b := range v.Raw {
switch b {
case '\\':
dst = append(dst, '\\', '\\')
case '\t':
dst = append(dst, '\\', 't')
case '\n':
dst = append(dst, '\\', 'n')
case '\r':
dst = append(dst, '\\', 'r')
case 0:
dst = append(dst, '\\', '0')
case '\b':
dst = append(dst, '\\', 'b')
case 0x1a:
dst = append(dst, '\\', 'Z')
default:
dst = append(dst, b)
}
}
}
return append(dst, '\n'), nil
}
+52
View File
@@ -0,0 +1,52 @@
package row
import "fmt"
// Options carries encoder-level knobs shared by all formats.
type Options struct {
CSVNull string
}
// Encoder appends the textual representation of a row to dst. It is not safe
// for concurrent use; each shard owns its own encoder.
type Encoder interface {
// Header returns bytes that must be written at the start of every shard.
Header() []byte
// Encode appends one row to dst.
Encode(dst []byte, r Row) ([]byte, error)
}
// NewEncoder builds an encoder for the given format. cols describes every
// column in the row (select order) and idx selects which of them are emitted,
// in output order.
func NewEncoder(format string, cols []Column, idx []int, opts Options) (Encoder, error) {
if len(idx) == 0 {
idx = make([]int, len(cols))
for i := range cols {
idx[i] = i
}
}
switch format {
case "dump":
return &dumpEncoder{idx: idx}, nil
case "csv":
return &csvEncoder{cols: cols, idx: idx, null: opts.CSVNull}, nil
case "jsonl":
return &jsonlEncoder{cols: cols, idx: idx}, nil
default:
return nil, fmt.Errorf("unsupported format %q", format)
}
}
// FormatExt returns the file extension for a format.
func FormatExt(format string) string {
switch format {
case "dump":
return "dump"
case "csv":
return "csv"
case "jsonl":
return "jsonl"
}
return format
}
+105
View File
@@ -0,0 +1,105 @@
package row
import (
"strings"
"testing"
)
func vals(raw ...any) Row {
r := Row{}
for _, v := range raw {
switch x := v.(type) {
case nil:
r.Values = append(r.Values, Value{})
case string:
r.Values = append(r.Values, Value{Raw: []byte(x)})
}
}
return r
}
func cols(names ...string) []Column {
out := make([]Column, len(names))
for i, n := range names {
out[i] = Column{Name: n, Type: "varchar"}
}
return out
}
func TestDumpEncoding(t *testing.T) {
enc, err := NewEncoder("dump", cols("a", "b", "c"), nil, Options{})
if err != nil {
t.Fatal(err)
}
r := vals("x\ty", nil, `a\nb`)
got, err := enc.Encode(nil, r)
if err != nil {
t.Fatal(err)
}
want := "x\\ty\t\\N\ta\\\\nb\n"
if string(got) != want {
t.Fatalf("got %q want %q", got, want)
}
}
func TestDumpEscapesControlBytes(t *testing.T) {
enc, _ := NewEncoder("dump", cols("a"), nil, Options{})
r := Row{Values: []Value{{Raw: []byte{'a', 0, '\b', 0x1a, '\r'}}}}
got, _ := enc.Encode(nil, r)
want := "a\\0\\b\\Z\\r\n"
if string(got) != want {
t.Fatalf("got %q want %q", got, want)
}
}
func TestCSVEncoding(t *testing.T) {
enc, err := NewEncoder("csv", cols("a", "b", "c", "d"), nil, Options{CSVNull: `\N`})
if err != nil {
t.Fatal(err)
}
if h := string(enc.Header()); h != "a,b,c,d\r\n" {
t.Fatalf("header %q", h)
}
r := vals(`a,"b"`, "", nil, "plain")
got, _ := enc.Encode(nil, r)
want := "\"a,\"\"b\"\"\",,\\N,plain\r\n"
if string(got) != want {
t.Fatalf("got %q want %q", got, want)
}
}
func TestCSVNullToken(t *testing.T) {
enc, _ := NewEncoder("csv", cols("a"), nil, Options{CSVNull: "NULL"})
got, _ := enc.Encode(nil, vals(nil))
if string(got) != "NULL\r\n" {
t.Fatalf("got %q", got)
}
}
func TestJSONLTypes(t *testing.T) {
columns := []Column{
{Name: "id", Type: "bigint"},
{Name: "price", Type: "decimal"},
{Name: "created", Type: "datetime"},
{Name: "day", Type: "date"},
{Name: "blob", Type: "blob"},
{Name: "note", Type: "varchar"},
{Name: "missing", Type: "varchar"},
}
enc, _ := NewEncoder("jsonl", columns, nil, Options{})
r := vals("42", "12.30", "2024-01-02 03:04:05", "2024-01-02", "hi", `line "one"`, nil)
got, _ := enc.Encode(nil, r)
want := `{"id":42,"price":"12.30","created":"2024-01-02T03:04:05Z","day":"2024-01-02","blob":"aGk=","note":"line \"one\"","missing":null}` + "\n"
if string(got) != want {
t.Fatalf("got %s\nwant %s", got, want)
}
}
func TestJSONLColumnProjection(t *testing.T) {
columns := []Column{{Name: "a", Type: "int"}, {Name: "b", Type: "int"}, {Name: "c", Type: "int"}}
enc, _ := NewEncoder("jsonl", columns, []int{2, 0}, Options{})
got, _ := enc.Encode(nil, vals("1", "2", "3"))
if strings.TrimSpace(string(got)) != `{"c":3,"a":1}` {
t.Fatalf("got %s", got)
}
}
+125
View File
@@ -0,0 +1,125 @@
package row
import (
"encoding/base64"
"time"
)
// jsonlEncoder emits one JSON object per line. Types are mapped deliberately
// so downstream tools see useful values: decimals become strings to preserve
// precision, binary columns become base64, date/time columns become strings
// and NULL becomes JSON null.
type jsonlEncoder struct {
cols []Column
idx []int
}
func (e *jsonlEncoder) Header() []byte { return nil }
func (e *jsonlEncoder) Encode(dst []byte, r Row) ([]byte, error) {
dst = append(dst, '{')
for n, i := range e.idx {
if n > 0 {
dst = append(dst, ',')
}
dst = appendJSONString(dst, []byte(e.cols[i].Name))
dst = append(dst, ':')
dst = appendJSONValue(dst, e.cols[i], r.Values[i])
}
return append(dst, '}', '\n'), nil
}
func appendJSONValue(dst []byte, col Column, v Value) []byte {
if v.IsNull() {
return append(dst, "null"...)
}
switch {
case IsInteger(col.Type), IsFloat(col.Type):
if looksNumeric(v.Raw) {
return append(dst, v.Raw...)
}
return appendJSONString(dst, v.Raw)
case IsDecimal(col.Type):
return appendJSONString(dst, v.Raw)
case IsBinary(col.Type):
dst = append(dst, '"')
dst = append(dst, base64.StdEncoding.EncodeToString(v.Raw)...)
return append(dst, '"')
case IsDate(col.Type):
return appendJSONString(dst, v.Raw)
case IsDateTime(col.Type):
if s, ok := formatRFC3339(v.Raw); ok {
return appendJSONString(dst, []byte(s))
}
return appendJSONString(dst, v.Raw)
default:
return appendJSONString(dst, v.Raw)
}
}
func looksNumeric(b []byte) bool {
if len(b) == 0 {
return false
}
for i, c := range b {
if (c >= '0' && c <= '9') || c == '-' || c == '+' || c == '.' || c == 'e' || c == 'E' {
continue
}
if i == 0 && c == ' ' {
continue
}
return false
}
return true
}
var dateTimeLayouts = []string{
"2006-01-02 15:04:05.999999999",
"2006-01-02 15:04:05",
"2006-01-02T15:04:05Z07:00",
"2006-01-02T15:04:05.999999999Z07:00",
}
func formatRFC3339(raw []byte) (string, bool) {
s := string(raw)
for _, layout := range dateTimeLayouts {
t, err := time.ParseInLocation(layout, s, time.UTC)
if err != nil {
continue
}
if t.Nanosecond() != 0 {
return t.UTC().Format(time.RFC3339Nano), true
}
return t.UTC().Format(time.RFC3339), true
}
return "", false
}
const hexDigits = "0123456789abcdef"
func appendJSONString(dst, s []byte) []byte {
dst = append(dst, '"')
for _, c := range s {
switch c {
case '"', '\\':
dst = append(dst, '\\', c)
case '\n':
dst = append(dst, '\\', 'n')
case '\r':
dst = append(dst, '\\', 'r')
case '\t':
dst = append(dst, '\\', 't')
case '\b':
dst = append(dst, '\\', 'b')
case '\f':
dst = append(dst, '\\', 'f')
default:
if c < 0x20 {
dst = append(dst, '\\', 'u', '0', '0', hexDigits[c>>4], hexDigits[c&0xf])
} else {
dst = append(dst, c)
}
}
}
return append(dst, '"')
}
+152
View File
@@ -0,0 +1,152 @@
// Package row defines scanned values, key tuples and the line encoders used
// to serialize rows into dump, csv and jsonl shards.
package row
import (
"strings"
)
// Value is one column value as returned by the MySQL driver with
// parseTime disabled. A nil Raw means SQL NULL; an empty non-nil Raw means
// the empty string.
type Value struct {
Raw []byte
}
func (v Value) IsNull() bool { return v.Raw == nil }
func (v Value) Bytes() []byte { return v.Raw }
func (v Value) String() string { return string(v.Raw) }
// DriverValue returns the value as a bind parameter.
func (v Value) DriverValue() any {
if v.Raw == nil {
return nil
}
return v.Raw
}
// KeyPart is a single component of a key tuple. Values are held as text so a
// key can be serialized into state.json and rebound to the driver without
// carrying driver-specific types.
type KeyPart struct {
Null bool `json:"null,omitempty"`
Val string `json:"val,omitempty"`
}
// DriverValue converts the part into a value accepted by database/sql.
func (p KeyPart) DriverValue() any {
if p.Null {
return nil
}
return p.Val
}
// Key is a tuple of values identifying a row position in a unique index.
type Key []KeyPart
func KeyOf(values ...string) Key {
parts := make(Key, len(values))
for i, v := range values {
parts[i] = KeyPart{Val: v}
}
return parts
}
// Args returns the key as driver bind parameters.
func (k Key) Args() []any {
out := make([]any, len(k))
for i, p := range k {
out[i] = p.DriverValue()
}
return out
}
func (k Key) IsZero() bool { return len(k) == 0 }
func (k Key) Equal(o Key) bool {
if len(k) != len(o) {
return false
}
for i := range k {
if k[i] != o[i] {
return false
}
}
return true
}
func (k Key) String() string {
if len(k) == 0 {
return "()"
}
parts := make([]string, len(k))
for i, p := range k {
if p.Null {
parts[i] = "NULL"
} else {
parts[i] = p.Val
}
}
return "(" + strings.Join(parts, ",") + ")"
}
// FirstLast returns the two parts of a composite (or single) key.
func (k Key) FirstLast() (KeyPart, KeyPart) {
if len(k) == 0 {
return KeyPart{Null: true}, KeyPart{Null: true}
}
return k[0], k[len(k)-1]
}
// Row holds a scanned row in select-list order.
type Row struct {
Values []Value
}
// Chunk is a batch of rows produced by one scan step. Keys is parallel to
// Rows and holds the unique-index key of each row; Buckets is parallel to
// Rows and is populated only in date sharding mode.
type Chunk struct {
Rows []Row
Keys []Key
Buckets []string
FirstKey Key
LastKey Key
}
func (c *Chunk) Len() int { return len(c.Rows) }
// Column describes one exported column for encoding purposes.
type Column struct {
Name string
Type string // lowercase information_schema DATA_TYPE
}
func isOneOf(t string, set ...string) bool {
for _, s := range set {
if t == s {
return true
}
}
return false
}
func IsInteger(t string) bool {
return isOneOf(t, "tinyint", "smallint", "mediumint", "int", "integer", "bigint", "year")
}
func IsFloat(t string) bool {
return isOneOf(t, "float", "double", "real")
}
func IsDecimal(t string) bool { return isOneOf(t, "decimal", "numeric") }
func IsBinary(t string) bool {
return isOneOf(t, "binary", "varbinary", "tinyblob", "blob", "mediumblob", "longblob", "bit", "geometry", "geometrycollection", "point", "linestring", "polygon", "multipoint", "multilinestring", "multipolygon")
}
func IsDate(t string) bool { return t == "date" }
func IsDateTime(t string) bool { return isOneOf(t, "datetime", "timestamp") }
func IsTimeType(t string) bool { return isOneOf(t, "date", "datetime", "timestamp") }
+148
View File
@@ -0,0 +1,148 @@
package scan
import (
"context"
"io"
"strings"
"archer/internal/row"
"archer/internal/sqlutil"
)
// DateAScanner uses an index that starts with (shard_column, key...) and
// performs keyset pagination over it. NULL rows are scanned in a separate
// first phase because row-constructor comparisons do not advance past NULL.
type DateAScanner struct {
o *Options
phase string // null | value
cursor row.Key
done bool
}
func NewDateA(o Options) *DateAScanner {
s := &DateAScanner{o: &o, phase: "null"}
if o.Resume != nil {
switch o.Resume.Phase {
case "null":
s.phase = "null"
s.cursor = o.Resume.Key
case "value":
s.phase = "value"
s.cursor = o.Resume.Key
}
}
return s
}
func (s *DateAScanner) Next(ctx context.Context) (*row.Chunk, error) {
for {
if s.done {
return nil, io.EOF
}
var (
c *row.Chunk
exhausted bool
err error
)
if s.phase == "null" {
c, exhausted, err = s.queryNull(ctx)
} else {
c, exhausted, err = s.queryValue(ctx)
}
if err != nil {
return nil, err
}
if c.Len() == 0 {
if s.phase == "null" {
s.phase = "value"
s.cursor = nil
continue
}
s.done = true
return nil, io.EOF
}
if s.phase == "null" {
if exhausted {
s.phase = "value"
s.cursor = nil
} else {
s.cursor = c.LastKey
}
} else {
s.cursor = s.o.compositeCursor(c.Rows[len(c.Rows)-1])
c.LastKey = s.cursor
if exhausted {
s.done = true
}
}
return c, nil
}
}
func (s *DateAScanner) queryNull(ctx context.Context) (*row.Chunk, bool, error) {
var b strings.Builder
b.WriteString(s.o.selectSQL())
b.WriteString(" AND ")
b.WriteString(sqlutil.Ident(s.o.Shard.Column))
b.WriteString(" IS NULL")
var args []any
if len(s.cursor) > 0 {
b.WriteString(" AND ")
b.WriteString(sqlutil.GreaterThan(s.o.KeyCols))
args = append(args, s.cursor.Args()...)
}
b.WriteString(" ORDER BY ")
b.WriteString(sqlutil.Columns(s.o.KeyCols))
b.WriteString(" LIMIT ?")
args = append(args, s.o.ChunkSize)
return s.run(ctx, b.String(), args)
}
func (s *DateAScanner) queryValue(ctx context.Context) (*row.Chunk, bool, error) {
var b strings.Builder
b.WriteString(s.o.selectSQL())
b.WriteString(" AND ")
b.WriteString(sqlutil.Ident(s.o.Shard.Column))
b.WriteString(" IS NOT NULL")
cols := append([]string{s.o.Shard.Column}, s.o.KeyCols...)
var args []any
if len(s.cursor) > 0 {
b.WriteString(" AND ")
b.WriteString(sqlutil.GreaterThan(cols))
args = append(args, s.cursor.Args()...)
}
b.WriteString(" ORDER BY ")
b.WriteString(sqlutil.Columns(cols))
b.WriteString(" LIMIT ?")
args = append(args, s.o.ChunkSize)
c, exhausted, err := s.run(ctx, b.String(), args)
if err != nil {
return nil, false, err
}
if c.Len() > 0 {
c.LastKey = s.o.compositeCursor(c.Rows[len(c.Rows)-1])
}
return c, exhausted, nil
}
func (s *DateAScanner) run(ctx context.Context, q string, args []any) (*row.Chunk, bool, error) {
rows, err := s.o.DB.QueryContext(ctx, q, args...)
if err != nil {
return nil, false, err
}
defer rows.Close()
c, exhausted, err := s.o.readChunk(rows, s.o.ChunkSize)
if err != nil {
return nil, false, err
}
if err := s.o.fillBuckets(c); err != nil {
return nil, false, err
}
return c, exhausted, nil
}
func (s *DateAScanner) ResumeState() CursorState {
return CursorState{Phase: s.phase, Key: s.cursor}
}
func (s *DateAScanner) Close() error { return nil }
+350
View File
@@ -0,0 +1,350 @@
package scan
import (
"context"
"database/sql"
"fmt"
"io"
"strconv"
"strings"
"time"
"archer/internal/row"
"archer/internal/schema"
"archer/internal/sqlutil"
)
const zeroDay = "0000-00-00"
// DateBScanner enumerates days one at a time and streams each day's rows over
// a dedicated connection. It is used when only a single-column index on the
// shard column exists.
type DateBScanner struct {
o *Options
conn *sql.Conn
phase string // null | day
rows *sql.Rows
cursor row.Key
done bool
daysInited bool
hasDays bool
zeroPending bool
zeroDone bool
nextDay time.Time
maxDay time.Time
curDay string
nextDayStr string
resumeDay string
resumeKey row.Key
resumeUse bool
}
func NewDateB(ctx context.Context, o Options) (*DateBScanner, error) {
conn, err := o.DB.Conn(ctx)
if err != nil {
return nil, fmt.Errorf("acquire streaming connection: %w", err)
}
s := &DateBScanner{o: &o, conn: conn, phase: "null"}
if o.Resume != nil {
switch o.Resume.Phase {
case "null":
s.phase = "null"
s.cursor = o.Resume.Key
case "day":
s.phase = "day"
s.resumeDay = o.Resume.DayStart
s.resumeKey = o.Resume.Key
}
}
return s, nil
}
func (s *DateBScanner) Next(ctx context.Context) (*row.Chunk, error) {
for {
if s.done {
return nil, io.EOF
}
if s.rows == nil {
if err := s.openNext(ctx); err != nil {
if err == io.EOF {
s.done = true
return nil, io.EOF
}
return nil, err
}
}
c, exhausted, err := s.o.readChunk(s.rows, s.o.ChunkSize)
if err != nil {
return nil, err
}
if err := s.o.fillBuckets(c); err != nil {
return nil, err
}
if c.Len() > 0 {
if s.phase == "day" {
s.cursor = s.o.compositeCursor(c.Rows[len(c.Rows)-1])
} else {
s.cursor = c.Keys[len(c.Keys)-1]
}
c.LastKey = s.cursor
}
if exhausted {
if err := s.rows.Close(); err != nil {
return nil, err
}
s.rows = nil
if s.phase == "null" {
s.phase = "day"
s.cursor = nil
}
}
if c.Len() == 0 {
continue
}
return c, nil
}
}
func (s *DateBScanner) openNext(ctx context.Context) error {
if s.phase == "null" {
return s.openNull(ctx)
}
if !s.daysInited {
if err := s.initDays(ctx); err != nil {
return err
}
}
if s.zeroPending && !s.zeroDone {
s.zeroPending = false
s.zeroDone = true
s.curDay = zeroDay
s.nextDayStr = zeroDay
return s.openZero(ctx)
}
if !s.advanceDay() {
return io.EOF
}
return s.openDay(ctx)
}
func (s *DateBScanner) openNull(ctx context.Context) error {
var b strings.Builder
b.WriteString(s.o.selectSQL())
b.WriteString(" AND ")
b.WriteString(sqlutil.Ident(s.o.Shard.Column))
b.WriteString(" IS NULL")
var args []any
if len(s.cursor) > 0 {
b.WriteString(" AND ")
b.WriteString(sqlutil.GreaterThan(s.o.KeyCols))
args = append(args, s.cursor.Args()...)
}
b.WriteString(" ORDER BY ")
b.WriteString(sqlutil.Columns(s.o.KeyCols))
return s.query(ctx, b.String(), args)
}
func (s *DateBScanner) openZero(ctx context.Context) error {
var b strings.Builder
b.WriteString(s.o.selectSQL())
b.WriteString(" AND ")
b.WriteString(sqlutil.Ident(s.o.Shard.Column))
b.WriteString(" < ?")
upper := "0001-01-01"
if s.o.Shard.Kind == schema.ShardDateTime {
upper = "0001-01-01 00:00:00"
}
args := []any{upper}
cols := append([]string{s.o.Shard.Column}, s.o.KeyCols...)
if len(s.cursor) > 0 {
b.WriteString(" AND ")
b.WriteString(sqlutil.GreaterThan(cols))
args = append(args, s.cursor.Args()...)
}
b.WriteString(" ORDER BY ")
b.WriteString(sqlutil.Columns(cols))
s.nextDayStr = zeroDay
return s.query(ctx, b.String(), args)
}
func (s *DateBScanner) openDay(ctx context.Context) error {
var b strings.Builder
b.WriteString(s.o.selectSQL())
b.WriteString(" AND ")
col := sqlutil.Ident(s.o.Shard.Column)
b.WriteString(col)
b.WriteString(" >= ? AND ")
b.WriteString(col)
b.WriteString(" < ?")
var lo, hi any
switch s.o.Shard.Kind {
case schema.ShardDate:
lo = s.curDay
hi = s.nextDayStr
case schema.ShardDateTime:
lo = s.curDay + " 00:00:00"
hi = s.nextDayStr + " 00:00:00"
case schema.ShardEpoch:
lo = schema.TimeToEpoch(s.dayTime(s.curDay), s.o.Shard.Unit)
hi = schema.TimeToEpoch(s.dayTime(s.curDay).AddDate(0, 0, 1), s.o.Shard.Unit)
}
args := []any{lo, hi}
cols := append([]string{s.o.Shard.Column}, s.o.KeyCols...)
if len(s.cursor) > 0 {
b.WriteString(" AND ")
b.WriteString(sqlutil.GreaterThan(cols))
args = append(args, s.cursor.Args()...)
}
b.WriteString(" ORDER BY ")
b.WriteString(sqlutil.Columns(cols))
return s.query(ctx, b.String(), args)
}
func (s *DateBScanner) query(ctx context.Context, q string, args []any) error {
rows, err := s.conn.QueryContext(ctx, q, args...)
if err != nil {
return err
}
s.rows = rows
return nil
}
func (s *DateBScanner) dayTime(day string) time.Time {
t, _ := time.ParseInLocation("2006-01-02", day, s.o.ShardLoc)
return t
}
func (s *DateBScanner) initDays(ctx context.Context) error {
s.daysInited = true
q := fmt.Sprintf("SELECT MIN(%s), MAX(%s) FROM %s WHERE (%s) AND %s IS NOT NULL",
sqlutil.Ident(s.o.Shard.Column), sqlutil.Ident(s.o.Shard.Column),
s.o.qualified(), s.o.Where, sqlutil.Ident(s.o.Shard.Column))
var minRaw, maxRaw sql.NullString
if err := s.conn.QueryRowContext(ctx, q).Scan(&minRaw, &maxRaw); err != nil {
return fmt.Errorf("read shard column range: %w", err)
}
if !minRaw.Valid || !maxRaw.Valid {
return nil
}
switch s.o.Shard.Kind {
case schema.ShardDate, schema.ShardDateTime:
minDay := s.first10(minRaw.String)
maxDay := s.first10(maxRaw.String)
zeroExists := strings.HasPrefix(minDay, "0000-")
if zeroExists {
minDay = "0001-01-01"
}
s.zeroPending = s.wantZero(zeroExists)
if strings.HasPrefix(maxDay, "0000-") {
return nil
}
minT, err := time.ParseInLocation("2006-01-02", minDay, s.o.ShardLoc)
if err != nil {
return fmt.Errorf("parse shard range minimum %q: %w", minRaw.String, err)
}
maxT, err := time.ParseInLocation("2006-01-02", maxDay, s.o.ShardLoc)
if err != nil {
return fmt.Errorf("parse shard range maximum %q: %w", maxRaw.String, err)
}
s.minMax(minT, maxT)
case schema.ShardEpoch:
minV, err := strconv.ParseInt(strings.TrimSpace(minRaw.String), 10, 64)
if err != nil {
return fmt.Errorf("parse shard range minimum: %w", err)
}
maxV, err := strconv.ParseInt(strings.TrimSpace(maxRaw.String), 10, 64)
if err != nil {
return fmt.Errorf("parse shard range maximum: %w", err)
}
minT := schema.EpochTime(minV, s.o.Shard.Unit, s.o.ShardLoc)
maxT := schema.EpochTime(maxV, s.o.Shard.Unit, s.o.ShardLoc)
s.minMax(dayStart(minT, s.o.ShardLoc), dayStart(maxT, s.o.ShardLoc))
}
return nil
}
func (s *DateBScanner) wantZero(zeroExists bool) bool {
if s.resumeDay == "" {
return zeroExists
}
return s.resumeDay == zeroDay
}
func (s *DateBScanner) minMax(minT, maxT time.Time) {
s.hasDays = true
s.nextDay = minT
s.maxDay = maxT
if s.resumeDay != "" && s.resumeDay != zeroDay {
if d, err := time.ParseInLocation("2006-01-02", s.resumeDay, s.o.ShardLoc); err == nil && d.After(minT) {
s.nextDay = d
}
}
if s.resumeDay == zeroDay {
s.nextDay = minT
}
}
func (s *DateBScanner) advanceDay() bool {
if !s.hasDays {
return false
}
for !s.nextDay.After(s.maxDay) {
day := s.nextDay.Format("2006-01-02")
if s.resumeDay != "" && s.resumeDay != zeroDay && day < s.resumeDay && !s.resumeUse {
s.nextDay = s.nextDay.AddDate(0, 0, 1)
continue
}
s.curDay = day
s.nextDayStr = s.nextDay.AddDate(0, 0, 1).Format("2006-01-02")
s.nextDay = s.nextDay.AddDate(0, 0, 1)
if s.resumeDay != "" && day == s.resumeDay && !s.resumeUse {
s.cursor = s.resumeKey
s.resumeUse = true
} else {
s.cursor = nil
}
return true
}
return false
}
func (s *DateBScanner) first10(v string) string {
if len(v) < 10 {
return v
}
return v[:10]
}
func dayStart(t time.Time, loc *time.Location) time.Time {
return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, loc)
}
func (s *DateBScanner) ResumeState() CursorState {
phase := s.phase
day := ""
if phase == "day" {
day = s.curDay
}
return CursorState{Phase: phase, Key: s.cursor, DayStart: day}
}
func (s *DateBScanner) Close() error {
var firstErr error
if s.rows != nil {
if err := s.rows.Close(); err != nil {
firstErr = err
}
s.rows = nil
}
if s.conn != nil {
if err := s.conn.Close(); err != nil && firstErr == nil {
firstErr = err
}
s.conn = nil
}
return firstErr
}
+74
View File
@@ -0,0 +1,74 @@
package scan
import (
"context"
"io"
"strings"
"archer/internal/row"
"archer/internal/sqlutil"
)
// PKScanner walks the table by its chosen unique key using
// key > :last ORDER BY key LIMIT :chunk_size.
type PKScanner struct {
o *Options
cursor row.Key
done bool
}
func NewPK(o Options) *PKScanner {
s := &PKScanner{o: &o}
if o.Resume != nil && (o.Resume.Phase == "" || o.Resume.Phase == "pk") {
s.cursor = o.Resume.Key
}
return s
}
func (s *PKScanner) Next(ctx context.Context) (*row.Chunk, error) {
if s.done {
return nil, io.EOF
}
q, args := s.query()
rows, err := s.o.DB.QueryContext(ctx, q, args...)
if err != nil {
return nil, err
}
defer rows.Close()
chunk, exhausted, err := s.o.readChunk(rows, s.o.ChunkSize)
if err != nil {
return nil, err
}
if len(chunk.Rows) == 0 {
s.done = true
return nil, io.EOF
}
if exhausted {
s.done = true
}
s.cursor = chunk.LastKey
return chunk, nil
}
func (s *PKScanner) query() (string, []any) {
var b strings.Builder
b.WriteString(s.o.selectSQL())
var args []any
if len(s.cursor) > 0 {
b.WriteString(" AND ")
b.WriteString(sqlutil.GreaterThan(s.o.KeyCols))
args = append(args, s.cursor.Args()...)
}
b.WriteString(" ORDER BY ")
b.WriteString(sqlutil.Columns(s.o.KeyCols))
b.WriteString(" LIMIT ?")
args = append(args, s.o.ChunkSize)
return b.String(), args
}
func (s *PKScanner) Close() error { return nil }
func (s *PKScanner) ResumeState() CursorState {
return CursorState{Phase: "pk", Key: s.cursor}
}
+142
View File
@@ -0,0 +1,142 @@
// Package scan implements keyset (seek) chunked scanning of a source table.
package scan
import (
"context"
"database/sql"
"fmt"
"time"
"archer/internal/row"
"archer/internal/schema"
"archer/internal/sqlutil"
)
// CursorState is the resumable position of a scanner.
type CursorState struct {
Phase string
Key row.Key
DayStart string
}
// Scanner produces chunks in scan order.
type Scanner interface {
Next(ctx context.Context) (*row.Chunk, error) // io.EOF when exhausted
ResumeState() CursorState
Close() error
}
// Resume is the persisted scan position.
type Resume struct {
Phase string // pk|null|value|day
Key row.Key
DayStart string
}
// Options configures a Scanner.
type Options struct {
DB *sql.DB
DBName string
Table string
Where string
SelectCols []string
KeyCols []string
KeyIdx []int
ChunkSize int
Shard *schema.ShardSpec // nil outside date mode
ShardIdx int // -1 outside date mode
ShardLoc *time.Location
ZeroDate string
Resume *Resume
}
func (o *Options) qualified() string { return sqlutil.Qualified(o.DBName, o.Table) }
func (o *Options) selectSQL() string {
return "SELECT " + sqlutil.Columns(o.SelectCols) +
" FROM " + o.qualified() +
" WHERE (" + o.Where + ")"
}
func (o *Options) keyOf(r row.Row) row.Key {
k := make(row.Key, len(o.KeyIdx))
for i, idx := range o.KeyIdx {
v := r.Values[idx]
if v.IsNull() {
k[i] = row.KeyPart{Null: true}
} else {
k[i] = row.KeyPart{Val: string(v.Raw)}
}
}
return k
}
func (o *Options) compositeCursor(r row.Row) row.Key {
k := o.keyOf(r)
out := make(row.Key, 0, 1+len(k))
v := r.Values[o.ShardIdx]
if v.IsNull() {
out = append(out, row.KeyPart{Null: true})
} else {
out = append(out, row.KeyPart{Val: string(v.Raw)})
}
return append(out, k...)
}
func (o *Options) bucket(r row.Row) (string, error) {
return schema.BucketOf(r.Values[o.ShardIdx], o.Shard, o.ShardLoc, o.ZeroDate, o.keyOf(r))
}
// readChunk consumes up to limit rows from an open result set.
func (o *Options) readChunk(rows *sql.Rows, limit int) (*row.Chunk, bool, error) {
chunk := &row.Chunk{}
raw := make([]sql.RawBytes, len(o.SelectCols))
ptrs := make([]any, len(raw))
for i := range raw {
ptrs[i] = &raw[i]
}
exhausted := false
for len(chunk.Rows) < limit {
if !rows.Next() {
exhausted = true
break
}
if err := rows.Scan(ptrs...); err != nil {
return nil, false, fmt.Errorf("scan row: %w", err)
}
vals := make([]row.Value, len(raw))
for i, b := range raw {
if b == nil {
continue
}
c := make([]byte, len(b))
copy(c, b)
vals[i] = row.Value{Raw: c}
}
chunk.Rows = append(chunk.Rows, row.Row{Values: vals})
}
if err := rows.Err(); err != nil {
return nil, false, fmt.Errorf("read rows: %w", err)
}
for i := range chunk.Rows {
chunk.Keys = append(chunk.Keys, o.keyOf(chunk.Rows[i]))
}
if len(chunk.Rows) > 0 {
chunk.FirstKey = chunk.Keys[0]
chunk.LastKey = chunk.Keys[len(chunk.Keys)-1]
}
return chunk, exhausted, nil
}
func (o *Options) fillBuckets(c *row.Chunk) error {
c.Buckets = make([]string, len(c.Rows))
for i := range c.Rows {
b, err := o.bucket(c.Rows[i])
if err != nil {
return err
}
c.Buckets[i] = b
}
return nil
}
+219
View File
@@ -0,0 +1,219 @@
package schema
import (
"context"
"database/sql"
"fmt"
"strconv"
"strings"
"time"
"archer/internal/row"
"archer/internal/sqlutil"
)
// ShardKind is how a shard column's date is derived from its raw value.
type ShardKind int
const (
ShardNone ShardKind = iota
ShardDate
ShardDateTime
ShardEpoch
)
// ShardSpec is the resolved shard column and how to interpret it.
type ShardSpec struct {
Column string
Kind ShardKind
Unit int64 // epoch multiplier to nanoseconds: 1e9, 1e6, 1e3
Type string
}
// ResolveShardColumn validates --shard-column against the table and the unit
// flag. It returns a descriptive error listing candidates when the column is
// missing or the type is unsupported.
func ResolveShardColumn(ctx context.Context, db *sql.DB, t *Table, column, unitFlag, where string, loc *time.Location) (*ShardSpec, error) {
if column == "" {
return nil, fmt.Errorf("--shard-column is required for --shard-by date\ncandidate columns:\n%s", describeTimeColumns(t))
}
c, ok := t.Column(column)
if !ok {
return nil, fmt.Errorf("shard column %q does not exist; candidate columns:\n%s", column, describeTimeColumns(t))
}
switch c.DataType {
case "date":
return &ShardSpec{Column: c.Name, Kind: ShardDate, Unit: 1, Type: c.DataType}, nil
case "datetime", "timestamp":
return &ShardSpec{Column: c.Name, Kind: ShardDateTime, Unit: 1, Type: c.DataType}, nil
}
if row.IsInteger(c.DataType) {
unit, err := ParseEpochUnit(unitFlag)
if err != nil {
minV, maxV, ok, serr := SampleEpochRange(ctx, db, t, c.Name, where)
if serr != nil {
return nil, fmt.Errorf("%w (also failed to sample %s: %v)", err, c.Name, serr)
}
if !ok {
return nil, fmt.Errorf("%w; column %s has no non-NULL values to sample", err, c.Name)
}
return nil, fmt.Errorf("%w\nsampled %s range: %d .. %d\n seconds: %s .. %s\n milliseconds: %s .. %s\n microseconds: %s .. %s",
err, c.Name, minV, maxV,
epochString(minV, 1, loc), epochString(maxV, 1, loc),
epochString(minV, 1_000, loc), epochString(maxV, 1_000, loc),
epochString(minV, 1_000_000, loc), epochString(maxV, 1_000_000, loc))
}
return &ShardSpec{Column: c.Name, Kind: ShardEpoch, Unit: unit, Type: c.DataType}, nil
}
return nil, fmt.Errorf("column %q has unsupported type %s; v1 supports DATE, DATETIME, TIMESTAMP and integer epoch columns with --shard-column-type\ncandidate columns:\n%s",
c.Name, c.DataType, describeTimeColumns(t))
}
func describeTimeColumns(t *Table) string {
var b strings.Builder
found := false
for _, c := range t.TimeColumns() {
found = true
fmt.Fprintf(&b, " - %s %s%s\n", c.Name, c.ColumnType, nullableSuffix(c))
}
for _, c := range t.Columns {
if row.IsInteger(c.DataType) {
found = true
fmt.Fprintf(&b, " - %s %s (integer; requires --shard-column-type)%s\n", c.Name, c.ColumnType, nullableSuffix(c))
}
}
if !found {
return " (none)"
}
return strings.TrimRight(b.String(), "\n")
}
func nullableSuffix(c Column) string {
if c.Nullable {
return " NULL"
}
return " NOT NULL"
}
// ParseEpochUnit maps --shard-column-type to raw units per second: 1 for
// seconds, 1000 for milliseconds, 1000000 for microseconds.
func ParseEpochUnit(flag string) (int64, error) {
switch strings.ToLower(strings.TrimSpace(flag)) {
case "epoch-seconds", "seconds", "second", "s", "1":
return 1, nil
case "epoch-milliseconds", "epoch-millis", "milliseconds", "millisecond", "ms", "1000":
return 1_000, nil
case "epoch-microseconds", "epoch-micros", "microseconds", "microsecond", "us", "1000000":
return 1_000_000, nil
case "":
return 0, fmt.Errorf("integer shard column requires --shard-column-type (epoch-seconds, epoch-milliseconds or epoch-microseconds)")
default:
return 0, fmt.Errorf("unsupported --shard-column-type %q; use epoch-seconds, epoch-milliseconds or epoch-microseconds", flag)
}
}
// SampleEpochRange returns MIN/MAX of an integer column under the base where.
func SampleEpochRange(ctx context.Context, db *sql.DB, t *Table, col, where string) (int64, int64, bool, error) {
q := fmt.Sprintf("SELECT MIN(%s), MAX(%s) FROM %s", sqlutil.Ident(col), sqlutil.Ident(col), sqlutil.Qualified(t.DB, t.Name))
if strings.TrimSpace(where) != "" {
q += " WHERE (" + where + ")"
}
var minRaw, maxRaw sql.NullString
if err := db.QueryRowContext(ctx, q).Scan(&minRaw, &maxRaw); err != nil {
return 0, 0, false, err
}
if !minRaw.Valid || !maxRaw.Valid {
return 0, 0, false, nil
}
minV, err := strconv.ParseInt(strings.TrimSpace(minRaw.String), 10, 64)
if err != nil {
return 0, 0, false, fmt.Errorf("parse min %q: %w", minRaw.String, err)
}
maxV, err := strconv.ParseInt(strings.TrimSpace(maxRaw.String), 10, 64)
if err != nil {
return 0, 0, false, fmt.Errorf("parse max %q: %w", maxRaw.String, err)
}
return minV, maxV, true, nil
}
func epochString(v, unit int64, loc *time.Location) string {
return EpochTime(v, unit, loc).Format("2006-01-02 15:04:05")
}
// EpochTime converts an epoch value with the given nanoseconds-per-unit to a
// time in loc.
func EpochTime(v, unit int64, loc *time.Location) time.Time {
sec := v / unit
frac := v % unit
if frac < 0 {
sec--
frac += unit
}
nsec := frac * (1_000_000_000 / unit)
return time.Unix(sec, nsec).In(loc)
}
// TimeToEpoch is the inverse of EpochTime.
func TimeToEpoch(t time.Time, unit int64) int64 {
return t.Unix()*unit + int64(t.Nanosecond())/(1_000_000_000/unit)
}
const nullBucket = "_null"
const zeroBucket = "0000-00-00"
// BucketOf maps a raw shard value to its date bucket. key is used only in
// error messages.
func BucketOf(v row.Value, spec *ShardSpec, loc *time.Location, zeroDate string, key row.Key) (string, error) {
if v.IsNull() {
return nullBucket, nil
}
switch spec.Kind {
case ShardEpoch:
n, err := strconv.ParseInt(strings.TrimSpace(string(v.Raw)), 10, 64)
if err != nil {
return "", fmt.Errorf("row %s: cannot parse epoch value %q in column %s: %w", key, string(v.Raw), spec.Column, err)
}
return EpochTime(n, spec.Unit, loc).Format("2006-01-02"), nil
case ShardDate, ShardDateTime:
s := string(v.Raw)
if len(s) < 10 {
return "", fmt.Errorf("row %s: cannot parse date value %q in column %s", key, s, spec.Column)
}
day := s[:10]
if strings.HasPrefix(day, "0000-") {
if zeroDate == "bucket" {
return zeroBucket, nil
}
return "", fmt.Errorf("row %s has zero date %q in column %s; pass --zero-date bucket to route it to date=0000-00-00", key, s, spec.Column)
}
if _, err := time.Parse("2006-01-02", day); err != nil {
return "", fmt.Errorf("row %s: invalid date %q in column %s", key, s, spec.Column)
}
return day, nil
}
return "", fmt.Errorf("internal: shard kind not resolved")
}
// BucketLess orders buckets in scan order: NULL first, then zero dates, then
// real dates.
func BucketLess(a, b string) bool {
ca, ka := bucketClass(a)
cb, kb := bucketClass(b)
if ca != cb {
return ca < cb
}
return ka < kb
}
func bucketClass(b string) (int, string) {
switch b {
case "":
return -1, ""
case nullBucket:
return 0, ""
case zeroBucket:
return 1, ""
default:
return 2, b
}
}
+111
View File
@@ -0,0 +1,111 @@
package schema
import (
"testing"
"time"
"archer/internal/row"
)
func TestBucketOfDateTime(t *testing.T) {
spec := &ShardSpec{Column: "created_at", Kind: ShardDateTime}
loc := time.UTC
cases := []struct {
raw []byte
want string
}{
{[]byte("2025-06-01 13:45:00"), "2025-06-01"},
{[]byte("2025-12-31 00:00:00"), "2025-12-31"},
}
for _, c := range cases {
got, err := BucketOf(row.Value{Raw: c.raw}, spec, loc, "", row.KeyOf("1"))
if err != nil || got != c.want {
t.Fatalf("%s: got %q err %v want %q", c.raw, got, err, c.want)
}
}
}
func TestBucketOfNull(t *testing.T) {
spec := &ShardSpec{Column: "c", Kind: ShardDateTime}
got, err := BucketOf(row.Value{}, spec, time.UTC, "", row.KeyOf("1"))
if err != nil || got != "_null" {
t.Fatalf("got %q err %v", got, err)
}
}
func TestBucketOfZeroDate(t *testing.T) {
spec := &ShardSpec{Column: "c", Kind: ShardDateTime}
if _, err := BucketOf(row.Value{Raw: []byte("0000-00-00 00:00:00")}, spec, time.UTC, "", row.KeyOf("7")); err == nil {
t.Fatal("expected zero-date error")
}
got, err := BucketOf(row.Value{Raw: []byte("0000-00-00 00:00:00")}, spec, time.UTC, "bucket", row.KeyOf("7"))
if err != nil || got != "0000-00-00" {
t.Fatalf("got %q err %v", got, err)
}
}
func TestBucketOfEpoch(t *testing.T) {
loc, err := time.LoadLocation("UTC")
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
unit int64
raw string
want string
}{
{1, "1717200000", "2024-06-01"},
{1_000, "1717200000000", "2024-06-01"},
{1_000_000, "1717200000000000", "2024-06-01"},
} {
spec := &ShardSpec{Column: "ts", Kind: ShardEpoch, Unit: tc.unit}
got, err := BucketOf(row.Value{Raw: []byte(tc.raw)}, spec, loc, "", row.KeyOf("1"))
if err != nil || got != tc.want {
t.Fatalf("unit %d: got %q err %v want %q", tc.unit, got, err, tc.want)
}
}
}
func TestBucketLess(t *testing.T) {
order := []string{"_null", "0000-00-00", "2024-01-01", "2024-01-02", "2025-01-01"}
for i := 0; i+1 < len(order); i++ {
if !BucketLess(order[i], order[i+1]) {
t.Fatalf("expected %q < %q", order[i], order[i+1])
}
if BucketLess(order[i+1], order[i]) {
t.Fatalf("expected %q not < %q", order[i+1], order[i])
}
}
}
func TestParseEpochUnit(t *testing.T) {
cases := map[string]int64{
"epoch-seconds": 1,
"seconds": 1,
"epoch-milliseconds": 1_000,
"ms": 1_000,
"epoch-microseconds": 1_000_000,
"us": 1_000_000,
}
for in, want := range cases {
got, err := ParseEpochUnit(in)
if err != nil || got != want {
t.Fatalf("%s: got %d err %v want %d", in, got, err, want)
}
}
if _, err := ParseEpochUnit(""); err == nil {
t.Fatal("empty unit should error")
}
}
func TestEpochRoundTrip(t *testing.T) {
loc := time.UTC
for _, unit := range []int64{1, 1_000, 1_000_000} {
now := time.Date(2024, 6, 1, 12, 30, 15, 0, loc)
v := TimeToEpoch(now, unit)
back := EpochTime(v, unit, loc)
if !back.Equal(now) {
t.Fatalf("unit %d: got %s want %s", unit, back, now)
}
}
}
+263
View File
@@ -0,0 +1,263 @@
// Package schema loads MySQL table metadata and answers questions about
// columns and indexes needed to choose a scan strategy.
package schema
import (
"context"
"database/sql"
"fmt"
"strings"
"archer/internal/row"
)
// Column is one column of the source table.
type Column struct {
Name string
Ordinal int
DataType string
ColumnType string
Nullable bool
Extra string
Key string
}
func (c Column) IsAutoIncrement() bool {
return strings.Contains(strings.ToLower(c.Extra), "auto_increment")
}
// Index is one index of the source table, in index-column order.
type Index struct {
Name string
Unique bool
Columns []string
}
// Table is the loaded metadata of the source table.
type Table struct {
DB string
Name string
Columns []Column
byName map[string]Column
Indexes []Index
}
func (t *Table) Column(name string) (Column, bool) {
c, ok := t.byName[strings.ToLower(name)]
return c, ok
}
func (t *Table) ColumnNames() []string {
out := make([]string, len(t.Columns))
for i, c := range t.Columns {
out[i] = c.Name
}
return out
}
// TimeColumns returns every DATE/DATETIME/TIMESTAMP/IPv4-ish candidate used to
// explain to the user which shard columns are available.
func (t *Table) TimeColumns() []Column {
var out []Column
for _, c := range t.Columns {
if c.DataType == "date" || c.DataType == "datetime" || c.DataType == "timestamp" {
out = append(out, c)
}
}
return out
}
// Load reads column and index metadata from information_schema.
func Load(ctx context.Context, db *sql.DB, dbName, table string) (*Table, error) {
t := &Table{DB: dbName, Name: table, byName: map[string]Column{}}
rows, err := db.QueryContext(ctx, `
SELECT COLUMN_NAME, ORDINAL_POSITION, LOWER(DATA_TYPE), COLUMN_TYPE, IS_NULLABLE, EXTRA, COLUMN_KEY
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
ORDER BY ORDINAL_POSITION`, dbName, table)
if err != nil {
return nil, fmt.Errorf("load columns: %w", err)
}
defer rows.Close()
for rows.Next() {
var c Column
var nullable string
if err := rows.Scan(&c.Name, &c.Ordinal, &c.DataType, &c.ColumnType, &nullable, &c.Extra, &c.Key); err != nil {
return nil, fmt.Errorf("scan column: %w", err)
}
c.Nullable = strings.EqualFold(nullable, "YES")
t.Columns = append(t.Columns, c)
t.byName[strings.ToLower(c.Name)] = c
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("load columns: %w", err)
}
if len(t.Columns) == 0 {
return nil, fmt.Errorf("table %s.%s not found or has no columns", dbName, table)
}
idxRows, err := db.QueryContext(ctx, `
SELECT INDEX_NAME, NON_UNIQUE, SEQ_IN_INDEX, COLUMN_NAME
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
ORDER BY INDEX_NAME, SEQ_IN_INDEX`, dbName, table)
if err != nil {
return nil, fmt.Errorf("load indexes: %w", err)
}
defer idxRows.Close()
byIdx := map[string]*Index{}
for idxRows.Next() {
var name, col string
var nonUnique, seq int
if err := idxRows.Scan(&name, &nonUnique, &seq, &col); err != nil {
return nil, fmt.Errorf("scan index: %w", err)
}
if col == "" {
continue
}
ix := byIdx[name]
if ix == nil {
ix = &Index{Name: name, Unique: nonUnique == 0}
byIdx[name] = ix
t.Indexes = append(t.Indexes, *ix)
}
ix.Columns = append(ix.Columns, col)
}
if err := idxRows.Err(); err != nil {
return nil, fmt.Errorf("load indexes: %w", err)
}
// Rebuild pointers into the slice since append may have copied structs.
for i := range t.Indexes {
t.Indexes[i] = *byIdx[t.Indexes[i].Name]
}
return t, nil
}
// KeyInfo describes the unique index chosen as scan key.
type KeyInfo struct {
Name string
Columns []string
IsPrimary bool
}
func (t *Table) PrimaryKey() *Index {
for i := range t.Indexes {
if t.Indexes[i].Name == "PRIMARY" {
return &t.Indexes[i]
}
}
return nil
}
// ChooseKey selects the scan key: an explicitly requested index, else the
// primary key, else the first unique index. Every key column must be NOT NULL.
func (t *Table) ChooseKey(forced string) (KeyInfo, error) {
var chosen *Index
switch {
case forced != "":
for i := range t.Indexes {
if strings.EqualFold(t.Indexes[i].Name, forced) {
chosen = &t.Indexes[i]
break
}
}
if chosen == nil {
return KeyInfo{}, fmt.Errorf("index %q does not exist on %s", forced, t.Name)
}
if !chosen.Unique {
return KeyInfo{}, fmt.Errorf("index %q is not unique; a unique index is required for seek pagination", forced)
}
case t.PrimaryKey() != nil:
chosen = t.PrimaryKey()
default:
for i := range t.Indexes {
if t.Indexes[i].Unique {
chosen = &t.Indexes[i]
break
}
}
if chosen == nil {
return KeyInfo{}, fmt.Errorf("table %s has no primary key or unique index; create one or pass --index", t.Name)
}
}
for _, col := range chosen.Columns {
c, ok := t.Column(col)
if !ok {
return KeyInfo{}, fmt.Errorf("index %q references unknown column %q", chosen.Name, col)
}
if c.Nullable {
return KeyInfo{}, fmt.Errorf("index %q column %q is nullable; seek pagination needs a NOT NULL unique key", chosen.Name, col)
}
}
return KeyInfo{Name: chosen.Name, Columns: chosen.Columns, IsPrimary: chosen.Name == "PRIMARY"}, nil
}
// FindIndexExact returns the index whose columns are exactly cols.
func (t *Table) FindIndexExact(cols []string) *Index {
for i := range t.Indexes {
if equalFoldSlice(t.Indexes[i].Columns, cols) {
return &t.Indexes[i]
}
}
return nil
}
// FindIndexPrefix returns the first index whose leading columns are exactly
// prefix, and its full column list.
func (t *Table) FindIndexPrefix(prefix []string) *Index {
for i := range t.Indexes {
ix := &t.Indexes[i]
if len(ix.Columns) < len(prefix) {
continue
}
if equalFoldSlice(ix.Columns[:len(prefix)], prefix) {
return ix
}
}
return nil
}
// HasColumnInIndex reports whether col is the first column of some index.
func (t *Table) HasColumnPrefixIndex(col string) *Index {
return t.FindIndexPrefix([]string{col})
}
func equalFoldSlice(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if !strings.EqualFold(a[i], b[i]) {
return false
}
}
return true
}
// OutputColumns resolves the requested column list (empty means all) and
// validates names against the table.
func (t *Table) OutputColumns(requested []string) ([]row.Column, error) {
if len(requested) == 0 {
out := make([]row.Column, len(t.Columns))
for i, c := range t.Columns {
out[i] = row.Column{Name: c.Name, Type: c.DataType}
}
return out, nil
}
out := make([]row.Column, 0, len(requested))
seen := map[string]bool{}
for _, name := range requested {
c, ok := t.Column(name)
if !ok {
return nil, fmt.Errorf("unknown column %q", name)
}
if seen[c.Name] {
continue
}
seen[c.Name] = true
out = append(out, row.Column{Name: c.Name, Type: c.DataType})
}
return out, nil
}
+132
View File
@@ -0,0 +1,132 @@
package shard
import (
"compress/gzip"
"fmt"
"io"
"github.com/klauspost/compress/zstd"
"github.com/ulikunitz/xz"
)
// DefaultLevel is the compression level used when none is specified. It
// matches the byte output of archer before --compress-level existed.
const DefaultLevel = 3
// xzReadDictCap is the largest dictionary xz can produce here (level 9).
// Readers always allow it so lowering the level with --force-resume does not
// make previously written shards unreadable.
const xzReadDictCap = 64 << 20
// Compressor wraps shard byte streams. Every shard is compressed on its own
// so a single file can be decompressed and imported without its siblings.
// Level is a format-neutral 1..9 scale where 9 is the strongest setting.
type Compressor struct {
Name string // "none", "gzip", "zstd", "xz"
Ext string // "", ".gz", ".zst", ".xz"
Level int
}
func NewCompressor(name string, level int) (*Compressor, error) {
if level < 1 || level > 9 {
return nil, fmt.Errorf("compression level %d out of range 1-9", level)
}
switch name {
case "", "none":
return &Compressor{Name: "none", Level: level}, nil
case "gzip":
return &Compressor{Name: "gzip", Ext: ".gz", Level: level}, nil
case "zstd":
return &Compressor{Name: "zstd", Ext: ".zst", Level: level}, nil
case "xz":
return &Compressor{Name: "xz", Ext: ".xz", Level: level}, nil
default:
return nil, fmt.Errorf("unsupported compression %q", name)
}
}
func (c *Compressor) Wrap(w io.Writer) (io.WriteCloser, error) {
switch c.Name {
case "gzip":
return gzip.NewWriterLevel(w, gzipLevel(c.Level))
case "zstd":
return zstd.NewWriter(w, zstd.WithEncoderLevel(zstdLevel(c.Level)))
case "xz":
xw, err := xz.WriterConfig{DictCap: xzDictCap(c.Level)}.NewWriter(w)
if err != nil {
return nil, err
}
return xw, nil
default:
return nopCloser{w}, nil
}
}
// Reader returns a decompressing reader for an existing shard.
func (c *Compressor) Reader(r io.Reader) (io.ReadCloser, error) {
switch c.Name {
case "gzip":
return gzip.NewReader(r)
case "zstd":
zr, err := zstd.NewReader(r)
if err != nil {
return nil, err
}
return zr.IOReadCloser(), nil
case "xz":
xr, err := xz.ReaderConfig{DictCap: xzReadDictCap}.NewReader(r)
if err != nil {
return nil, err
}
return io.NopCloser(xr), nil
default:
return io.NopCloser(r), nil
}
}
// gzipLevel passes the neutral level straight through.
func gzipLevel(level int) int {
return level
}
func zstdLevel(level int) zstd.EncoderLevel {
switch {
case level <= 2:
return zstd.SpeedFastest
case level <= 5:
return zstd.SpeedDefault
case level <= 8:
return zstd.SpeedBetterCompression
default:
return zstd.SpeedBestCompression
}
}
// xzDictCap maps the neutral level onto xz dictionary sizes; the library has
// no preset parameter, so the dictionary is the only strength knob.
func xzDictCap(level int) int {
switch level {
case 1:
return 1 << 20
case 2:
return 2 << 20
case 3, 4:
return 4 << 20
case 5, 6:
return 8 << 20
case 7:
return 16 << 20
case 8:
return 32 << 20
default:
return 64 << 20
}
}
type nopCloser struct{ io.Writer }
func (nopCloser) Close() error { return nil }
type flushCloser interface {
Flush() error
}
+279
View File
@@ -0,0 +1,279 @@
package shard
import (
"bufio"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
)
// Entry is one line of manifest.jsonl.
type Entry struct {
Seq int64 `json:"seq"`
Path string `json:"path"`
Date string `json:"date,omitempty"`
PKLo string `json:"pk_lo"`
PKHi string `json:"pk_hi"`
Rows int64 `json:"rows"`
Bytes int64 `json:"bytes"`
SHA256 string `json:"sha256"`
ClosedAt string `json:"closed_at"`
}
// Manifest appends sealed-shard records to manifest.jsonl and knows which
// shards already exist.
type Manifest struct {
path string
f *os.File
seen map[string]bool
entries []Entry
}
const manifestName = "manifest.jsonl"
func OpenManifest(root string) (*Manifest, error) {
path := filepath.Join(root, manifestName)
m := &Manifest{path: path, seen: map[string]bool{}}
if err := m.readExisting(); err != nil {
return nil, err
}
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
if err != nil {
return nil, fmt.Errorf("open manifest: %w", err)
}
m.f = f
return m, nil
}
func (m *Manifest) readExisting() error {
f, err := os.Open(m.path)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return fmt.Errorf("read manifest: %w", err)
}
defer f.Close()
sc := bufio.NewScanner(f)
sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
for sc.Scan() {
line := strings.TrimSpace(sc.Text())
if line == "" {
continue
}
var e Entry
if err := json.Unmarshal([]byte(line), &e); err != nil {
continue
}
m.entries = append(m.entries, e)
m.seen[e.Path] = true
}
return sc.Err()
}
func (m *Manifest) Has(rel string) bool { return m.seen[rel] }
func (m *Manifest) Entries() []Entry { return m.entries }
// Append writes one record and fsyncs it. A sealed shard is only considered
// complete once its manifest line is durable.
func (m *Manifest) Append(e Entry) error {
if e.ClosedAt == "" {
e.ClosedAt = time.Now().UTC().Format(time.RFC3339)
}
b, err := json.Marshal(e)
if err != nil {
return err
}
b = append(b, '\n')
if _, err := m.f.Write(b); err != nil {
return fmt.Errorf("append manifest: %w", err)
}
if err := m.f.Sync(); err != nil {
return fmt.Errorf("sync manifest: %w", err)
}
m.seen[e.Path] = true
m.entries = append(m.entries, e)
return nil
}
func (m *Manifest) Close() error {
if m.f == nil {
return nil
}
return m.f.Close()
}
var shardNameRE = regexp.MustCompile(`^(\d+)\..*?\.([A-Za-z0-9]+?)(\.gz|\.zst|\.xz)?$`)
// Adopt scans the output tree for sealed shard files that have no manifest
// record (a crash after rename but before the manifest append) and records
// them, so resume never overwrites durable data. It returns the maximum seq
// seen per bucket.
func (m *Manifest) Adopt(root, format string, comp *Compressor) (map[string]int64, error) {
maxSeq := map[string]int64{}
root = filepath.Clean(root)
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
name := d.Name()
if name == manifestName || name == "state.json" || name == "state.json.tmp" ||
strings.HasSuffix(name, ".partial") || strings.HasSuffix(name, ".partial.meta") {
return nil
}
mt := shardNameRE.FindStringSubmatch(name)
if mt == nil {
return nil
}
if mt[2] != format || mt[3] != comp.Ext {
return nil
}
rel, err := filepath.Rel(root, path)
if err != nil {
return err
}
rel = filepath.ToSlash(rel)
bucket := bucketOfRel(rel)
var seq int64
fmt.Sscanf(mt[1], "%d", &seq)
if seq > maxSeq[bucket] {
maxSeq[bucket] = seq
}
if m.seen[rel] {
return nil
}
rows, err := countRows(path, comp, format)
if err != nil {
return err
}
sum, size, err := hashFile(path)
if err != nil {
return err
}
entry := Entry{
Seq: seq,
Path: rel,
Date: bucket,
Rows: rows,
Bytes: size,
SHA256: sum,
}
return m.Append(entry)
})
if err != nil {
return nil, fmt.Errorf("adopt sealed shards: %w", err)
}
return maxSeq, nil
}
func bucketOfRel(rel string) string {
dir := filepath.ToSlash(filepath.Dir(rel))
if dir == "." {
return ""
}
parts := strings.Split(dir, "/")
for _, p := range parts {
if strings.HasPrefix(p, "date=") {
return strings.TrimPrefix(p, "date=")
}
}
return ""
}
// MaxSeqByBucket returns the highest seq recorded per bucket.
func (m *Manifest) MaxSeqByBucket() map[string]int64 {
out := map[string]int64{}
for _, e := range m.entries {
if e.Seq > out[e.Date] {
out[e.Date] = e.Seq
}
}
return out
}
func hashFile(path string) (string, int64, error) {
f, err := os.Open(path)
if err != nil {
return "", 0, err
}
defer f.Close()
h := sha256.New()
n, err := io.Copy(h, f)
if err != nil {
return "", 0, err
}
return hex.EncodeToString(h.Sum(nil)), n, nil
}
func countRows(path string, comp *Compressor, format string) (int64, error) {
f, err := os.Open(path)
if err != nil {
return 0, err
}
defer f.Close()
r, err := comp.Reader(f)
if err != nil {
return 0, err
}
defer r.Close()
br := bufio.NewReaderSize(r, 256*1024)
var n int64
for {
_, err := br.ReadBytes('\n')
if err == io.EOF {
break
}
if err != nil {
return 0, err
}
n++
}
if format == "csv" && n > 0 {
n--
}
return n, nil
}
// CleanupPartials removes leftover .partial files and their metadata, keeping
// keepPartial (if non-empty) and its sidecar.
func CleanupPartials(root, keepPartial string) error {
var paths []string
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
if keepPartial != "" && (path == keepPartial || path == keepPartial+".meta") {
return nil
}
name := d.Name()
if strings.HasSuffix(name, ".partial") || strings.HasSuffix(name, ".partial.meta") {
paths = append(paths, path)
}
return nil
})
if err != nil {
return err
}
sort.Strings(paths)
for _, p := range paths {
if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
return err
}
}
return nil
}
+63
View File
@@ -0,0 +1,63 @@
package shard
import (
"fmt"
"path/filepath"
"strings"
"archer/internal/row"
)
// FinalName builds the sealed shard filename:
// <seq:06d>.<lo>-<hi>.<format>[.<comp>]
func FinalName(seq int64, lo, hi, format, compExt string) string {
return fmt.Sprintf("%06d.%s-%s.%s%s", seq, safeNamePart(lo), safeNamePart(hi), format, compExt)
}
// KeyFilePart renders a key tuple into a filename-safe fragment.
func KeyFilePart(k row.Key) string {
if len(k) == 0 {
return "NULL"
}
parts := make([]string, len(k))
for i, p := range k {
if p.Null {
parts[i] = "NULL"
} else {
parts[i] = p.Val
}
}
return strings.Join(parts, ",")
}
func safeNamePart(s string) string {
if s == "" {
return "_"
}
var b strings.Builder
for _, r := range s {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
b.WriteRune(r)
case r == '.', r == '_', r == '-':
b.WriteRune(r)
default:
b.WriteByte('_')
}
if b.Len() >= 96 {
break
}
}
if b.Len() == 0 {
return "_"
}
return b.String()
}
// BucketDir returns the directory that holds shards for a bucket.
func BucketDir(root, bucket string) string {
if bucket == "" {
return root
}
return filepath.Join(root, "date="+safeNamePart(bucket))
}
+90
View File
@@ -0,0 +1,90 @@
package shard
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"archer/internal/atomicfile"
"archer/internal/row"
)
// PartialMeta describes the durable prefix of an in-progress shard. It is
// written before each chunk is committed, so a resume can truncate the file
// back to a consistent offset and continue appending.
type PartialMeta struct {
Seq int64 `json:"seq"`
Bucket string `json:"bucket,omitempty"`
Rows int64 `json:"rows"`
Bytes int64 `json:"bytes"`
FirstKey row.Key `json:"first_key"`
LastKey row.Key `json:"last_key"`
}
func partialPath(dir string, seq int64) string {
return filepath.Join(dir, fmt.Sprintf("%06d.partial", seq))
}
func partialMetaPath(dir string, seq int64) string {
return partialPath(dir, seq) + ".meta"
}
// PartialPath exposes the partial path for tests and diagnostics.
func PartialPath(root, bucket string, seq int64) string {
return partialPath(BucketDir(root, bucket), seq)
}
// LoadPartialMeta reads the sidecar for an in-progress shard. A missing file
// returns (nil, nil).
func LoadPartialMeta(root, bucket string, seq int64) (*PartialMeta, error) {
path := partialMetaPath(BucketDir(root, bucket), seq)
b, err := os.ReadFile(path)
if os.IsNotExist(err) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("read partial meta: %w", err)
}
var m PartialMeta
if err := json.Unmarshal(b, &m); err != nil {
return nil, fmt.Errorf("parse partial meta: %w", err)
}
return &m, nil
}
func savePartialMeta(dir string, m *PartialMeta) error {
path := partialMetaPath(dir, m.Seq)
b, err := json.Marshal(m)
if err != nil {
return err
}
tmp := path + ".tmp"
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
if err != nil {
return err
}
if _, err := f.Write(b); err != nil {
f.Close()
os.Remove(tmp)
return err
}
if err := f.Sync(); err != nil {
f.Close()
os.Remove(tmp)
return err
}
if err := f.Close(); err != nil {
os.Remove(tmp)
return err
}
if err := atomicfile.Replace(tmp, path); err != nil {
os.Remove(tmp)
return err
}
return nil
}
func removePartialMeta(dir string, seq int64) {
os.Remove(partialMetaPath(dir, seq))
}
+426
View File
@@ -0,0 +1,426 @@
// Package shard writes row streams into ordered, independently importable
// shard files and records them in manifest.jsonl.
package shard
import (
"bufio"
"fmt"
"io"
"os"
"path/filepath"
"archer/internal/atomicfile"
"archer/internal/row"
"archer/internal/schema"
)
// Options configures a Writer.
type Options struct {
Dir string
Mode string // none|count|date
ShardRows int64
Format string
Compressor *Compressor
Columns []row.Column
OutIdx []int
CSVNull string
StartBucket string
StartSeq int64
}
// Writer owns the single active shard file.
type Writer struct {
opt Options
man *Manifest
seq int64
curBucket string
cur *shardFile
totalRows int64
shards int64
bytes int64
}
type shardFile struct {
bucket string
seq int64
dir string
partial string
file *os.File
cw *countWriter
compW io.WriteCloser
bw *bufio.Writer
enc row.Encoder
buf []byte
rows int64
memberRows int64
firstKey row.Key
lastKey row.Key
}
type countWriter struct {
w io.Writer
n int64
}
func (c *countWriter) Write(p []byte) (int, error) {
n, err := c.w.Write(p)
c.n += int64(n)
return n, err
}
// Open creates a writer. It does not open a shard until the first row arrives.
func Open(opt Options, man *Manifest) (*Writer, error) {
w := &Writer{opt: opt, man: man, seq: opt.StartSeq, curBucket: opt.StartBucket}
return w, nil
}
// Resume reopens an existing partial shard and continues appending to it.
func (w *Writer) Resume(m *PartialMeta) error {
dir := BucketDir(w.opt.Dir, m.Bucket)
path := partialPath(dir, m.Seq)
f, err := os.OpenFile(path, os.O_RDWR, 0o644)
if err != nil {
return fmt.Errorf("resume partial shard: %w", err)
}
st, err := f.Stat()
if err != nil {
f.Close()
return err
}
if st.Size() < m.Bytes {
f.Close()
return fmt.Errorf("partial shard %s is shorter (%d bytes) than its metadata (%d bytes); refusing to resume", path, st.Size(), m.Bytes)
}
if st.Size() > m.Bytes {
if err := f.Truncate(m.Bytes); err != nil {
f.Close()
return fmt.Errorf("truncate partial shard: %w", err)
}
}
if _, err := f.Seek(m.Bytes, io.SeekStart); err != nil {
f.Close()
return err
}
enc, err := row.NewEncoder(w.opt.Format, w.opt.Columns, w.opt.OutIdx, row.Options{CSVNull: w.opt.CSVNull})
if err != nil {
f.Close()
return err
}
sf := &shardFile{
bucket: m.Bucket,
seq: m.Seq,
dir: dir,
partial: path,
file: f,
enc: enc,
rows: m.Rows,
memberRows: m.Rows,
firstKey: m.FirstKey,
lastKey: m.LastKey,
}
if err := sf.startChain(m.Bytes); err != nil {
f.Close()
return err
}
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
f.Close()
return err
}
w.cur = sf
w.seq = m.Seq
w.curBucket = m.Bucket
w.totalRows = m.Rows
return nil
}
func (sf *shardFile) startChain(offset int64) error {
sf.cw = &countWriter{w: sf.file, n: offset}
return nil
}
func (sf *shardFile) openChain(comp *Compressor, bufSize int) error {
cw, err := comp.Wrap(sf.cw)
if err != nil {
return err
}
sf.compW = cw
sf.bw = bufio.NewWriterSize(cw, bufSize)
sf.memberRows = sf.rows
return nil
}
func (w *Writer) openShard(bucket string) error {
if w.cur != nil {
return fmt.Errorf("internal: shard already open")
}
if bucket != w.curBucket {
w.seq = 0
w.curBucket = bucket
}
w.seq++
dir := BucketDir(w.opt.Dir, bucket)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("create shard directory: %w", err)
}
path := partialPath(dir, w.seq)
f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
if err != nil {
return fmt.Errorf("create partial shard: %w", err)
}
enc, err := row.NewEncoder(w.opt.Format, w.opt.Columns, w.opt.OutIdx, row.Options{CSVNull: w.opt.CSVNull})
if err != nil {
f.Close()
return err
}
sf := &shardFile{bucket: bucket, seq: w.seq, dir: dir, partial: path, file: f, enc: enc}
if err := sf.startChain(0); err != nil {
f.Close()
return err
}
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
f.Close()
return err
}
if hdr := enc.Header(); len(hdr) > 0 {
if _, err := sf.bw.Write(hdr); err != nil {
f.Close()
return err
}
}
w.cur = sf
return nil
}
// WriteChunk routes chunk rows into the active shard, sealing on bucket change
// or when --shard-rows is reached. A chunk may span multiple shards.
func (w *Writer) WriteChunk(c *row.Chunk) error {
for i := range c.Rows {
bucket := ""
if w.opt.Mode == "date" {
bucket = c.Buckets[i]
}
if w.cur != nil {
needSeal := false
if bucket != w.cur.bucket {
if !schema.BucketLess(w.cur.bucket, bucket) {
return fmt.Errorf("rows arrived out of shard order: %q after %q", bucket, w.cur.bucket)
}
needSeal = true
} else if w.opt.ShardRows > 0 && w.cur.rows >= w.opt.ShardRows {
needSeal = true
}
if needSeal {
if err := w.seal(); err != nil {
return err
}
}
}
if w.cur == nil {
if err := w.openShard(bucket); err != nil {
return err
}
}
if err := w.cur.writeRow(c.Rows[i]); err != nil {
return err
}
w.cur.trackKey(c.Keys[i])
w.totalRows++
}
return nil
}
func (sf *shardFile) writeRow(r row.Row) error {
var err error
sf.buf, err = sf.enc.Encode(sf.buf[:0], r)
if err != nil {
return err
}
if _, err := sf.bw.Write(sf.buf); err != nil {
return fmt.Errorf("write shard row: %w", err)
}
sf.rows++
return nil
}
func (sf *shardFile) trackKey(k row.Key) {
if sf.firstKey.IsZero() {
sf.firstKey = k
}
sf.lastKey = k
}
// Checkpoint makes the current shard bytes and their metadata durable. It is
// called before destructive database work when --purge is enabled.
func (w *Writer) Checkpoint() error {
if w.cur == nil {
return nil
}
sf := w.cur
if sf.rows > sf.memberRows {
if err := sf.closeMember(); err != nil {
return err
}
}
meta := &PartialMeta{
Seq: sf.seq,
Bucket: sf.bucket,
Rows: sf.rows,
Bytes: sf.cw.n,
FirstKey: sf.firstKey,
LastKey: sf.lastKey,
}
if err := savePartialMeta(sf.dir, meta); err != nil {
return fmt.Errorf("write partial metadata: %w", err)
}
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
return err
}
return nil
}
func (sf *shardFile) closeMember() error {
if err := sf.bw.Flush(); err != nil {
return fmt.Errorf("flush shard: %w", err)
}
if err := sf.compW.Close(); err != nil {
return fmt.Errorf("close shard compressor: %w", err)
}
if err := sf.file.Sync(); err != nil {
return fmt.Errorf("sync shard: %w", err)
}
return nil
}
// Flush pushes buffered rows to the OS without sealing the shard.
func (w *Writer) Flush() error {
if w.cur == nil || w.cur.bw == nil {
return nil
}
return w.cur.bw.Flush()
}
// SealCurrent closes and publishes the active shard even if it is not full.
func (w *Writer) SealCurrent() error {
if w.cur == nil {
return nil
}
if w.cur.rows == 0 {
return w.discardCurrent()
}
return w.seal()
}
func (w *Writer) discardCurrent() error {
sf := w.cur
sf.file.Close()
os.Remove(sf.partial)
removePartialMeta(sf.dir, sf.seq)
w.cur = nil
return nil
}
func (w *Writer) seal() error {
sf := w.cur
if sf.rows == 0 {
return w.discardCurrent()
}
if sf.rows > sf.memberRows || sf.compW != nil {
if err := sf.closeMember(); err != nil {
return err
}
}
if err := sf.file.Close(); err != nil {
return fmt.Errorf("close partial shard: %w", err)
}
lo := KeyFilePart(sf.firstKey)
hi := KeyFilePart(sf.lastKey)
name := FinalName(sf.seq, lo, hi, w.opt.Format, w.opt.Compressor.Ext)
finalPath := filepath.Join(sf.dir, name)
if err := os.Remove(finalPath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("remove stale shard: %w", err)
}
if err := atomicfile.Replace(sf.partial, finalPath); err != nil {
return fmt.Errorf("publish shard: %w", err)
}
sum, size, err := hashFile(finalPath)
if err != nil {
return fmt.Errorf("hash shard: %w", err)
}
rel, err := filepath.Rel(w.opt.Dir, finalPath)
if err != nil {
return err
}
entry := Entry{
Seq: sf.seq,
Path: filepath.ToSlash(rel),
Date: sf.bucket,
PKLo: lo,
PKHi: hi,
Rows: sf.rows,
Bytes: size,
SHA256: sum,
}
if err := w.man.Append(entry); err != nil {
return err
}
removePartialMeta(sf.dir, sf.seq)
w.shards++
w.bytes += size
w.cur = nil
return nil
}
// Close publishes any in-progress shard and closes the manifest.
func (w *Writer) Close() error {
if err := w.SealCurrent(); err != nil {
return err
}
return w.man.Close()
}
// StatePosition reports the bucket and the last sealed seq within that bucket,
// which is what a checkpoint must record for resume.
func (w *Writer) StatePosition() (bucket string, sealedSeq int64) {
if w.cur != nil {
return w.cur.bucket, w.cur.seq - 1
}
return w.curBucket, w.seq
}
// Abandon closes file handles without sealing the active shard, leaving the
// partial and its metadata for a later resume.
func (w *Writer) Abandon() error {
if w.cur != nil {
if w.cur.bw != nil {
_ = w.cur.bw.Flush()
}
_ = w.cur.file.Close()
w.cur = nil
}
return w.man.Close()
}
// Stats returns rows written, shards sealed and bytes sealed.
func (w *Writer) Stats() (rows, shards, bytes int64) {
return w.totalRows, w.shards, w.bytes
}
// CurrentShard returns the sequence number of the active shard, or the last
// one when no shard is currently open.
func (w *Writer) CurrentShard() int64 {
if w.cur != nil {
return w.cur.seq
}
return w.seq
}
// WrittenBytes returns sealed bytes plus the bytes already written to the
// active partial shard.
func (w *Writer) WrittenBytes() int64 {
n := w.bytes
if w.cur != nil && w.cur.cw != nil {
n += w.cur.cw.n
}
return n
}
+459
View File
@@ -0,0 +1,459 @@
package shard
import (
"bytes"
"io"
"os"
"path/filepath"
"strings"
"testing"
"github.com/klauspost/compress/zstd"
"archer/internal/row"
)
func testOpts(dir, mode string) Options {
return Options{
Dir: dir,
Mode: mode,
Format: "dump",
Compressor: &Compressor{Name: "none"},
Columns: []row.Column{{Name: "id", Type: "int"}, {Name: "v", Type: "varchar"}},
}
}
func chunk(start int, values ...string) *row.Chunk {
c := &row.Chunk{}
for i, v := range values {
id := start + i
c.Rows = append(c.Rows, row.Row{Values: []row.Value{
{Raw: []byte(itoa(id))},
{Raw: []byte(v)},
}})
c.Keys = append(c.Keys, row.KeyOf(itoa(id)))
}
if len(c.Keys) > 0 {
c.FirstKey = c.Keys[0]
c.LastKey = c.Keys[len(c.Keys)-1]
}
return c
}
func itoa(i int) string {
if i == 0 {
return "0"
}
var b [20]byte
p := len(b)
neg := i < 0
if neg {
i = -i
}
for i > 0 {
p--
b[p] = byte('0' + i%10)
i /= 10
}
if neg {
p--
b[p] = '-'
}
return string(b[p:])
}
func TestCountShardsOrderAndNames(t *testing.T) {
dir := t.TempDir()
man, err := OpenManifest(dir)
if err != nil {
t.Fatal(err)
}
opt := testOpts(dir, "count")
opt.ShardRows = 2
w, err := Open(opt, man)
if err != nil {
t.Fatal(err)
}
// A chunk of 3 rows spans two shards.
if err := w.WriteChunk(chunk(1, "a", "b", "c")); err != nil {
t.Fatal(err)
}
if err := w.WriteChunk(chunk(4, "d", "e")); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
want := map[string]string{
"000001.1-2.dump": "1\ta\n2\tb\n",
"000002.3-4.dump": "3\tc\n4\td\n",
"000003.5-5.dump": "5\te\n",
}
for name, content := range want {
b, err := os.ReadFile(filepath.Join(dir, name))
if err != nil {
t.Fatalf("%s: %v", name, err)
}
if string(b) != content {
t.Fatalf("%s: got %q want %q", name, b, content)
}
}
entries := man.Entries()
if len(entries) != 3 {
t.Fatalf("manifest entries: %d", len(entries))
}
if entries[0].PKLo != "1" || entries[0].PKHi != "2" || entries[0].Rows != 2 {
t.Fatalf("entry0 %+v", entries[0])
}
if entries[2].Path != "000003.5-5.dump" || entries[2].SHA256 == "" {
t.Fatalf("entry2 %+v", entries[2])
}
}
func TestDateShardsDirectories(t *testing.T) {
dir := t.TempDir()
man, _ := OpenManifest(dir)
opt := testOpts(dir, "date")
w, _ := Open(opt, man)
c := chunk(1, "a", "b", "c")
c.Buckets = []string{"2025-06-01", "2025-06-01", "2025-06-02"}
if err := w.WriteChunk(c); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(dir, "date=2025-06-01", "000001.1-2.dump")); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(dir, "date=2025-06-02", "000001.3-3.dump")); err != nil {
t.Fatal(err)
}
}
func TestDateOrderViolation(t *testing.T) {
dir := t.TempDir()
man, _ := OpenManifest(dir)
w, _ := Open(testOpts(dir, "date"), man)
c := chunk(1, "a", "b")
c.Buckets = []string{"2025-06-02", "2025-06-01"}
if err := w.WriteChunk(c); err == nil {
t.Fatal("expected ordering error")
}
_ = w.Abandon()
}
func TestResumeAppendsAfterTruncatingGarbage(t *testing.T) {
dir := t.TempDir()
man, _ := OpenManifest(dir)
opt := testOpts(dir, "count")
opt.ShardRows = 10
w, _ := Open(opt, man)
if err := w.WriteChunk(chunk(1, "a", "b")); err != nil {
t.Fatal(err)
}
if err := w.Checkpoint(); err != nil {
t.Fatal(err)
}
meta, err := LoadPartialMeta(dir, "", 1)
if err != nil || meta == nil {
t.Fatalf("meta %v %v", meta, err)
}
if err := w.Abandon(); err != nil {
t.Fatal(err)
}
// Simulate bytes from an uncommitted chunk that were flushed but never
// checkpointed: resume must truncate them.
partial := PartialPath(dir, "", 1)
f, err := os.OpenFile(partial, os.O_APPEND|os.O_WRONLY, 0o644)
if err != nil {
t.Fatal(err)
}
f.WriteString("garbage\n")
f.Close()
man2, _ := OpenManifest(dir)
w2, err := Open(opt, man2)
if err != nil {
t.Fatal(err)
}
if err := w2.Resume(meta); err != nil {
t.Fatal(err)
}
if err := w2.WriteChunk(chunk(3, "c", "d")); err != nil {
t.Fatal(err)
}
if err := w2.Checkpoint(); err != nil {
t.Fatal(err)
}
if err := w2.Close(); err != nil {
t.Fatal(err)
}
b, err := os.ReadFile(filepath.Join(dir, "000001.1-4.dump"))
if err != nil {
t.Fatal(err)
}
if string(b) != "1\ta\n2\tb\n3\tc\n4\td\n" {
t.Fatalf("got %q", b)
}
if _, err := os.Stat(partial); !os.IsNotExist(err) {
t.Fatal("partial should be gone after seal")
}
}
func TestManifestAdoptOrphan(t *testing.T) {
dir := t.TempDir()
man, _ := OpenManifest(dir)
orphan := filepath.Join(dir, "000007.9-12.dump")
if err := os.WriteFile(orphan, []byte("9\tx\n10\ty\n"), 0o644); err != nil {
t.Fatal(err)
}
comp := &Compressor{Name: "none"}
maxSeq, err := man.Adopt(dir, "dump", comp)
if err != nil {
t.Fatal(err)
}
if maxSeq[""] != 7 {
t.Fatalf("maxSeq %+v", maxSeq)
}
if len(man.Entries()) != 1 || man.Entries()[0].Rows != 2 {
t.Fatalf("entries %+v", man.Entries())
}
// A second adopt must not duplicate the entry.
if _, err := man.Adopt(dir, "dump", comp); err != nil {
t.Fatal(err)
}
if len(man.Entries()) != 1 {
t.Fatalf("entries %+v", man.Entries())
}
man.Close()
}
func TestSafeNamePart(t *testing.T) {
got := safeNamePart("2024-01-01 00:00:00")
if strings.ContainsAny(got, `:/\*?"<>|`) {
t.Fatalf("unsafe characters kept: %q", got)
}
if got != "2024-01-01_00_00_00" {
t.Fatalf("got %q", got)
}
if safeNamePart("") != "_" {
t.Fatal("empty name part")
}
}
func TestCompressedShardIsIndependent(t *testing.T) {
for _, compName := range []string{"gzip", "zstd", "xz"} {
dir := t.TempDir()
man, _ := OpenManifest(dir)
comp, err := NewCompressor(compName, DefaultLevel)
if err != nil {
t.Fatal(err)
}
opt := testOpts(dir, "none")
opt.Compressor = comp
w, _ := Open(opt, man)
if err := w.WriteChunk(chunk(1, "a", "b")); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
var path string
entries := man.Entries()
if len(entries) != 1 {
t.Fatalf("%s: entries %d", compName, len(entries))
}
path = filepath.Join(dir, entries[0].Path)
f, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
r, err := comp.Reader(f)
if err != nil {
t.Fatal(err)
}
var b strings.Builder
buf := make([]byte, 1024)
for {
n, err := r.Read(buf)
b.Write(buf[:n])
if err != nil {
break
}
}
r.Close()
f.Close()
if b.String() != "1\ta\n2\tb\n" {
t.Fatalf("%s: got %q", compName, b.String())
}
}
}
func TestCompressionLevelMapping(t *testing.T) {
xzDictMiB := []int{1, 2, 4, 4, 8, 8, 16, 32, 64}
for level := 1; level <= 9; level++ {
c, err := NewCompressor("gzip", level)
if err != nil {
t.Fatalf("level %d: %v", level, err)
}
if c.Level != level || c.Name != "gzip" || c.Ext != ".gz" {
t.Fatalf("level %d: %+v", level, c)
}
if got := gzipLevel(level); got != level {
t.Fatalf("gzip level %d -> %d", level, got)
}
var wantZstd zstd.EncoderLevel
switch {
case level <= 2:
wantZstd = zstd.SpeedFastest
case level <= 5:
wantZstd = zstd.SpeedDefault
case level <= 8:
wantZstd = zstd.SpeedBetterCompression
default:
wantZstd = zstd.SpeedBestCompression
}
if got := zstdLevel(level); got != wantZstd {
t.Fatalf("zstd level %d -> %v want %v", level, got, wantZstd)
}
if got, want := xzDictCap(level), xzDictMiB[level-1]<<20; got != want {
t.Fatalf("xz level %d -> %d want %d", level, got, want)
}
}
for _, name := range []string{"none", "gzip", "zstd", "xz"} {
for _, level := range []int{-1, 0, 10} {
if _, err := NewCompressor(name, level); err == nil {
t.Fatalf("%s level %d: expected error", name, level)
}
}
}
if _, err := NewCompressor("brotli", DefaultLevel); err == nil {
t.Fatal("unsupported compressor must fail")
}
for name, ext := range map[string]string{"none": "", "gzip": ".gz", "zstd": ".zst", "xz": ".xz"} {
c, err := NewCompressor(name, DefaultLevel)
if err != nil {
t.Fatalf("%s: %v", name, err)
}
if c.Ext != ext {
t.Fatalf("%s: ext %q want %q", name, c.Ext, ext)
}
}
}
func TestXZCheckpointKeepsAllStreams(t *testing.T) {
dir := t.TempDir()
man, _ := OpenManifest(dir)
defer man.Close()
comp, err := NewCompressor("xz", 9)
if err != nil {
t.Fatal(err)
}
opt := testOpts(dir, "none")
opt.Compressor = comp
w, _ := Open(opt, man)
if err := w.WriteChunk(chunk(1, "a", "b")); err != nil {
t.Fatal(err)
}
if err := w.Checkpoint(); err != nil {
t.Fatal(err)
}
if err := w.WriteChunk(chunk(3, "c", "d")); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
entries := man.Entries()
if len(entries) != 1 {
t.Fatalf("entries %d", len(entries))
}
if entries[0].Rows != 4 {
t.Fatalf("rows %d", entries[0].Rows)
}
if !strings.HasSuffix(entries[0].Path, ".dump.xz") {
t.Fatalf("path %q", entries[0].Path)
}
f, err := os.Open(filepath.Join(dir, entries[0].Path))
if err != nil {
t.Fatal(err)
}
defer f.Close()
r, err := comp.Reader(f)
if err != nil {
t.Fatal(err)
}
defer r.Close()
got, err := io.ReadAll(r)
if err != nil {
t.Fatal(err)
}
if string(got) != "1\ta\n2\tb\n3\tc\n4\td\n" {
t.Fatalf("got %q", got)
}
}
func TestManifestAdoptXZOrphan(t *testing.T) {
dir := t.TempDir()
man, _ := OpenManifest(dir)
defer man.Close()
comp, err := NewCompressor("xz", DefaultLevel)
if err != nil {
t.Fatal(err)
}
var buf bytes.Buffer
xw, err := comp.Wrap(&buf)
if err != nil {
t.Fatal(err)
}
if _, err := xw.Write([]byte("9\tx\n10\ty\n")); err != nil {
t.Fatal(err)
}
if err := xw.Close(); err != nil {
t.Fatal(err)
}
orphan := filepath.Join(dir, "000007.9-12.dump.xz")
if err := os.WriteFile(orphan, buf.Bytes(), 0o644); err != nil {
t.Fatal(err)
}
maxSeq, err := man.Adopt(dir, "dump", comp)
if err != nil {
t.Fatal(err)
}
if maxSeq[""] != 7 {
t.Fatalf("maxSeq %+v", maxSeq)
}
entries := man.Entries()
if len(entries) != 1 || entries[0].Seq != 7 || entries[0].Rows != 2 {
t.Fatalf("entries %+v", entries)
}
if entries[0].Path != "000007.9-12.dump.xz" {
t.Fatalf("path %q", entries[0].Path)
}
// A resumed writer must continue after the adopted seq, not overwrite it.
opt := testOpts(dir, "none")
opt.Compressor = comp
opt.StartSeq = maxSeq[""]
w, _ := Open(opt, man)
if err := w.WriteChunk(chunk(1, "a")); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(dir, "000008.1-1.dump.xz")); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(orphan); err != nil {
t.Fatalf("adopted shard was overwritten: %v", err)
}
}
+132
View File
@@ -0,0 +1,132 @@
package sink
import (
"context"
"database/sql"
"fmt"
"strings"
"archer/internal/row"
"archer/internal/sqlutil"
)
// Execer is satisfied by *sql.DB, *sql.Tx and *sql.Conn.
type Execer interface {
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
}
const defaultMaxPacket = 4 << 20
// DB inserts chunk rows into the destination table in batches that stay under
// max_allowed_packet.
type DB struct {
DBName string
Table string
Columns []string
Mode string // "", "ignore", "replace"
MaxPacket int64
OutIdx []int
}
// NewDB reads max_allowed_packet and returns a sink. cols are the output
// column names; outIdx maps them to positions in a scanned row.
func NewDB(ctx context.Context, db *sql.DB, dbName, table string, cols []string, mode string, outIdx []int) (*DB, error) {
maxPacket := int64(defaultMaxPacket)
var v sql.NullInt64
if err := db.QueryRowContext(ctx, "SELECT @@max_allowed_packet").Scan(&v); err == nil && v.Valid && v.Int64 > 0 {
maxPacket = v.Int64
}
return &DB{
DBName: dbName,
Table: table,
Columns: cols,
Mode: mode,
MaxPacket: maxPacket,
OutIdx: outIdx,
}, nil
}
func (d *DB) table() string { return sqlutil.Qualified(d.DBName, d.Table) }
func (d *DB) verb() string {
switch d.Mode {
case "ignore":
return "INSERT IGNORE INTO"
case "replace":
return "REPLACE INTO"
default:
return "INSERT INTO"
}
}
// InsertChunk writes all chunk rows through exec.
func (d *DB) InsertChunk(ctx context.Context, exec Execer, c *row.Chunk) (int64, error) {
if len(c.Rows) == 0 {
return 0, nil
}
limit := d.MaxPacket * 8 / 10
var total int64
start := 0
for start < len(c.Rows) {
end := start
var size int64
for end < len(c.Rows) {
rowSize := int64(8 + len(d.Columns)*4)
for _, i := range d.OutIdx {
rowSize += int64(len(c.Rows[end].Values[i].Raw))
}
if end > start && size+rowSize > limit {
break
}
size += rowSize
end++
}
if end == start {
end = start + 1
}
n, err := d.insertBatch(ctx, exec, c.Rows[start:end])
if err != nil {
return total, err
}
total += n
start = end
}
return total, nil
}
func (d *DB) insertBatch(ctx context.Context, exec Execer, rows []row.Row) (int64, error) {
var b strings.Builder
b.WriteString(d.verb())
b.WriteByte(' ')
b.WriteString(d.table())
b.WriteString(" (")
b.WriteString(sqlutil.Columns(d.Columns))
b.WriteString(") VALUES ")
tuple := sqlutil.ParamTuple(len(d.Columns))
args := make([]any, 0, len(rows)*len(d.Columns))
for n, r := range rows {
if n > 0 {
b.WriteByte(',')
}
b.WriteString(tuple)
for _, i := range d.OutIdx {
args = append(args, r.Values[i].DriverValue())
}
}
res, err := exec.ExecContext(ctx, b.String(), args...)
if err != nil {
return 0, fmt.Errorf("insert into %s: %w", d.Table, err)
}
n, err := res.RowsAffected()
if err != nil {
return 0, nil
}
return n, nil
}
// DryRunSQL returns a representative insert statement.
func (d *DB) DryRunSQL() []string {
tuple := sqlutil.ParamTuple(len(d.Columns))
return []string{fmt.Sprintf("%s %s (%s) VALUES %s, %s, ...",
d.verb(), d.table(), sqlutil.Columns(d.Columns), tuple, tuple)}
}
+21
View File
@@ -0,0 +1,21 @@
// Package sink contains the file and database write targets.
package sink
import (
"context"
"archer/internal/row"
"archer/internal/shard"
)
// File writes chunks into shard files.
type File struct {
W *shard.Writer
}
func (f *File) WriteChunk(_ context.Context, c *row.Chunk) error { return f.W.WriteChunk(c) }
func (f *File) Flush(context.Context) error { return f.W.Flush() }
func (f *File) Close() error { return f.W.Close() }
// Checkpoint makes buffered shard data durable before a destructive commit.
func (f *File) Checkpoint() error { return f.W.Checkpoint() }
+71
View File
@@ -0,0 +1,71 @@
// Package sqlutil holds small helpers for building SQL text safely.
package sqlutil
import (
"strings"
)
// Ident quotes a MySQL identifier.
func Ident(s string) string {
return "`" + strings.ReplaceAll(s, "`", "``") + "`"
}
// Qualified quotes a database-qualified table name.
func Qualified(db, table string) string {
if db == "" {
return Ident(table)
}
return Ident(db) + "." + Ident(table)
}
// Columns joins quoted column names with commas.
func Columns(cols []string) string {
parts := make([]string, len(cols))
for i, c := range cols {
parts[i] = Ident(c)
}
return strings.Join(parts, ", ")
}
// Placeholders returns n comma separated question marks.
func Placeholders(n int) string {
if n <= 0 {
return ""
}
return strings.Repeat("?, ", n-1) + "?"
}
// RowTuple renders a parenthesized, quoted column list.
func RowTuple(cols []string) string {
return "(" + Columns(cols) + ")"
}
// ParamTuple renders a parenthesized list of n placeholders.
func ParamTuple(n int) string {
return "(" + Placeholders(n) + ")"
}
// GreaterThan renders a keyset comparison (cols) > (?, ?, ...). For a single
// column it emits the simpler col > ?.
func GreaterThan(cols []string) string {
if len(cols) == 1 {
return Ident(cols[0]) + " > ?"
}
return RowTuple(cols) + " > " + ParamTuple(len(cols))
}
// InList renders (cols) IN ((?,?),(?,?),...). count is the number of tuples.
func InList(cols []string, count int) string {
if count == 0 {
return "1=0"
}
tuple := ParamTuple(len(cols))
tuples := make([]string, count)
for i := range tuples {
tuples[i] = tuple
}
if len(cols) == 1 {
return Ident(cols[0]) + " IN (" + Placeholders(count) + ")"
}
return RowTuple(cols) + " IN (" + strings.Join(tuples, ", ") + ")"
}
+36
View File
@@ -0,0 +1,36 @@
package sqlutil
import "testing"
func TestIdentQuoting(t *testing.T) {
if got := Ident("a`b"); got != "`a``b`" {
t.Fatalf("got %s", got)
}
if got := Qualified("db", "t"); got != "`db`.`t`" {
t.Fatalf("got %s", got)
}
if got := Qualified("", "t"); got != "`t`" {
t.Fatalf("got %s", got)
}
}
func TestGreaterThan(t *testing.T) {
if got := GreaterThan([]string{"id"}); got != "`id` > ?" {
t.Fatalf("got %s", got)
}
if got := GreaterThan([]string{"a", "b"}); got != "(`a`, `b`) > (?, ?)" {
t.Fatalf("got %s", got)
}
}
func TestInList(t *testing.T) {
if got := InList([]string{"id"}, 3); got != "`id` IN (?, ?, ?)" {
t.Fatalf("got %s", got)
}
if got := InList([]string{"a", "b"}, 2); got != "(`a`, `b`) IN ((?, ?), (?, ?))" {
t.Fatalf("got %s", got)
}
if got := InList([]string{"id"}, 0); got != "1=0" {
t.Fatalf("got %s", got)
}
}
+14
View File
@@ -0,0 +1,14 @@
// Package version reports build information injected at link time.
package version
import "fmt"
var (
Version = "dev"
Commit = "none"
Date = "unknown"
)
func String() string {
return fmt.Sprintf("archer %s (commit %s, built %s)", Version, Commit, Date)
}