Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,705 @@
|
||||
// Package app wires the CLI, schema probe, scanners, shard writer and
|
||||
// database targets into one run.
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/checkpoint"
|
||||
"archer/internal/cli"
|
||||
"archer/internal/lag"
|
||||
"archer/internal/progress"
|
||||
"archer/internal/purge"
|
||||
"archer/internal/row"
|
||||
"archer/internal/scan"
|
||||
"archer/internal/shard"
|
||||
"archer/internal/sink"
|
||||
)
|
||||
|
||||
// Stats collects counters and phase timings for --statistics.
|
||||
type Stats struct {
|
||||
Started time.Time
|
||||
RowsRead int64
|
||||
RowsCommitted int64
|
||||
RowsInserted int64
|
||||
RowsDeleted int64
|
||||
ScanDur time.Duration
|
||||
FileDur time.Duration
|
||||
DBDur time.Duration
|
||||
CheckpointDur time.Duration
|
||||
Shards int64
|
||||
Bytes int64
|
||||
}
|
||||
|
||||
// App holds a run's state.
|
||||
type App struct {
|
||||
cfg *cli.Config
|
||||
srcDB *sql.DB
|
||||
destDB *sql.DB
|
||||
srcCfg *mysql.Config
|
||||
destCfg *mysql.Config
|
||||
|
||||
loc *time.Location
|
||||
probe *probeResult
|
||||
cfgHash string
|
||||
cfgFields map[string]string
|
||||
|
||||
scanner scan.Scanner
|
||||
fileSink *sink.File
|
||||
writer *shard.Writer
|
||||
manifest *shard.Manifest
|
||||
destSink *sink.DB
|
||||
purger *purge.Purger
|
||||
lagChecker *lag.Checker
|
||||
|
||||
sameInstance bool
|
||||
resume *scan.Resume
|
||||
alreadyComplete bool
|
||||
|
||||
st *checkpoint.State
|
||||
stats Stats
|
||||
|
||||
reporter *progress.Reporter
|
||||
estimate *progress.Estimate
|
||||
baseRows int64
|
||||
}
|
||||
|
||||
// Run executes one archiving run.
|
||||
func Run(ctx context.Context, cfg *cli.Config) error {
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
a := &App{cfg: cfg, stats: Stats{Started: time.Now()}}
|
||||
defer a.close()
|
||||
if !cfg.DryRun {
|
||||
a.startReporter()
|
||||
defer a.reporter.Stop()
|
||||
}
|
||||
|
||||
if err := a.setupSource(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.DryRun {
|
||||
return a.printDryRun(os.Stdout)
|
||||
}
|
||||
if err := a.prepare(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if a.alreadyComplete {
|
||||
fmt.Fprintf(os.Stdout, "archer: output in %s is already complete; nothing to do\n", cfg.FileDir)
|
||||
return nil
|
||||
}
|
||||
return a.execute(ctx)
|
||||
}
|
||||
|
||||
func (a *App) startReporter() {
|
||||
tty := progress.IsTerminal(os.Stderr)
|
||||
enabled := a.cfg.Progress.Enabled(tty)
|
||||
a.reporter = progress.New(progress.Options{
|
||||
Out: os.Stderr,
|
||||
Enabled: enabled,
|
||||
Live: enabled && tty,
|
||||
Interval: a.cfg.ProgressInterval,
|
||||
RowEvery: a.cfg.Progress.Rows,
|
||||
})
|
||||
a.reporter.Start()
|
||||
}
|
||||
|
||||
func (a *App) phase(name, reason string) {
|
||||
if a.reporter != nil {
|
||||
a.reporter.Phase(name, reason)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) message(format string, args ...any) {
|
||||
if a.reporter != nil {
|
||||
a.reporter.Message(format, args...)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
||||
}
|
||||
|
||||
func (a *App) setupSource(ctx context.Context) error {
|
||||
a.phase("probe", "")
|
||||
loc, err := time.LoadLocation(a.cfg.ShardTimeZone)
|
||||
if err != nil {
|
||||
return archererr.Newf(archererr.KindConfig, "invalid --shard-time-zone: %v", err)
|
||||
}
|
||||
a.loc = loc
|
||||
srcCfg, err := a.cfg.SourceConfig()
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConfig, err)
|
||||
}
|
||||
if srcCfg.DBName == "" {
|
||||
return archererr.Newf(archererr.KindConfig, "--source must include a database name")
|
||||
}
|
||||
a.srcCfg = srcCfg
|
||||
a.srcDB, err = openDB(srcCfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.probe, err = probeTable(ctx, a.srcDB, a.cfg, srcCfg.DBName, loc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.cfgHash, a.cfgFields, err = a.cfg.Fingerprint()
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConfig, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) prepare(ctx context.Context) error {
|
||||
a.phase("resume", "")
|
||||
if err := a.openDest(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if a.cfg.Purge {
|
||||
autoInc := false
|
||||
if len(a.probe.key.Columns) == 1 {
|
||||
if c, ok := a.probe.table.Column(a.probe.key.Columns[0]); ok {
|
||||
autoInc = c.IsAutoIncrement()
|
||||
}
|
||||
}
|
||||
a.purger = purge.New(a.srcDB, a.srcCfg.DBName, a.cfg.Table, a.probe.key.Columns,
|
||||
a.cfg.Where, a.cfg.BulkDelete, a.cfg.SafeAutoIncrement, autoInc)
|
||||
}
|
||||
if a.cfg.CheckReplicaLag != "" {
|
||||
c, err := lag.Open(a.cfg.CheckReplicaLag, a.cfg.MaxLag, a.cfg.CheckInterval)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConnection, err)
|
||||
}
|
||||
c.OnWait = func(reason string) {
|
||||
a.phase("lag-wait", reason)
|
||||
}
|
||||
a.lagChecker = c
|
||||
}
|
||||
if a.cfg.HasFile() {
|
||||
if err := a.prepareFile(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if a.alreadyComplete {
|
||||
return nil
|
||||
}
|
||||
a.phase("estimate", "")
|
||||
a.estimate = a.queryEstimate(ctx)
|
||||
if a.reporter != nil {
|
||||
a.reporter.SetEstimate(a.estimate)
|
||||
}
|
||||
return a.openScanner(ctx)
|
||||
}
|
||||
|
||||
func (a *App) openDest(ctx context.Context) error {
|
||||
if a.cfg.DestDSN == "" {
|
||||
return nil
|
||||
}
|
||||
destCfg, err := a.cfg.DestConfig()
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConfig, err)
|
||||
}
|
||||
if destCfg.DBName == "" {
|
||||
return archererr.Newf(archererr.KindConfig, "--dest must include a database name")
|
||||
}
|
||||
a.destCfg = destCfg
|
||||
a.sameInstance = cli.SameInstance(a.srcCfg, destCfg)
|
||||
a.destDB, err = openDB(destCfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names := make([]string, len(a.probe.outCols))
|
||||
for i, c := range a.probe.outCols {
|
||||
names[i] = c.Name
|
||||
}
|
||||
s, err := sink.NewDB(ctx, a.destDB, destCfg.DBName, a.cfg.DestTable, names, a.cfg.InsertMode, a.probe.outIdx)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
a.destSink = s
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) prepareFile(ctx context.Context) error {
|
||||
if err := os.MkdirAll(a.cfg.FileDir, 0o755); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("create output directory: %w", err))
|
||||
}
|
||||
comp, err := shard.NewCompressor(a.cfg.Compress, a.cfg.CompressLevel)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConfig, err)
|
||||
}
|
||||
man, err := shard.OpenManifest(a.cfg.FileDir)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
a.manifest = man
|
||||
maxSeq, err := man.Adopt(a.cfg.FileDir, a.cfg.Format, comp)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
|
||||
st, err := checkpoint.Load(a.cfg.FileDir)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
startBucket, startSeq := "", int64(0)
|
||||
if st != nil {
|
||||
if st.ConfigHash != a.cfgHash {
|
||||
if !a.cfg.ForceResume {
|
||||
diffs := cli.DiffFields(st.Config, a.cfgFields)
|
||||
if len(diffs) == 0 {
|
||||
diffs = []string{"fingerprint changed but no field differences were recorded"}
|
||||
}
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"checkpoint in %s was created with a different configuration:\n %s\nre-run with --force-resume to ignore this",
|
||||
a.cfg.FileDir, strings.Join(diffs, "\n "))
|
||||
}
|
||||
a.message("warning: --force-resume ignores the changed configuration fingerprint")
|
||||
}
|
||||
if st.Complete {
|
||||
a.alreadyComplete = true
|
||||
a.st = st
|
||||
return nil
|
||||
}
|
||||
a.st = st
|
||||
a.baseRows = st.RowsDone
|
||||
a.resume = &scan.Resume{Phase: st.Cursor.Phase, Key: st.Cursor.Key, DayStart: st.Cursor.DayStart}
|
||||
startBucket, startSeq = st.LastBucket, st.LastShardSeq
|
||||
} else {
|
||||
a.st = &checkpoint.State{RunID: newRunID(), ShardBy: a.cfg.ShardBy}
|
||||
}
|
||||
if a.reporter != nil {
|
||||
a.reporter.SetBase(a.baseRows)
|
||||
}
|
||||
if maxSeq[startBucket] > startSeq {
|
||||
startSeq = maxSeq[startBucket]
|
||||
}
|
||||
|
||||
var keep string
|
||||
var resumeMeta *shard.PartialMeta
|
||||
if a.cfg.Purge {
|
||||
if m, err := shard.LoadPartialMeta(a.cfg.FileDir, startBucket, startSeq+1); err != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, err)
|
||||
} else if m != nil {
|
||||
resumeMeta = m
|
||||
keep = shard.PartialPath(a.cfg.FileDir, startBucket, startSeq+1)
|
||||
}
|
||||
}
|
||||
if err := shard.CleanupPartials(a.cfg.FileDir, keep); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
|
||||
w, err := shard.Open(shard.Options{
|
||||
Dir: a.cfg.FileDir,
|
||||
Mode: a.cfg.ShardBy,
|
||||
ShardRows: int64(a.cfg.ShardRows),
|
||||
Format: a.cfg.Format,
|
||||
Compressor: comp,
|
||||
Columns: a.probe.selectMeta,
|
||||
OutIdx: a.probe.outIdx,
|
||||
CSVNull: a.cfg.CSVNull,
|
||||
StartBucket: startBucket,
|
||||
StartSeq: startSeq,
|
||||
}, man)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
if resumeMeta != nil {
|
||||
if err := w.Resume(resumeMeta); err != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
}
|
||||
a.writer = w
|
||||
a.fileSink = &sink.File{W: w}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) openScanner(ctx context.Context) error {
|
||||
opts := scan.Options{
|
||||
DB: a.srcDB,
|
||||
DBName: a.srcCfg.DBName,
|
||||
Table: a.cfg.Table,
|
||||
Where: a.cfg.Where,
|
||||
SelectCols: a.probe.selectCols,
|
||||
KeyCols: a.probe.key.Columns,
|
||||
KeyIdx: a.probe.keyIdx,
|
||||
ChunkSize: a.cfg.ChunkSize,
|
||||
Shard: a.probe.shard,
|
||||
ShardIdx: a.probe.shardIdx,
|
||||
ShardLoc: a.loc,
|
||||
ZeroDate: a.cfg.ZeroDate,
|
||||
Resume: a.resume,
|
||||
}
|
||||
switch a.probe.path {
|
||||
case "A":
|
||||
a.scanner = scan.NewDateA(opts)
|
||||
case "B":
|
||||
s, err := scan.NewDateB(ctx, opts)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConnection, err)
|
||||
}
|
||||
a.scanner = s
|
||||
default:
|
||||
a.scanner = scan.NewPK(opts)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) execute(ctx context.Context) error {
|
||||
if a.cfg.HasFile() {
|
||||
a.message("note: file output is at-least-once; a crash and rerun may duplicate rows in shard files")
|
||||
}
|
||||
if a.cfg.Purge {
|
||||
a.message("warning: --purge deletes archived rows from the source table")
|
||||
}
|
||||
if a.cfg.BulkDelete {
|
||||
a.message("warning: --bulk-delete removes all rows in the scanned key range that match --where, including concurrent inserts")
|
||||
}
|
||||
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return a.interrupt(err)
|
||||
}
|
||||
if a.lagChecker != nil {
|
||||
if err := a.lagChecker.Wait(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return a.interrupt(err)
|
||||
}
|
||||
return archererr.Wrap(archererr.KindConnection, err)
|
||||
}
|
||||
}
|
||||
a.phase("scan", "")
|
||||
start := time.Now()
|
||||
chunk, err := a.scanner.Next(ctx)
|
||||
a.stats.ScanDur += time.Since(start)
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return a.interrupt(err)
|
||||
}
|
||||
return archererr.Wrap(archererr.KindOther, fmt.Errorf("scan: %w", err))
|
||||
}
|
||||
if err := a.processChunk(ctx, chunk); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return a.interrupt(err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
a.stats.RowsRead += int64(chunk.Len())
|
||||
a.publishProgress(chunk)
|
||||
}
|
||||
return a.finish()
|
||||
}
|
||||
|
||||
func (a *App) processChunk(ctx context.Context, chunk *row.Chunk) error {
|
||||
fileStart := time.Now()
|
||||
if a.fileSink != nil {
|
||||
a.phase("file", "")
|
||||
if err := a.fileSink.WriteChunk(ctx, chunk); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
if a.cfg.Purge {
|
||||
if err := a.fileSink.Checkpoint(); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
a.stats.FileDur += time.Since(fileStart)
|
||||
|
||||
dbStart := time.Now()
|
||||
if a.destSink != nil || a.purger != nil {
|
||||
a.phase("db", "")
|
||||
}
|
||||
if err := a.runDB(ctx, chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
a.stats.DBDur += time.Since(dbStart)
|
||||
|
||||
if a.fileSink != nil && !a.cfg.Purge {
|
||||
if err := a.fileSink.Flush(ctx); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
}
|
||||
|
||||
a.stats.RowsCommitted += int64(chunk.Len())
|
||||
if a.cfg.HasFile() {
|
||||
a.phase("checkpoint", "")
|
||||
}
|
||||
cpStart := time.Now()
|
||||
if err := a.saveState(false); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("save checkpoint: %w", err))
|
||||
}
|
||||
a.stats.CheckpointDur += time.Since(cpStart)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) runDB(ctx context.Context, chunk *row.Chunk) error {
|
||||
attempts := a.cfg.Retries + 1
|
||||
if a.destSink != nil && a.purger != nil && a.sameInstance {
|
||||
err := retry(ctx, attempts, func() error {
|
||||
tx, err := a.destDB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := a.destSink.InsertChunk(ctx, tx, chunk)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
d, err := a.purger.DeleteChunk(ctx, tx, chunk)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
a.stats.RowsInserted += n
|
||||
a.stats.RowsDeleted += d
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if a.destSink != nil {
|
||||
err := retry(ctx, attempts, func() error {
|
||||
tx, err := a.destDB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := a.destSink.InsertChunk(ctx, tx, chunk)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
a.stats.RowsInserted += n
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
}
|
||||
if a.purger != nil {
|
||||
err := retry(ctx, attempts, func() error {
|
||||
tx, err := a.srcDB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d, err := a.purger.DeleteChunk(ctx, tx, chunk)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
a.stats.RowsDeleted += d
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func retry(ctx context.Context, attempts int, fn func() error) error {
|
||||
var err error
|
||||
for i := 0; i < attempts; i++ {
|
||||
if i > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(time.Duration(i) * time.Second):
|
||||
}
|
||||
}
|
||||
if err = fn(); err == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *App) saveState(complete bool) error {
|
||||
if a.cfg.FileDir == "" || a.st == nil || a.alreadyComplete {
|
||||
return nil
|
||||
}
|
||||
st := a.st
|
||||
st.ConfigHash = a.cfgHash
|
||||
st.Config = a.cfgFields
|
||||
st.ShardBy = a.cfg.ShardBy
|
||||
st.RowsDone = a.baseRows + a.stats.RowsCommitted
|
||||
if a.scanner != nil {
|
||||
cs := a.scanner.ResumeState()
|
||||
st.Cursor = checkpoint.Cursor{Phase: cs.Phase, Key: cs.Key, DayStart: cs.DayStart}
|
||||
}
|
||||
if a.writer != nil {
|
||||
st.LastBucket, st.LastShardSeq = a.writer.StatePosition()
|
||||
}
|
||||
st.Complete = complete
|
||||
return checkpoint.Save(a.cfg.FileDir, st)
|
||||
}
|
||||
|
||||
func (a *App) finish() error {
|
||||
a.phase("finalize", "")
|
||||
if a.writer != nil {
|
||||
if err := a.writer.Close(); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
var shards, bytes int64
|
||||
_, shards, bytes = a.writer.Stats()
|
||||
a.stats.Shards, a.stats.Bytes = shards, bytes
|
||||
}
|
||||
if err := a.saveState(true); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("save checkpoint: %w", err))
|
||||
}
|
||||
a.printStats()
|
||||
if a.reporter != nil {
|
||||
if a.writer != nil {
|
||||
a.reporter.SetShards(a.stats.Shards)
|
||||
a.reporter.SetBytes(a.stats.Bytes)
|
||||
}
|
||||
a.reporter.Done()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) interrupt(cause error) error {
|
||||
if a.reporter != nil {
|
||||
a.reporter.Interrupted()
|
||||
}
|
||||
if a.writer != nil {
|
||||
if err := a.writer.SealCurrent(); err != nil {
|
||||
a.message("warning: could not seal current shard: %v", err)
|
||||
}
|
||||
}
|
||||
if err := a.saveState(false); err != nil {
|
||||
a.message("warning: could not save checkpoint: %v", err)
|
||||
}
|
||||
return archererr.Wrap(archererr.KindInterrupted, cause)
|
||||
}
|
||||
|
||||
func (a *App) publishProgress(chunk *row.Chunk) {
|
||||
if a.reporter == nil {
|
||||
return
|
||||
}
|
||||
if a.estimate != nil {
|
||||
switch a.estimate.Kind {
|
||||
case "day":
|
||||
if len(chunk.Buckets) > 0 {
|
||||
b := chunk.Buckets[len(chunk.Buckets)-1]
|
||||
if b != "" && b != "_null" && !strings.HasPrefix(b, "0000-") {
|
||||
if t, err := time.ParseInLocation("2006-01-02", b, a.loc); err == nil {
|
||||
a.reporter.SetPosition(float64(t.Unix()))
|
||||
}
|
||||
}
|
||||
}
|
||||
case "key":
|
||||
if len(chunk.LastKey) == 1 && !chunk.LastKey[0].Null {
|
||||
if n, err := strconv.ParseInt(chunk.LastKey[0].Val, 10, 64); err == nil {
|
||||
a.reporter.SetPosition(float64(n))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if a.writer != nil {
|
||||
a.reporter.SetShard(a.writer.CurrentShard())
|
||||
a.reporter.SetBytes(a.writer.WrittenBytes())
|
||||
}
|
||||
a.reporter.AddRows(int64(chunk.Len()))
|
||||
}
|
||||
|
||||
func (a *App) printStats() {
|
||||
if !a.cfg.Statistics {
|
||||
return
|
||||
}
|
||||
fmt.Fprintln(os.Stdout, "this run:")
|
||||
fmt.Fprintf(os.Stdout, " rows read: %d\n", a.stats.RowsRead)
|
||||
fmt.Fprintf(os.Stdout, " rows committed: %d\n", a.stats.RowsCommitted)
|
||||
fmt.Fprintf(os.Stdout, " rows inserted: %d\n", a.stats.RowsInserted)
|
||||
fmt.Fprintf(os.Stdout, " rows deleted: %d\n", a.stats.RowsDeleted)
|
||||
if a.writer != nil {
|
||||
fmt.Fprintf(os.Stdout, " shards: %d\n", a.stats.Shards)
|
||||
fmt.Fprintf(os.Stdout, " bytes written: %d\n", a.stats.Bytes)
|
||||
}
|
||||
fmt.Fprintf(os.Stdout, " elapsed: %s (scan %s, file %s, db %s, checkpoint %s)\n",
|
||||
time.Since(a.stats.Started).Round(time.Millisecond),
|
||||
a.stats.ScanDur.Round(time.Millisecond), a.stats.FileDur.Round(time.Millisecond),
|
||||
a.stats.DBDur.Round(time.Millisecond), a.stats.CheckpointDur.Round(time.Millisecond))
|
||||
|
||||
fmt.Fprintln(os.Stdout, "cumulative (including prior runs):")
|
||||
fmt.Fprintf(os.Stdout, " rows read: %d\n", a.baseRows+a.stats.RowsRead)
|
||||
fmt.Fprintf(os.Stdout, " rows committed: %d\n", a.baseRows+a.stats.RowsCommitted)
|
||||
if a.writer != nil && a.manifest != nil {
|
||||
var rows, bytes int64
|
||||
entries := a.manifest.Entries()
|
||||
for _, e := range entries {
|
||||
rows += e.Rows
|
||||
bytes += e.Bytes
|
||||
}
|
||||
fmt.Fprintf(os.Stdout, " shards: %d\n", len(entries))
|
||||
fmt.Fprintf(os.Stdout, " rows written: %d\n", rows)
|
||||
fmt.Fprintf(os.Stdout, " bytes written: %d\n", bytes)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) close() {
|
||||
if a.scanner != nil {
|
||||
_ = a.scanner.Close()
|
||||
}
|
||||
if a.writer != nil {
|
||||
_ = a.writer.Abandon()
|
||||
} else if a.manifest != nil {
|
||||
_ = a.manifest.Close()
|
||||
}
|
||||
if a.lagChecker != nil {
|
||||
_ = a.lagChecker.Close()
|
||||
}
|
||||
if a.destDB != nil {
|
||||
_ = a.destDB.Close()
|
||||
}
|
||||
if a.srcDB != nil {
|
||||
_ = a.srcDB.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func openDB(cfg *mysql.Config) (*sql.DB, error) {
|
||||
db, err := sql.Open("mysql", cfg.FormatDSN())
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindConnection, err)
|
||||
}
|
||||
db.SetMaxOpenConns(8)
|
||||
db.SetMaxIdleConns(4)
|
||||
db.SetConnMaxLifetime(time.Hour)
|
||||
if err := db.Ping(); err != nil {
|
||||
db.Close()
|
||||
return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("connect %s: %w", cfg.Addr, err))
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func newRunID() string {
|
||||
b := make([]byte, 8)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return time.Now().UTC().Format("20060102T150405Z")
|
||||
}
|
||||
return time.Now().UTC().Format("20060102T150405Z") + "-" + hex.EncodeToString(b)
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
|
||||
"archer/internal/purge"
|
||||
"archer/internal/shard"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
func (a *App) printDryRun(w io.Writer) error {
|
||||
cfg := a.cfg
|
||||
comp, err := shard.NewCompressor(cfg.Compress, cfg.CompressLevel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(w, "== resolved configuration ==")
|
||||
fmt.Fprintf(w, "source: %s\n", redactDSN(cfg.SourceDSN))
|
||||
if cfg.DestDSN != "" {
|
||||
fmt.Fprintf(w, "dest: %s\n", redactDSN(cfg.DestDSN))
|
||||
}
|
||||
fmt.Fprintf(w, "table: %s\n", cfg.Table)
|
||||
if cfg.DestDSN != "" {
|
||||
fmt.Fprintf(w, "dest table: %s\n", cfg.DestTable)
|
||||
}
|
||||
fmt.Fprintf(w, "where: %s\n", cfg.Where)
|
||||
if len(cfg.Columns) > 0 {
|
||||
fmt.Fprintf(w, "columns: %s\n", strings.Join(cfg.Columns, ", "))
|
||||
} else {
|
||||
fmt.Fprintf(w, "columns: (all)\n")
|
||||
}
|
||||
targets := []string{}
|
||||
if cfg.HasFile() {
|
||||
targets = append(targets, "file="+cfg.FileDir)
|
||||
}
|
||||
if cfg.DestDSN != "" {
|
||||
targets = append(targets, "dest")
|
||||
}
|
||||
if cfg.Purge {
|
||||
targets = append(targets, "purge")
|
||||
}
|
||||
fmt.Fprintf(w, "targets: %s\n", strings.Join(targets, ", "))
|
||||
fmt.Fprintf(w, "chunk size: %d\n", cfg.ChunkSize)
|
||||
fmt.Fprintf(w, "shard by: %s\n", cfg.ShardBy)
|
||||
if cfg.ShardBy != "none" {
|
||||
fmt.Fprintf(w, "shard rows: %d\n", cfg.ShardRows)
|
||||
}
|
||||
if a.probe.shard != nil {
|
||||
fmt.Fprintf(w, "shard column: %s (%s)\n", a.probe.shard.Column, a.probe.shard.Type)
|
||||
fmt.Fprintf(w, "shard time zone: %s\n", cfg.ShardTimeZone)
|
||||
}
|
||||
fmt.Fprintf(w, "format: %s\n", cfg.Format)
|
||||
if comp.Name == "none" {
|
||||
fmt.Fprintf(w, "compress: none\n")
|
||||
} else {
|
||||
fmt.Fprintf(w, "compress: %s (level %d)\n", comp.Name, comp.Level)
|
||||
}
|
||||
if cfg.InsertMode != "" {
|
||||
fmt.Fprintf(w, "insert mode: %s\n", cfg.InsertMode)
|
||||
}
|
||||
fmt.Fprintf(w, "max lag: %s (check interval %s)\n", cfg.MaxLag, cfg.CheckInterval)
|
||||
fmt.Fprintf(w, "retries: %d\n", cfg.Retries)
|
||||
fmt.Fprintf(w, "safe auto inc: %t\n", cfg.SafeAutoIncrement)
|
||||
fmt.Fprintf(w, "config hash: %s\n", a.cfgHash)
|
||||
|
||||
fmt.Fprintln(w, "\n== index probe ==")
|
||||
fmt.Fprintf(w, "scan key: %s (%s)\n", a.probe.key.Name, strings.Join(a.probe.key.Columns, ", "))
|
||||
if a.probe.path == "A" {
|
||||
fmt.Fprintf(w, "scan path: A (keyset chunks using index %s)\n", a.probe.joint.Name)
|
||||
} else if a.probe.path == "B" {
|
||||
if a.probe.single != nil {
|
||||
fmt.Fprintf(w, "scan path: B (day-at-a-time using index %s)\n", a.probe.single.Name)
|
||||
} else {
|
||||
fmt.Fprintln(w, "scan path: B (day-at-a-time, full table scan allowed)")
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintln(w, "scan path: primary key seek")
|
||||
}
|
||||
|
||||
fmt.Fprintln(w, "\n== first SELECT ==")
|
||||
for _, q := range a.dryRunSelects() {
|
||||
fmt.Fprintf(w, " %s\n", q)
|
||||
}
|
||||
|
||||
fmt.Fprintln(w, "\n== write statements ==")
|
||||
if cfg.DestDSN != "" {
|
||||
fmt.Fprintf(w, " %s\n", a.dryRunInsert())
|
||||
}
|
||||
if cfg.Purge {
|
||||
p := purge.New(nil, a.srcCfg.DBName, cfg.Table, a.probe.key.Columns, cfg.Where, cfg.BulkDelete, cfg.SafeAutoIncrement, false)
|
||||
for _, q := range p.DryRunSQL() {
|
||||
fmt.Fprintf(w, " %s\n", q)
|
||||
}
|
||||
}
|
||||
if cfg.HasFile() {
|
||||
fmt.Fprintf(w, " file: %s/[...]<seq>.<pk_lo>-<pk_hi>.%s%s (written as .partial until sealed)\n",
|
||||
cfg.FileDir, cfg.Format, comp.Ext)
|
||||
}
|
||||
fmt.Fprintln(w, "\nnote: file output is at-least-once; a crash and rerun may duplicate rows in shard files")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) dryRunSelects() []string {
|
||||
cols := sqlutil.Columns(a.probe.selectCols)
|
||||
table := sqlutil.Qualified(a.srcCfg.DBName, a.cfg.Table)
|
||||
where := a.cfg.Where
|
||||
key := sqlutil.Columns(a.probe.key.Columns)
|
||||
limit := fmt.Sprintf(" LIMIT %d", a.cfg.ChunkSize)
|
||||
switch a.probe.path {
|
||||
case "A":
|
||||
shard := sqlutil.Ident(a.probe.shard.Column)
|
||||
colsWithShard := sqlutil.Columns(append([]string{a.probe.shard.Column}, a.probe.key.Columns...))
|
||||
return []string{
|
||||
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s IS NULL ORDER BY %s%s", cols, table, where, shard, key, limit),
|
||||
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s IS NOT NULL AND (%s) > (?, ...) ORDER BY %s%s", cols, table, where, shard, colsWithShard, colsWithShard, limit),
|
||||
}
|
||||
case "B":
|
||||
shard := sqlutil.Ident(a.probe.shard.Column)
|
||||
colsWithShard := sqlutil.Columns(append([]string{a.probe.shard.Column}, a.probe.key.Columns...))
|
||||
return []string{
|
||||
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s IS NULL ORDER BY %s", cols, table, where, shard, key),
|
||||
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s >= ? AND %s < ? ORDER BY %s", cols, table, where, shard, shard, colsWithShard),
|
||||
}
|
||||
default:
|
||||
return []string{fmt.Sprintf("SELECT %s FROM %s WHERE (%s) ORDER BY %s%s", cols, table, where, key, limit)}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) dryRunInsert() string {
|
||||
destCfg, err := a.cfg.DestConfig()
|
||||
dbName := ""
|
||||
if err == nil && destCfg != nil {
|
||||
dbName = destCfg.DBName
|
||||
}
|
||||
verb := "INSERT INTO"
|
||||
switch a.cfg.InsertMode {
|
||||
case "ignore":
|
||||
verb = "INSERT IGNORE INTO"
|
||||
case "replace":
|
||||
verb = "REPLACE INTO"
|
||||
}
|
||||
names := make([]string, len(a.probe.outCols))
|
||||
for i, c := range a.probe.outCols {
|
||||
names[i] = c.Name
|
||||
}
|
||||
tuple := sqlutil.ParamTuple(len(names))
|
||||
return fmt.Sprintf("%s %s (%s) VALUES %s, %s, ...",
|
||||
verb, sqlutil.Qualified(dbName, a.cfg.DestTable), sqlutil.Columns(names), tuple, tuple)
|
||||
}
|
||||
|
||||
func redactDSN(dsn string) string {
|
||||
c, err := mysql.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
return "<invalid dsn>"
|
||||
}
|
||||
c.Passwd = ""
|
||||
return c.FormatDSN()
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"archer/internal/estimate"
|
||||
"archer/internal/progress"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
// estimateTimeout bounds the one-off MIN/MAX query at startup.
|
||||
const estimateTimeout = 2 * time.Second
|
||||
|
||||
// estimateSpec decides which cheap position metric this run can expose.
|
||||
func (a *App) estimateSpec() estimate.Spec {
|
||||
if a.cfg.IsDateShard() && a.probe.shard != nil {
|
||||
return estimate.Spec{
|
||||
Kind: "day",
|
||||
Column: a.probe.shard.Column,
|
||||
ShardKind: a.probe.shard.Kind,
|
||||
Unit: a.probe.shard.Unit,
|
||||
Loc: a.loc,
|
||||
}
|
||||
}
|
||||
types := make([]string, len(a.probe.key.Columns))
|
||||
for i, name := range a.probe.key.Columns {
|
||||
c, _ := a.probe.table.Column(name)
|
||||
types[i] = c.DataType
|
||||
}
|
||||
col, ok := estimate.KeyColumn(a.probe.key.Columns, types)
|
||||
if !ok {
|
||||
return estimate.Spec{}
|
||||
}
|
||||
return estimate.Spec{Kind: "key", Column: col}
|
||||
}
|
||||
|
||||
// queryEstimate runs the bounded range query on a dedicated connection and
|
||||
// silently returns nil when the metric does not apply or the query fails.
|
||||
func (a *App) queryEstimate(ctx context.Context) *progress.Estimate {
|
||||
spec := a.estimateSpec()
|
||||
if spec.Kind == "" {
|
||||
return nil
|
||||
}
|
||||
conn, err := a.srcDB.Conn(ctx)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer conn.Close()
|
||||
query := func(qctx context.Context, column string) (sql.NullString, sql.NullString, error) {
|
||||
col := sqlutil.Ident(column)
|
||||
q := fmt.Sprintf("SELECT MIN(%s), MAX(%s) FROM %s WHERE (%s)",
|
||||
col, col, sqlutil.Qualified(a.srcCfg.DBName, a.cfg.Table), a.cfg.Where)
|
||||
if spec.Kind == "day" {
|
||||
q += " AND " + col + " IS NOT NULL"
|
||||
}
|
||||
var minRaw, maxRaw sql.NullString
|
||||
err := conn.QueryRowContext(qctx, q).Scan(&minRaw, &maxRaw)
|
||||
return minRaw, maxRaw, err
|
||||
}
|
||||
return estimate.Bounded(ctx, query, spec, estimateTimeout)
|
||||
}
|
||||
@@ -0,0 +1,918 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/ulikunitz/xz"
|
||||
|
||||
"archer/internal/cli"
|
||||
)
|
||||
|
||||
func testDSN(t *testing.T) string {
|
||||
d := os.Getenv("ARCHER_TEST_DSN")
|
||||
if d == "" {
|
||||
t.Skip("ARCHER_TEST_DSN is not set; skipping MySQL integration test")
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
if dsn := os.Getenv("ARCHER_TEST_DSN"); dsn != "" {
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err == nil && cfg.DBName != "" {
|
||||
dbName := cfg.DBName
|
||||
cfg.DBName = ""
|
||||
db, err := sql.Open("mysql", cfg.FormatDSN())
|
||||
if err == nil {
|
||||
_, _ = db.Exec("CREATE DATABASE IF NOT EXISTS `" + dbName + "`")
|
||||
_, _ = db.Exec("SET GLOBAL local_infile=ON")
|
||||
_, _ = db.Exec("SET GLOBAL sql_mode=''")
|
||||
db.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func openRoot(t *testing.T, dsn string) *sql.DB {
|
||||
t.Helper()
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db, err := sql.Open("mysql", cfg.FormatDSN())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Ping(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
func execSQL(t *testing.T, db *sql.DB, q string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := db.Exec(q, args...); err != nil {
|
||||
t.Fatalf("exec %q: %v", q, err)
|
||||
}
|
||||
}
|
||||
|
||||
var stdoutMu sync.Mutex
|
||||
var stderrMu sync.Mutex
|
||||
|
||||
func captureStdout(fn func() error) (string, error) {
|
||||
stdoutMu.Lock()
|
||||
defer stdoutMu.Unlock()
|
||||
old := os.Stdout
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
os.Stdout = w
|
||||
var buf bytes.Buffer
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
_, _ = io.Copy(&buf, r)
|
||||
close(done)
|
||||
}()
|
||||
runErr := fn()
|
||||
_ = w.Close()
|
||||
os.Stdout = old
|
||||
<-done
|
||||
return buf.String(), runErr
|
||||
}
|
||||
|
||||
func captureStderr(fn func() error) (string, error) {
|
||||
stderrMu.Lock()
|
||||
defer stderrMu.Unlock()
|
||||
old := os.Stderr
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
os.Stderr = w
|
||||
var buf bytes.Buffer
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
_, _ = io.Copy(&buf, r)
|
||||
close(done)
|
||||
}()
|
||||
runErr := fn()
|
||||
_ = w.Close()
|
||||
os.Stderr = old
|
||||
<-done
|
||||
return buf.String(), runErr
|
||||
}
|
||||
|
||||
func runCLI(t *testing.T, ctx context.Context, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
cfg, err := cli.Parse(args, io.Discard)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return captureStdout(func() error { return Run(ctx, cfg) })
|
||||
}
|
||||
|
||||
func runCLIStderr(t *testing.T, ctx context.Context, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
cfg, err := cli.Parse(args, io.Discard)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return captureStderr(func() error { return Run(ctx, cfg) })
|
||||
}
|
||||
|
||||
func baseCLI(dsn, table, out string, extra ...string) []string {
|
||||
args := []string{
|
||||
"--source", dsn,
|
||||
"--table", table,
|
||||
"--where", "1=1",
|
||||
"--file", out,
|
||||
}
|
||||
return append(args, extra...)
|
||||
}
|
||||
|
||||
func listFiles(t *testing.T, dir, pattern string) []string {
|
||||
t.Helper()
|
||||
matches, err := filepath.Glob(filepath.Join(dir, pattern))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sort.Strings(matches)
|
||||
return matches
|
||||
}
|
||||
|
||||
func dumpIDs(t *testing.T, path string) []int {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ids []int
|
||||
for _, line := range strings.Split(strings.TrimRight(string(b), "\n"), "\n") {
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
id, err := strconv.Atoi(strings.SplitN(line, "\t", 2)[0])
|
||||
if err != nil {
|
||||
t.Fatalf("%s: bad line %q", path, line)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func TestIntegrationCountShards(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_count")
|
||||
execSQL(t, db, "CREATE TABLE it_count (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
||||
var b strings.Builder
|
||||
b.WriteString("INSERT INTO it_count (id, v) VALUES ")
|
||||
for i := 1; i <= 25; i++ {
|
||||
if i > 1 {
|
||||
b.WriteString(",")
|
||||
}
|
||||
fmt.Fprintf(&b, "(%d,'v%d')", i, i)
|
||||
}
|
||||
execSQL(t, db, b.String())
|
||||
|
||||
out := t.TempDir()
|
||||
_, err := runCLI(t, context.Background(), baseCLI(dsn, "it_count", out,
|
||||
"--shard-by", "count", "--shard-rows", "10", "--chunk-size", "7", "--format", "dump")...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files := listFiles(t, out, "*.dump")
|
||||
if len(files) != 3 {
|
||||
t.Fatalf("got %d shard files: %v", len(files), files)
|
||||
}
|
||||
wantNames := []string{"000001.1-10.dump", "000002.11-20.dump", "000003.21-25.dump"}
|
||||
var got []int
|
||||
for i, f := range files {
|
||||
if filepath.Base(f) != wantNames[i] {
|
||||
t.Fatalf("file %d = %s want %s", i, filepath.Base(f), wantNames[i])
|
||||
}
|
||||
got = append(got, dumpIDs(t, f)...)
|
||||
}
|
||||
if len(got) != 25 {
|
||||
t.Fatalf("got %d rows", len(got))
|
||||
}
|
||||
for i, id := range got {
|
||||
if id != i+1 {
|
||||
t.Fatalf("row %d = %d, shards are not contiguous", i, id)
|
||||
}
|
||||
}
|
||||
|
||||
entries := readManifest(t, out)
|
||||
if len(entries) != 3 {
|
||||
t.Fatalf("manifest entries %d", len(entries))
|
||||
}
|
||||
for i, e := range entries {
|
||||
if e.Seq != int64(i+1) || e.Rows != int64(len(dumpIDs(t, filepath.Join(out, e.Path)))) {
|
||||
t.Fatalf("entry %d %+v", i, e)
|
||||
}
|
||||
if e.Bytes == 0 || e.SHA == "" {
|
||||
t.Fatalf("entry %d missing bytes/hash", i)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(out, "state.json")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var complete bool
|
||||
st, err := os.ReadFile(filepath.Join(out, "state.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
complete = bytes.Contains(st, []byte(`"complete": true`))
|
||||
if !complete {
|
||||
t.Fatalf("state not complete: %s", st)
|
||||
}
|
||||
}
|
||||
|
||||
type manifestEntry struct {
|
||||
Seq int64 `json:"seq"`
|
||||
Path string `json:"path"`
|
||||
Date string `json:"date"`
|
||||
Rows int64 `json:"rows"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
SHA string `json:"sha256"`
|
||||
PKLo string `json:"pk_lo"`
|
||||
PKHi string `json:"pk_hi"`
|
||||
}
|
||||
|
||||
func readManifest(t *testing.T, dir string) []manifestEntry {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(filepath.Join(dir, "manifest.jsonl"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []manifestEntry
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(b)), "\n") {
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var e manifestEntry
|
||||
if err := json.Unmarshal([]byte(line), &e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestIntegrationDatePathsAgree(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
for _, spec := range []struct{ name, index string }{
|
||||
{"it_date_a", "KEY da (created_at, id)"},
|
||||
{"it_date_b", "KEY db (created_at)"},
|
||||
} {
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS "+spec.name)
|
||||
execSQL(t, db, "CREATE TABLE "+spec.name+` (
|
||||
id BIGINT NOT NULL,
|
||||
v VARCHAR(32),
|
||||
created_at DATETIME NULL,
|
||||
PRIMARY KEY (id),
|
||||
`+spec.index+`)`)
|
||||
execSQL(t, db, `INSERT INTO `+spec.name+` (id, v, created_at) VALUES
|
||||
(1,'a','2025-01-01 10:00:00'),
|
||||
(2,'b','2025-01-01 12:00:00'),
|
||||
(3,'c','2025-01-02 01:00:00'),
|
||||
(4,'d',NULL),
|
||||
(5,'e','2025-01-01 11:00:00'),
|
||||
(6,'f','2025-01-03 00:00:00')`)
|
||||
}
|
||||
outA := t.TempDir()
|
||||
outB := t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_a", outA,
|
||||
"--shard-by", "date", "--shard-column", "created_at", "--format", "dump", "--chunk-size", "2")...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_b", outB,
|
||||
"--shard-by", "date", "--shard-column", "created_at", "--format", "dump", "--chunk-size", "2")...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := readTree(t, outA)
|
||||
b := readTree(t, outB)
|
||||
if len(a) != len(b) {
|
||||
t.Fatalf("file counts differ: %d vs %d (%v / %v)", len(a), len(b), a, b)
|
||||
}
|
||||
for k, va := range a {
|
||||
if b[k] != va {
|
||||
t.Fatalf("%s:\nA=%q\nB=%q", k, va, b[k])
|
||||
}
|
||||
}
|
||||
if _, ok := a["date=_null/000001.4-4.dump"]; !ok {
|
||||
t.Fatalf("NULL row not routed to _null: %v", a)
|
||||
}
|
||||
if len(a) != 4 {
|
||||
t.Fatalf("expected 4 shards (null + 3 days), got %v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func readTree(t *testing.T, dir string) map[string]string {
|
||||
t.Helper()
|
||||
out := map[string]string{}
|
||||
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return err
|
||||
}
|
||||
if !strings.HasSuffix(d.Name(), ".dump") {
|
||||
return nil
|
||||
}
|
||||
rel, _ := filepath.Rel(dir, path)
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out[filepath.ToSlash(rel)] = string(b)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestIntegrationZeroDateAndNull(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_zero")
|
||||
execSQL(t, db, `CREATE TABLE it_zero (
|
||||
id BIGINT NOT NULL,
|
||||
d DATE NULL,
|
||||
PRIMARY KEY (id),
|
||||
KEY dz (d, id))`)
|
||||
execSQL(t, db, `INSERT INTO it_zero (id, d) VALUES
|
||||
(1, NULL), (2, '0000-00-00'), (3, '2024-01-01'), (4, '2024-01-01')`)
|
||||
|
||||
out := t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_zero", out,
|
||||
"--shard-by", "date", "--shard-column", "d")...); err == nil {
|
||||
t.Fatal("expected zero-date error")
|
||||
} else if !strings.Contains(err.Error(), "zero date") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
out = t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_zero", out,
|
||||
"--shard-by", "date", "--shard-column", "d", "--zero-date", "bucket")...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tree := readTree(t, out)
|
||||
for _, want := range []string{
|
||||
"date=_null/000001.1-1.dump",
|
||||
"date=0000-00-00/000001.2-2.dump",
|
||||
"date=2024-01-01/000001.3-4.dump",
|
||||
} {
|
||||
if _, ok := tree[want]; !ok {
|
||||
t.Fatalf("missing %s in %v", want, tree)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationEpochUnits(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_epoch")
|
||||
execSQL(t, db, `CREATE TABLE it_epoch (
|
||||
id BIGINT NOT NULL,
|
||||
ts BIGINT NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
KEY e (ts, id))`)
|
||||
execSQL(t, db, `INSERT INTO it_epoch (id, ts) VALUES
|
||||
(1, 1717200000), (2, 1717200001)`)
|
||||
|
||||
for _, tc := range []struct {
|
||||
unit string
|
||||
vals string
|
||||
}{
|
||||
{"epoch-seconds", "(3, 1717200000), (4, 1717200001)"},
|
||||
{"epoch-milliseconds", "(3, 1717200000000), (4, 1717200001000)"},
|
||||
{"epoch-microseconds", "(3, 1717200000000000), (4, 1717200001000000)"},
|
||||
} {
|
||||
table := "it_epoch_" + strings.Split(tc.unit, "-")[1]
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS "+table)
|
||||
execSQL(t, db, `CREATE TABLE `+table+` (
|
||||
id BIGINT NOT NULL, ts BIGINT NOT NULL,
|
||||
PRIMARY KEY (id), KEY e (ts, id))`)
|
||||
execSQL(t, db, "INSERT INTO "+table+" (id, ts) VALUES "+tc.vals)
|
||||
out := t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, table, out,
|
||||
"--shard-by", "date", "--shard-column", "ts", "--shard-column-type", tc.unit,
|
||||
"--shard-time-zone", "UTC")...); err != nil {
|
||||
t.Fatalf("%s: %v", tc.unit, err)
|
||||
}
|
||||
tree := readTree(t, out)
|
||||
if len(tree) != 1 {
|
||||
t.Fatalf("%s: expected one date bucket, got %v", tc.unit, tree)
|
||||
}
|
||||
for k := range tree {
|
||||
if !strings.Contains(k, "date=2024-06-01") {
|
||||
t.Fatalf("%s: wrong bucket %s", tc.unit, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationNoIndexFailFast(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_noindex")
|
||||
execSQL(t, db, `CREATE TABLE it_noindex (
|
||||
id BIGINT NOT NULL, d DATETIME NULL, PRIMARY KEY (id))`)
|
||||
execSQL(t, db, `INSERT INTO it_noindex (id, d) VALUES (1,'2024-01-01 00:00:00')`)
|
||||
|
||||
out := t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_noindex", out,
|
||||
"--shard-by", "date", "--shard-column", "d")...); err == nil {
|
||||
t.Fatal("expected no-index error")
|
||||
} else if !strings.Contains(err.Error(), "no usable index") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
out = t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_noindex", out,
|
||||
"--shard-by", "date", "--shard-column", "d", "--allow-full-scan")...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(out, "date=2024-01-01")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationDryRunCreatesNothing(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_dry")
|
||||
execSQL(t, db, "CREATE TABLE it_dry (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
||||
execSQL(t, db, "INSERT INTO it_dry (id, v) VALUES (1,'a')")
|
||||
|
||||
out := filepath.Join(t.TempDir(), "never-created")
|
||||
stdout, err := runCLI(t, context.Background(), baseCLI(dsn, "it_dry", out, "--dry-run")...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, statErr := os.Stat(out); !os.IsNotExist(statErr) {
|
||||
t.Fatal("dry-run created the output directory")
|
||||
}
|
||||
if !strings.Contains(stdout, "== first SELECT ==") || !strings.Contains(stdout, "config hash") {
|
||||
t.Fatalf("dry-run output incomplete:\n%s", stdout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationResumeAfterInterrupt(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_resume")
|
||||
execSQL(t, db, "CREATE TABLE it_resume (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
||||
var b strings.Builder
|
||||
b.WriteString("INSERT INTO it_resume (id, v) VALUES ")
|
||||
const total = 20000
|
||||
for i := 1; i <= total; i++ {
|
||||
if i > 1 {
|
||||
b.WriteString(",")
|
||||
}
|
||||
fmt.Fprintf(&b, "(%d,'v')", i)
|
||||
}
|
||||
execSQL(t, db, b.String())
|
||||
|
||||
out := t.TempDir()
|
||||
args := baseCLI(dsn, "it_resume", out, "--shard-by", "count", "--shard-rows", "137",
|
||||
"--chunk-size", "50", "--progress", "100")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
stderrMu.Lock()
|
||||
old := os.Stderr
|
||||
r, w, _ := os.Pipe()
|
||||
os.Stderr = w
|
||||
cfg, err := cli.Parse(args, io.Discard)
|
||||
if err != nil {
|
||||
os.Stderr = old
|
||||
stderrMu.Unlock()
|
||||
t.Fatal(err)
|
||||
}
|
||||
firstErr := make(chan error, 1)
|
||||
go func() { firstErr <- Run(ctx, cfg) }()
|
||||
go func() {
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
n, err := r.Read(buf)
|
||||
if n > 0 && strings.Contains(string(buf[:n]), "progress:") {
|
||||
cancel()
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
err = <-firstErr
|
||||
_ = w.Close()
|
||||
os.Stderr = old
|
||||
stderrMu.Unlock()
|
||||
if err == nil {
|
||||
t.Fatal("first run should have been interrupted")
|
||||
}
|
||||
|
||||
if _, err := runCLI(t, context.Background(), args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
seen := map[int]bool{}
|
||||
for _, f := range listFiles(t, out, "*.dump") {
|
||||
for _, id := range dumpIDs(t, f) {
|
||||
seen[id] = true
|
||||
}
|
||||
}
|
||||
for i := 1; i <= total; i++ {
|
||||
if !seen[i] {
|
||||
t.Fatalf("row %d missing after resume", i)
|
||||
}
|
||||
}
|
||||
st, err := os.ReadFile(filepath.Join(out, "state.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Contains(st, []byte(`"complete": true`)) {
|
||||
t.Fatalf("state not complete: %s", st)
|
||||
}
|
||||
var saved struct {
|
||||
RowsDone int64 `json:"rows_done"`
|
||||
}
|
||||
if err := json.Unmarshal(st, &saved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if saved.RowsDone != total {
|
||||
t.Fatalf("rows_done = %d after resume, want cumulative %d", saved.RowsDone, total)
|
||||
}
|
||||
if _, err := filepath.Glob(filepath.Join(out, "*.partial")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationProgressModes(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_progress")
|
||||
execSQL(t, db, "CREATE TABLE it_progress (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
||||
execSQL(t, db, "INSERT INTO it_progress (id, v) VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e')")
|
||||
|
||||
for _, extra := range [][]string{
|
||||
nil,
|
||||
{"--progress", "off"},
|
||||
{"--progress", "0"},
|
||||
} {
|
||||
out := t.TempDir()
|
||||
args := baseCLI(dsn, "it_progress", out, extra...)
|
||||
errOut, err := runCLIStderr(t, context.Background(), args...)
|
||||
if err != nil {
|
||||
t.Fatalf("%v: %v", extra, err)
|
||||
}
|
||||
if strings.Contains(errOut, "progress:") || strings.Contains(errOut, "done rows=") {
|
||||
t.Fatalf("%v: unexpected progress output %q", extra, errOut)
|
||||
}
|
||||
}
|
||||
|
||||
out := t.TempDir()
|
||||
errOut, err := runCLIStderr(t, context.Background(), baseCLI(dsn, "it_progress", out, "--progress", "on")...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(errOut, "done rows=5") {
|
||||
t.Fatalf("forced progress missing done summary: %q", errOut)
|
||||
}
|
||||
if strings.Contains(errOut, "\r") {
|
||||
t.Fatalf("redirected progress must not use carriage returns: %q", errOut)
|
||||
}
|
||||
|
||||
out = t.TempDir()
|
||||
errOut, err = runCLIStderr(t, context.Background(), baseCLI(dsn, "it_progress", out, "--progress", "1")...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(errOut, "progress: rows=5") {
|
||||
t.Fatalf("row-threshold progress line missing: %q", errOut)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationEstimateMetrics(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_estimate")
|
||||
execSQL(t, db, `CREATE TABLE it_estimate (
|
||||
id BIGINT NOT NULL,
|
||||
v VARCHAR(32),
|
||||
created_at DATETIME NULL,
|
||||
PRIMARY KEY (id),
|
||||
KEY de (created_at, id))`)
|
||||
execSQL(t, db, `INSERT INTO it_estimate (id, v, created_at) VALUES
|
||||
(1,'a','2025-01-01 01:00:00'),
|
||||
(2,'b','2025-01-01 02:00:00'),
|
||||
(3,'c','2025-01-02 01:00:00'),
|
||||
(4,'d','2025-01-03 01:00:00'),
|
||||
(5,'e','2025-01-03 02:00:00')`)
|
||||
|
||||
out := t.TempDir()
|
||||
errOut, err := runCLIStderr(t, context.Background(), baseCLI(dsn, "it_estimate", out, "--progress", "1")...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(errOut, "estimate=key") {
|
||||
t.Fatalf("primary key estimate missing: %q", errOut)
|
||||
}
|
||||
|
||||
out = t.TempDir()
|
||||
errOut, err = runCLIStderr(t, context.Background(),
|
||||
baseCLI(dsn, "it_estimate", out, "--progress", "1", "--shard-by", "date", "--shard-column", "created_at")...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(errOut, "estimate=day") {
|
||||
t.Fatalf("day estimate missing: %q", errOut)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationImportDump(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_import_src")
|
||||
execSQL(t, db, "CREATE TABLE it_import_src (id BIGINT NOT NULL, v VARCHAR(64), PRIMARY KEY (id))")
|
||||
execSQL(t, db, `INSERT INTO it_import_src (id, v) VALUES
|
||||
(1,'plain'), (2,'tab\there'), (3,'line\nbreak'), (4,'back\\slash')`)
|
||||
|
||||
for _, comp := range []string{"none", "gzip", "zstd", "xz"} {
|
||||
out := t.TempDir()
|
||||
args := baseCLI(dsn, "it_import_src", out, "--format", "dump", "--compress", comp)
|
||||
if _, err := runCLI(t, context.Background(), args...); err != nil {
|
||||
t.Fatalf("%s: %v", comp, err)
|
||||
}
|
||||
files := listFiles(t, out, "*.dump*")
|
||||
if len(files) != 1 {
|
||||
t.Fatalf("%s: files %v", comp, files)
|
||||
}
|
||||
loadPath := files[0]
|
||||
if comp != "none" {
|
||||
loadPath = filepath.Join(t.TempDir(), "decompressed.dump")
|
||||
if err := decompressFile(files[0], loadPath, comp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_import_dst")
|
||||
execSQL(t, db, "CREATE TABLE it_import_dst (id BIGINT NOT NULL, v VARCHAR(64), PRIMARY KEY (id))")
|
||||
loadLocal(t, dsn, loadPath, "it_import_dst")
|
||||
var n int
|
||||
if err := db.QueryRow("SELECT COUNT(*) FROM it_import_dst").Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 4 {
|
||||
t.Fatalf("%s: loaded %d rows", comp, n)
|
||||
}
|
||||
var v string
|
||||
if err := db.QueryRow("SELECT v FROM it_import_dst WHERE id=3").Scan(&v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v != "line\nbreak" {
|
||||
t.Fatalf("%s: round trip mismatch %q", comp, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func decompressFile(src, dst, comp string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
var r io.Reader
|
||||
switch comp {
|
||||
case "gzip":
|
||||
gr, err := gzip.NewReader(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gr.Close()
|
||||
r = gr
|
||||
case "zstd":
|
||||
zr, err := zstd.NewReader(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer zr.Close()
|
||||
r = zr
|
||||
case "xz":
|
||||
xr, err := xz.NewReader(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r = xr
|
||||
default:
|
||||
r = in
|
||||
}
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
_, err = io.Copy(out, r)
|
||||
return err
|
||||
}
|
||||
|
||||
func loadLocal(t *testing.T, dsn, path, table string) {
|
||||
t.Helper()
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.AllowAllFiles = true
|
||||
db, err := sql.Open("mysql", cfg.FormatDSN())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
p := filepath.ToSlash(path)
|
||||
p = strings.ReplaceAll(p, `'`, `''`)
|
||||
q := fmt.Sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE %s", p, table)
|
||||
if _, err := db.Exec(q); err != nil {
|
||||
t.Fatalf("load data: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationCSVAndJSONL(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_fmt")
|
||||
execSQL(t, db, `CREATE TABLE it_fmt (
|
||||
id BIGINT NOT NULL, name VARCHAR(64), price DECIMAL(10,2), created DATETIME,
|
||||
data BLOB, PRIMARY KEY (id))`)
|
||||
execSQL(t, db, `INSERT INTO it_fmt (id, name, price, created, data) VALUES
|
||||
(1,'a,\"b\"',12.30,'2024-01-02 03:04:05',X'6869'), (2,'',NULL,NULL,NULL)`)
|
||||
|
||||
// CSV
|
||||
out := t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_fmt", out, "--format", "csv")...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files := listFiles(t, out, "*.csv")
|
||||
if len(files) != 1 {
|
||||
t.Fatalf("csv files %v", files)
|
||||
}
|
||||
f, err := os.Open(files[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recs, err := csv.NewReader(f).ReadAll()
|
||||
f.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(recs) != 3 {
|
||||
t.Fatalf("csv records %d", len(recs))
|
||||
}
|
||||
if recs[0][0] != "id" || recs[0][1] != "name" {
|
||||
t.Fatalf("header %v", recs[0])
|
||||
}
|
||||
if recs[1][1] != `a,"b"` || recs[1][2] != "12.30" {
|
||||
t.Fatalf("row %v", recs[1])
|
||||
}
|
||||
if recs[2][2] != `\N` {
|
||||
t.Fatalf("NULL decimal %q", recs[2][2])
|
||||
}
|
||||
|
||||
// JSONL
|
||||
out = t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_fmt", out, "--format", "jsonl")...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files = listFiles(t, out, "*.jsonl")
|
||||
var shardFiles []string
|
||||
for _, f := range files {
|
||||
if filepath.Base(f) != "manifest.jsonl" {
|
||||
shardFiles = append(shardFiles, f)
|
||||
}
|
||||
}
|
||||
files = shardFiles
|
||||
if len(files) != 1 {
|
||||
t.Fatalf("jsonl files %v", files)
|
||||
}
|
||||
b, err := os.ReadFile(files[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
|
||||
if len(lines) != 2 {
|
||||
t.Fatalf("jsonl lines %d", len(lines))
|
||||
}
|
||||
var obj map[string]any
|
||||
if err := json.Unmarshal([]byte(lines[0]), &obj); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if obj["price"] != "12.30" {
|
||||
t.Fatalf("price %v", obj["price"])
|
||||
}
|
||||
if obj["created"] != "2024-01-02T03:04:05Z" {
|
||||
t.Fatalf("created %v", obj["created"])
|
||||
}
|
||||
if obj["data"] != "aGk=" {
|
||||
t.Fatalf("data %v", obj["data"])
|
||||
}
|
||||
var second map[string]any
|
||||
if err := json.Unmarshal([]byte(lines[1]), &second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second["price"] != nil || second["created"] != nil {
|
||||
t.Fatalf("null mapping %v", second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationPurgeAndDest(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_purge_src")
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_purge_dst")
|
||||
execSQL(t, db, "CREATE TABLE it_purge_src (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
||||
execSQL(t, db, "CREATE TABLE it_purge_dst (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
||||
execSQL(t, db, "INSERT INTO it_purge_src (id, v) VALUES (1,'a'),(2,'b'),(3,'c')")
|
||||
|
||||
out := t.TempDir()
|
||||
args := []string{
|
||||
"--source", dsn, "--table", "it_purge_src", "--where", "1=1",
|
||||
"--file", out, "--dest", dsn, "--dest-table", "it_purge_dst",
|
||||
"--purge", "--chunk-size", "2",
|
||||
}
|
||||
if _, err := runCLI(t, context.Background(), args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var srcCount, dstCount int
|
||||
db.QueryRow("SELECT COUNT(*) FROM it_purge_src").Scan(&srcCount)
|
||||
db.QueryRow("SELECT COUNT(*) FROM it_purge_dst").Scan(&dstCount)
|
||||
if srcCount != 0 || dstCount != 3 {
|
||||
t.Fatalf("src=%d dst=%d", srcCount, dstCount)
|
||||
}
|
||||
if len(listFiles(t, out, "*.dump")) != 1 {
|
||||
t.Fatal("expected one shard")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationCheckpointConfigMismatch(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_mismatch")
|
||||
execSQL(t, db, "CREATE TABLE it_mismatch (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
||||
execSQL(t, db, "INSERT INTO it_mismatch (id, v) VALUES (1,'a'),(2,'b')")
|
||||
out := t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_mismatch", out)...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Remove the complete flag so resume logic runs.
|
||||
stPath := filepath.Join(out, "state.json")
|
||||
b, _ := os.ReadFile(stPath)
|
||||
b = bytes.Replace(b, []byte(`"complete": true`), []byte(`"complete": false`), 1)
|
||||
if err := os.WriteFile(stPath, b, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
args := baseCLI(dsn, "it_mismatch", out)
|
||||
for i := range args {
|
||||
if args[i] == "1=1" {
|
||||
args[i] = "id > 0"
|
||||
}
|
||||
}
|
||||
if _, err := runCLI(t, context.Background(), args...); err == nil {
|
||||
t.Fatal("expected config mismatch error")
|
||||
} else if !strings.Contains(err.Error(), "different configuration") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
args = append(args, "--force-resume")
|
||||
if _, err := runCLI(t, context.Background(), args...); err != nil {
|
||||
t.Fatalf("force resume: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/cli"
|
||||
"archer/internal/row"
|
||||
"archer/internal/schema"
|
||||
)
|
||||
|
||||
// probeResult is everything learned about the source table needed to plan the
|
||||
// scan and the encoders.
|
||||
type probeResult struct {
|
||||
table *schema.Table
|
||||
key schema.KeyInfo
|
||||
shard *schema.ShardSpec
|
||||
path string // pk | A | B
|
||||
joint *schema.Index
|
||||
single *schema.Index
|
||||
selectCols []string
|
||||
selectMeta []row.Column
|
||||
keyIdx []int
|
||||
shardIdx int
|
||||
outCols []row.Column
|
||||
outIdx []int
|
||||
}
|
||||
|
||||
func probeTable(ctx context.Context, db *sql.DB, cfg *cli.Config, dbName string, loc *time.Location) (*probeResult, error) {
|
||||
table, err := schema.Load(ctx, db, dbName, cfg.Table)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
key, err := table.ChooseKey(cfg.Index)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
outCols, err := table.OutputColumns(cfg.Columns)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
p := &probeResult{table: table, key: key, path: "pk", shardIdx: -1, outCols: outCols}
|
||||
|
||||
if cfg.IsDateShard() {
|
||||
spec, err := schema.ResolveShardColumn(ctx, db, table, cfg.ShardColumn, cfg.ShardColumnType, cfg.Where, loc)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
p.shard = spec
|
||||
prefix := append([]string{spec.Column}, key.Columns...)
|
||||
p.joint = table.FindIndexPrefix(prefix)
|
||||
p.single = table.FindIndexPrefix([]string{spec.Column})
|
||||
switch {
|
||||
case p.joint != nil:
|
||||
p.path = "A"
|
||||
case p.single != nil:
|
||||
p.path = "B"
|
||||
case cfg.AllowFullScan:
|
||||
p.path = "B"
|
||||
default:
|
||||
return nil, archererr.Newf(archererr.KindValidation,
|
||||
"no usable index on %s for date sharding\n joint: create an index on (%s) for keyset chunks\n or: create an index on (%s) for day-at-a-time streaming\n or: pass --allow-full-scan to accept a full table scan\n%s",
|
||||
table.Name, strings.Join(prefix, ", "), spec.Column, describeIndexes(table))
|
||||
}
|
||||
}
|
||||
|
||||
needed := map[string]bool{}
|
||||
for _, c := range outCols {
|
||||
needed[strings.ToLower(c.Name)] = true
|
||||
}
|
||||
for _, c := range key.Columns {
|
||||
needed[strings.ToLower(c)] = true
|
||||
}
|
||||
if p.shard != nil {
|
||||
needed[strings.ToLower(p.shard.Column)] = true
|
||||
}
|
||||
for _, c := range table.Columns {
|
||||
if needed[strings.ToLower(c.Name)] {
|
||||
p.selectCols = append(p.selectCols, c.Name)
|
||||
p.selectMeta = append(p.selectMeta, row.Column{Name: c.Name, Type: c.DataType})
|
||||
}
|
||||
}
|
||||
pos := map[string]int{}
|
||||
for i, name := range p.selectCols {
|
||||
pos[name] = i
|
||||
}
|
||||
p.keyIdx = make([]int, len(key.Columns))
|
||||
for i, c := range key.Columns {
|
||||
idx, ok := pos[c]
|
||||
if !ok {
|
||||
return nil, archererr.Newf(archererr.KindValidation, "internal: key column %s missing from select list", c)
|
||||
}
|
||||
p.keyIdx[i] = idx
|
||||
}
|
||||
if p.shard != nil {
|
||||
idx, ok := pos[p.shard.Column]
|
||||
if !ok {
|
||||
return nil, archererr.Newf(archererr.KindValidation, "internal: shard column %s missing from select list", p.shard.Column)
|
||||
}
|
||||
p.shardIdx = idx
|
||||
}
|
||||
p.outIdx = make([]int, len(outCols))
|
||||
for i, c := range outCols {
|
||||
p.outIdx[i] = pos[c.Name]
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func describeIndexes(t *schema.Table) string {
|
||||
if len(t.Indexes) == 0 {
|
||||
return " indexes: (none)"
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString(" indexes:\n")
|
||||
for _, ix := range t.Indexes {
|
||||
kind := "index"
|
||||
if ix.Name == "PRIMARY" {
|
||||
kind = "primary key"
|
||||
} else if ix.Unique {
|
||||
kind = "unique index"
|
||||
}
|
||||
fmt.Fprintf(&b, " - %s (%s) %s\n", ix.Name, strings.Join(ix.Columns, ", "), kind)
|
||||
}
|
||||
return strings.TrimRight(b.String(), "\n")
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"archer/internal/checkpoint"
|
||||
"archer/internal/cli"
|
||||
)
|
||||
|
||||
func TestStateConfigStoresRedactedDSN(t *testing.T) {
|
||||
const source = "u:s3cr3t@tcp(h:3306)/db?charset=utf8mb4"
|
||||
cfg, err := cli.Parse([]string{"--source", source, "--table", "t", "--where", "id > 0", "--file", "out"}, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hash, fields, err := cfg.Fingerprint()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
if err := checkpoint.Save(dir, &checkpoint.State{ConfigHash: hash, Config: fields}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st, err := checkpoint.Load(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
want := redactDSN(source)
|
||||
if st.Config["source"] != want {
|
||||
t.Fatalf("config.source = %q, want %q", st.Config["source"], want)
|
||||
}
|
||||
raw, err := os.ReadFile(checkpoint.Path(dir))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bytes.Contains(raw, []byte("s3cr3t")) {
|
||||
t.Fatal("state.json must not contain the password")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user