Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,705 @@
|
||||
// Package app wires the CLI, schema probe, scanners, shard writer and
|
||||
// database targets into one run.
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/checkpoint"
|
||||
"archer/internal/cli"
|
||||
"archer/internal/lag"
|
||||
"archer/internal/progress"
|
||||
"archer/internal/purge"
|
||||
"archer/internal/row"
|
||||
"archer/internal/scan"
|
||||
"archer/internal/shard"
|
||||
"archer/internal/sink"
|
||||
)
|
||||
|
||||
// Stats collects counters and phase timings for --statistics.
|
||||
type Stats struct {
|
||||
Started time.Time
|
||||
RowsRead int64
|
||||
RowsCommitted int64
|
||||
RowsInserted int64
|
||||
RowsDeleted int64
|
||||
ScanDur time.Duration
|
||||
FileDur time.Duration
|
||||
DBDur time.Duration
|
||||
CheckpointDur time.Duration
|
||||
Shards int64
|
||||
Bytes int64
|
||||
}
|
||||
|
||||
// App holds a run's state.
|
||||
type App struct {
|
||||
cfg *cli.Config
|
||||
srcDB *sql.DB
|
||||
destDB *sql.DB
|
||||
srcCfg *mysql.Config
|
||||
destCfg *mysql.Config
|
||||
|
||||
loc *time.Location
|
||||
probe *probeResult
|
||||
cfgHash string
|
||||
cfgFields map[string]string
|
||||
|
||||
scanner scan.Scanner
|
||||
fileSink *sink.File
|
||||
writer *shard.Writer
|
||||
manifest *shard.Manifest
|
||||
destSink *sink.DB
|
||||
purger *purge.Purger
|
||||
lagChecker *lag.Checker
|
||||
|
||||
sameInstance bool
|
||||
resume *scan.Resume
|
||||
alreadyComplete bool
|
||||
|
||||
st *checkpoint.State
|
||||
stats Stats
|
||||
|
||||
reporter *progress.Reporter
|
||||
estimate *progress.Estimate
|
||||
baseRows int64
|
||||
}
|
||||
|
||||
// Run executes one archiving run.
|
||||
func Run(ctx context.Context, cfg *cli.Config) error {
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
a := &App{cfg: cfg, stats: Stats{Started: time.Now()}}
|
||||
defer a.close()
|
||||
if !cfg.DryRun {
|
||||
a.startReporter()
|
||||
defer a.reporter.Stop()
|
||||
}
|
||||
|
||||
if err := a.setupSource(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.DryRun {
|
||||
return a.printDryRun(os.Stdout)
|
||||
}
|
||||
if err := a.prepare(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if a.alreadyComplete {
|
||||
fmt.Fprintf(os.Stdout, "archer: output in %s is already complete; nothing to do\n", cfg.FileDir)
|
||||
return nil
|
||||
}
|
||||
return a.execute(ctx)
|
||||
}
|
||||
|
||||
func (a *App) startReporter() {
|
||||
tty := progress.IsTerminal(os.Stderr)
|
||||
enabled := a.cfg.Progress.Enabled(tty)
|
||||
a.reporter = progress.New(progress.Options{
|
||||
Out: os.Stderr,
|
||||
Enabled: enabled,
|
||||
Live: enabled && tty,
|
||||
Interval: a.cfg.ProgressInterval,
|
||||
RowEvery: a.cfg.Progress.Rows,
|
||||
})
|
||||
a.reporter.Start()
|
||||
}
|
||||
|
||||
func (a *App) phase(name, reason string) {
|
||||
if a.reporter != nil {
|
||||
a.reporter.Phase(name, reason)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) message(format string, args ...any) {
|
||||
if a.reporter != nil {
|
||||
a.reporter.Message(format, args...)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
||||
}
|
||||
|
||||
func (a *App) setupSource(ctx context.Context) error {
|
||||
a.phase("probe", "")
|
||||
loc, err := time.LoadLocation(a.cfg.ShardTimeZone)
|
||||
if err != nil {
|
||||
return archererr.Newf(archererr.KindConfig, "invalid --shard-time-zone: %v", err)
|
||||
}
|
||||
a.loc = loc
|
||||
srcCfg, err := a.cfg.SourceConfig()
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConfig, err)
|
||||
}
|
||||
if srcCfg.DBName == "" {
|
||||
return archererr.Newf(archererr.KindConfig, "--source must include a database name")
|
||||
}
|
||||
a.srcCfg = srcCfg
|
||||
a.srcDB, err = openDB(srcCfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.probe, err = probeTable(ctx, a.srcDB, a.cfg, srcCfg.DBName, loc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.cfgHash, a.cfgFields, err = a.cfg.Fingerprint()
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConfig, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) prepare(ctx context.Context) error {
|
||||
a.phase("resume", "")
|
||||
if err := a.openDest(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if a.cfg.Purge {
|
||||
autoInc := false
|
||||
if len(a.probe.key.Columns) == 1 {
|
||||
if c, ok := a.probe.table.Column(a.probe.key.Columns[0]); ok {
|
||||
autoInc = c.IsAutoIncrement()
|
||||
}
|
||||
}
|
||||
a.purger = purge.New(a.srcDB, a.srcCfg.DBName, a.cfg.Table, a.probe.key.Columns,
|
||||
a.cfg.Where, a.cfg.BulkDelete, a.cfg.SafeAutoIncrement, autoInc)
|
||||
}
|
||||
if a.cfg.CheckReplicaLag != "" {
|
||||
c, err := lag.Open(a.cfg.CheckReplicaLag, a.cfg.MaxLag, a.cfg.CheckInterval)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConnection, err)
|
||||
}
|
||||
c.OnWait = func(reason string) {
|
||||
a.phase("lag-wait", reason)
|
||||
}
|
||||
a.lagChecker = c
|
||||
}
|
||||
if a.cfg.HasFile() {
|
||||
if err := a.prepareFile(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if a.alreadyComplete {
|
||||
return nil
|
||||
}
|
||||
a.phase("estimate", "")
|
||||
a.estimate = a.queryEstimate(ctx)
|
||||
if a.reporter != nil {
|
||||
a.reporter.SetEstimate(a.estimate)
|
||||
}
|
||||
return a.openScanner(ctx)
|
||||
}
|
||||
|
||||
func (a *App) openDest(ctx context.Context) error {
|
||||
if a.cfg.DestDSN == "" {
|
||||
return nil
|
||||
}
|
||||
destCfg, err := a.cfg.DestConfig()
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConfig, err)
|
||||
}
|
||||
if destCfg.DBName == "" {
|
||||
return archererr.Newf(archererr.KindConfig, "--dest must include a database name")
|
||||
}
|
||||
a.destCfg = destCfg
|
||||
a.sameInstance = cli.SameInstance(a.srcCfg, destCfg)
|
||||
a.destDB, err = openDB(destCfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names := make([]string, len(a.probe.outCols))
|
||||
for i, c := range a.probe.outCols {
|
||||
names[i] = c.Name
|
||||
}
|
||||
s, err := sink.NewDB(ctx, a.destDB, destCfg.DBName, a.cfg.DestTable, names, a.cfg.InsertMode, a.probe.outIdx)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
a.destSink = s
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) prepareFile(ctx context.Context) error {
|
||||
if err := os.MkdirAll(a.cfg.FileDir, 0o755); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("create output directory: %w", err))
|
||||
}
|
||||
comp, err := shard.NewCompressor(a.cfg.Compress, a.cfg.CompressLevel)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConfig, err)
|
||||
}
|
||||
man, err := shard.OpenManifest(a.cfg.FileDir)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
a.manifest = man
|
||||
maxSeq, err := man.Adopt(a.cfg.FileDir, a.cfg.Format, comp)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
|
||||
st, err := checkpoint.Load(a.cfg.FileDir)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
startBucket, startSeq := "", int64(0)
|
||||
if st != nil {
|
||||
if st.ConfigHash != a.cfgHash {
|
||||
if !a.cfg.ForceResume {
|
||||
diffs := cli.DiffFields(st.Config, a.cfgFields)
|
||||
if len(diffs) == 0 {
|
||||
diffs = []string{"fingerprint changed but no field differences were recorded"}
|
||||
}
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"checkpoint in %s was created with a different configuration:\n %s\nre-run with --force-resume to ignore this",
|
||||
a.cfg.FileDir, strings.Join(diffs, "\n "))
|
||||
}
|
||||
a.message("warning: --force-resume ignores the changed configuration fingerprint")
|
||||
}
|
||||
if st.Complete {
|
||||
a.alreadyComplete = true
|
||||
a.st = st
|
||||
return nil
|
||||
}
|
||||
a.st = st
|
||||
a.baseRows = st.RowsDone
|
||||
a.resume = &scan.Resume{Phase: st.Cursor.Phase, Key: st.Cursor.Key, DayStart: st.Cursor.DayStart}
|
||||
startBucket, startSeq = st.LastBucket, st.LastShardSeq
|
||||
} else {
|
||||
a.st = &checkpoint.State{RunID: newRunID(), ShardBy: a.cfg.ShardBy}
|
||||
}
|
||||
if a.reporter != nil {
|
||||
a.reporter.SetBase(a.baseRows)
|
||||
}
|
||||
if maxSeq[startBucket] > startSeq {
|
||||
startSeq = maxSeq[startBucket]
|
||||
}
|
||||
|
||||
var keep string
|
||||
var resumeMeta *shard.PartialMeta
|
||||
if a.cfg.Purge {
|
||||
if m, err := shard.LoadPartialMeta(a.cfg.FileDir, startBucket, startSeq+1); err != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, err)
|
||||
} else if m != nil {
|
||||
resumeMeta = m
|
||||
keep = shard.PartialPath(a.cfg.FileDir, startBucket, startSeq+1)
|
||||
}
|
||||
}
|
||||
if err := shard.CleanupPartials(a.cfg.FileDir, keep); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
|
||||
w, err := shard.Open(shard.Options{
|
||||
Dir: a.cfg.FileDir,
|
||||
Mode: a.cfg.ShardBy,
|
||||
ShardRows: int64(a.cfg.ShardRows),
|
||||
Format: a.cfg.Format,
|
||||
Compressor: comp,
|
||||
Columns: a.probe.selectMeta,
|
||||
OutIdx: a.probe.outIdx,
|
||||
CSVNull: a.cfg.CSVNull,
|
||||
StartBucket: startBucket,
|
||||
StartSeq: startSeq,
|
||||
}, man)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
if resumeMeta != nil {
|
||||
if err := w.Resume(resumeMeta); err != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
}
|
||||
a.writer = w
|
||||
a.fileSink = &sink.File{W: w}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) openScanner(ctx context.Context) error {
|
||||
opts := scan.Options{
|
||||
DB: a.srcDB,
|
||||
DBName: a.srcCfg.DBName,
|
||||
Table: a.cfg.Table,
|
||||
Where: a.cfg.Where,
|
||||
SelectCols: a.probe.selectCols,
|
||||
KeyCols: a.probe.key.Columns,
|
||||
KeyIdx: a.probe.keyIdx,
|
||||
ChunkSize: a.cfg.ChunkSize,
|
||||
Shard: a.probe.shard,
|
||||
ShardIdx: a.probe.shardIdx,
|
||||
ShardLoc: a.loc,
|
||||
ZeroDate: a.cfg.ZeroDate,
|
||||
Resume: a.resume,
|
||||
}
|
||||
switch a.probe.path {
|
||||
case "A":
|
||||
a.scanner = scan.NewDateA(opts)
|
||||
case "B":
|
||||
s, err := scan.NewDateB(ctx, opts)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConnection, err)
|
||||
}
|
||||
a.scanner = s
|
||||
default:
|
||||
a.scanner = scan.NewPK(opts)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) execute(ctx context.Context) error {
|
||||
if a.cfg.HasFile() {
|
||||
a.message("note: file output is at-least-once; a crash and rerun may duplicate rows in shard files")
|
||||
}
|
||||
if a.cfg.Purge {
|
||||
a.message("warning: --purge deletes archived rows from the source table")
|
||||
}
|
||||
if a.cfg.BulkDelete {
|
||||
a.message("warning: --bulk-delete removes all rows in the scanned key range that match --where, including concurrent inserts")
|
||||
}
|
||||
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return a.interrupt(err)
|
||||
}
|
||||
if a.lagChecker != nil {
|
||||
if err := a.lagChecker.Wait(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return a.interrupt(err)
|
||||
}
|
||||
return archererr.Wrap(archererr.KindConnection, err)
|
||||
}
|
||||
}
|
||||
a.phase("scan", "")
|
||||
start := time.Now()
|
||||
chunk, err := a.scanner.Next(ctx)
|
||||
a.stats.ScanDur += time.Since(start)
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return a.interrupt(err)
|
||||
}
|
||||
return archererr.Wrap(archererr.KindOther, fmt.Errorf("scan: %w", err))
|
||||
}
|
||||
if err := a.processChunk(ctx, chunk); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return a.interrupt(err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
a.stats.RowsRead += int64(chunk.Len())
|
||||
a.publishProgress(chunk)
|
||||
}
|
||||
return a.finish()
|
||||
}
|
||||
|
||||
func (a *App) processChunk(ctx context.Context, chunk *row.Chunk) error {
|
||||
fileStart := time.Now()
|
||||
if a.fileSink != nil {
|
||||
a.phase("file", "")
|
||||
if err := a.fileSink.WriteChunk(ctx, chunk); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
if a.cfg.Purge {
|
||||
if err := a.fileSink.Checkpoint(); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
a.stats.FileDur += time.Since(fileStart)
|
||||
|
||||
dbStart := time.Now()
|
||||
if a.destSink != nil || a.purger != nil {
|
||||
a.phase("db", "")
|
||||
}
|
||||
if err := a.runDB(ctx, chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
a.stats.DBDur += time.Since(dbStart)
|
||||
|
||||
if a.fileSink != nil && !a.cfg.Purge {
|
||||
if err := a.fileSink.Flush(ctx); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
}
|
||||
|
||||
a.stats.RowsCommitted += int64(chunk.Len())
|
||||
if a.cfg.HasFile() {
|
||||
a.phase("checkpoint", "")
|
||||
}
|
||||
cpStart := time.Now()
|
||||
if err := a.saveState(false); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("save checkpoint: %w", err))
|
||||
}
|
||||
a.stats.CheckpointDur += time.Since(cpStart)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) runDB(ctx context.Context, chunk *row.Chunk) error {
|
||||
attempts := a.cfg.Retries + 1
|
||||
if a.destSink != nil && a.purger != nil && a.sameInstance {
|
||||
err := retry(ctx, attempts, func() error {
|
||||
tx, err := a.destDB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := a.destSink.InsertChunk(ctx, tx, chunk)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
d, err := a.purger.DeleteChunk(ctx, tx, chunk)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
a.stats.RowsInserted += n
|
||||
a.stats.RowsDeleted += d
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if a.destSink != nil {
|
||||
err := retry(ctx, attempts, func() error {
|
||||
tx, err := a.destDB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := a.destSink.InsertChunk(ctx, tx, chunk)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
a.stats.RowsInserted += n
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
}
|
||||
if a.purger != nil {
|
||||
err := retry(ctx, attempts, func() error {
|
||||
tx, err := a.srcDB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d, err := a.purger.DeleteChunk(ctx, tx, chunk)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
a.stats.RowsDeleted += d
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func retry(ctx context.Context, attempts int, fn func() error) error {
|
||||
var err error
|
||||
for i := 0; i < attempts; i++ {
|
||||
if i > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(time.Duration(i) * time.Second):
|
||||
}
|
||||
}
|
||||
if err = fn(); err == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *App) saveState(complete bool) error {
|
||||
if a.cfg.FileDir == "" || a.st == nil || a.alreadyComplete {
|
||||
return nil
|
||||
}
|
||||
st := a.st
|
||||
st.ConfigHash = a.cfgHash
|
||||
st.Config = a.cfgFields
|
||||
st.ShardBy = a.cfg.ShardBy
|
||||
st.RowsDone = a.baseRows + a.stats.RowsCommitted
|
||||
if a.scanner != nil {
|
||||
cs := a.scanner.ResumeState()
|
||||
st.Cursor = checkpoint.Cursor{Phase: cs.Phase, Key: cs.Key, DayStart: cs.DayStart}
|
||||
}
|
||||
if a.writer != nil {
|
||||
st.LastBucket, st.LastShardSeq = a.writer.StatePosition()
|
||||
}
|
||||
st.Complete = complete
|
||||
return checkpoint.Save(a.cfg.FileDir, st)
|
||||
}
|
||||
|
||||
func (a *App) finish() error {
|
||||
a.phase("finalize", "")
|
||||
if a.writer != nil {
|
||||
if err := a.writer.Close(); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
var shards, bytes int64
|
||||
_, shards, bytes = a.writer.Stats()
|
||||
a.stats.Shards, a.stats.Bytes = shards, bytes
|
||||
}
|
||||
if err := a.saveState(true); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("save checkpoint: %w", err))
|
||||
}
|
||||
a.printStats()
|
||||
if a.reporter != nil {
|
||||
if a.writer != nil {
|
||||
a.reporter.SetShards(a.stats.Shards)
|
||||
a.reporter.SetBytes(a.stats.Bytes)
|
||||
}
|
||||
a.reporter.Done()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) interrupt(cause error) error {
|
||||
if a.reporter != nil {
|
||||
a.reporter.Interrupted()
|
||||
}
|
||||
if a.writer != nil {
|
||||
if err := a.writer.SealCurrent(); err != nil {
|
||||
a.message("warning: could not seal current shard: %v", err)
|
||||
}
|
||||
}
|
||||
if err := a.saveState(false); err != nil {
|
||||
a.message("warning: could not save checkpoint: %v", err)
|
||||
}
|
||||
return archererr.Wrap(archererr.KindInterrupted, cause)
|
||||
}
|
||||
|
||||
func (a *App) publishProgress(chunk *row.Chunk) {
|
||||
if a.reporter == nil {
|
||||
return
|
||||
}
|
||||
if a.estimate != nil {
|
||||
switch a.estimate.Kind {
|
||||
case "day":
|
||||
if len(chunk.Buckets) > 0 {
|
||||
b := chunk.Buckets[len(chunk.Buckets)-1]
|
||||
if b != "" && b != "_null" && !strings.HasPrefix(b, "0000-") {
|
||||
if t, err := time.ParseInLocation("2006-01-02", b, a.loc); err == nil {
|
||||
a.reporter.SetPosition(float64(t.Unix()))
|
||||
}
|
||||
}
|
||||
}
|
||||
case "key":
|
||||
if len(chunk.LastKey) == 1 && !chunk.LastKey[0].Null {
|
||||
if n, err := strconv.ParseInt(chunk.LastKey[0].Val, 10, 64); err == nil {
|
||||
a.reporter.SetPosition(float64(n))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if a.writer != nil {
|
||||
a.reporter.SetShard(a.writer.CurrentShard())
|
||||
a.reporter.SetBytes(a.writer.WrittenBytes())
|
||||
}
|
||||
a.reporter.AddRows(int64(chunk.Len()))
|
||||
}
|
||||
|
||||
func (a *App) printStats() {
|
||||
if !a.cfg.Statistics {
|
||||
return
|
||||
}
|
||||
fmt.Fprintln(os.Stdout, "this run:")
|
||||
fmt.Fprintf(os.Stdout, " rows read: %d\n", a.stats.RowsRead)
|
||||
fmt.Fprintf(os.Stdout, " rows committed: %d\n", a.stats.RowsCommitted)
|
||||
fmt.Fprintf(os.Stdout, " rows inserted: %d\n", a.stats.RowsInserted)
|
||||
fmt.Fprintf(os.Stdout, " rows deleted: %d\n", a.stats.RowsDeleted)
|
||||
if a.writer != nil {
|
||||
fmt.Fprintf(os.Stdout, " shards: %d\n", a.stats.Shards)
|
||||
fmt.Fprintf(os.Stdout, " bytes written: %d\n", a.stats.Bytes)
|
||||
}
|
||||
fmt.Fprintf(os.Stdout, " elapsed: %s (scan %s, file %s, db %s, checkpoint %s)\n",
|
||||
time.Since(a.stats.Started).Round(time.Millisecond),
|
||||
a.stats.ScanDur.Round(time.Millisecond), a.stats.FileDur.Round(time.Millisecond),
|
||||
a.stats.DBDur.Round(time.Millisecond), a.stats.CheckpointDur.Round(time.Millisecond))
|
||||
|
||||
fmt.Fprintln(os.Stdout, "cumulative (including prior runs):")
|
||||
fmt.Fprintf(os.Stdout, " rows read: %d\n", a.baseRows+a.stats.RowsRead)
|
||||
fmt.Fprintf(os.Stdout, " rows committed: %d\n", a.baseRows+a.stats.RowsCommitted)
|
||||
if a.writer != nil && a.manifest != nil {
|
||||
var rows, bytes int64
|
||||
entries := a.manifest.Entries()
|
||||
for _, e := range entries {
|
||||
rows += e.Rows
|
||||
bytes += e.Bytes
|
||||
}
|
||||
fmt.Fprintf(os.Stdout, " shards: %d\n", len(entries))
|
||||
fmt.Fprintf(os.Stdout, " rows written: %d\n", rows)
|
||||
fmt.Fprintf(os.Stdout, " bytes written: %d\n", bytes)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) close() {
|
||||
if a.scanner != nil {
|
||||
_ = a.scanner.Close()
|
||||
}
|
||||
if a.writer != nil {
|
||||
_ = a.writer.Abandon()
|
||||
} else if a.manifest != nil {
|
||||
_ = a.manifest.Close()
|
||||
}
|
||||
if a.lagChecker != nil {
|
||||
_ = a.lagChecker.Close()
|
||||
}
|
||||
if a.destDB != nil {
|
||||
_ = a.destDB.Close()
|
||||
}
|
||||
if a.srcDB != nil {
|
||||
_ = a.srcDB.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func openDB(cfg *mysql.Config) (*sql.DB, error) {
|
||||
db, err := sql.Open("mysql", cfg.FormatDSN())
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindConnection, err)
|
||||
}
|
||||
db.SetMaxOpenConns(8)
|
||||
db.SetMaxIdleConns(4)
|
||||
db.SetConnMaxLifetime(time.Hour)
|
||||
if err := db.Ping(); err != nil {
|
||||
db.Close()
|
||||
return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("connect %s: %w", cfg.Addr, err))
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func newRunID() string {
|
||||
b := make([]byte, 8)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return time.Now().UTC().Format("20060102T150405Z")
|
||||
}
|
||||
return time.Now().UTC().Format("20060102T150405Z") + "-" + hex.EncodeToString(b)
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
|
||||
"archer/internal/purge"
|
||||
"archer/internal/shard"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
func (a *App) printDryRun(w io.Writer) error {
|
||||
cfg := a.cfg
|
||||
comp, err := shard.NewCompressor(cfg.Compress, cfg.CompressLevel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(w, "== resolved configuration ==")
|
||||
fmt.Fprintf(w, "source: %s\n", redactDSN(cfg.SourceDSN))
|
||||
if cfg.DestDSN != "" {
|
||||
fmt.Fprintf(w, "dest: %s\n", redactDSN(cfg.DestDSN))
|
||||
}
|
||||
fmt.Fprintf(w, "table: %s\n", cfg.Table)
|
||||
if cfg.DestDSN != "" {
|
||||
fmt.Fprintf(w, "dest table: %s\n", cfg.DestTable)
|
||||
}
|
||||
fmt.Fprintf(w, "where: %s\n", cfg.Where)
|
||||
if len(cfg.Columns) > 0 {
|
||||
fmt.Fprintf(w, "columns: %s\n", strings.Join(cfg.Columns, ", "))
|
||||
} else {
|
||||
fmt.Fprintf(w, "columns: (all)\n")
|
||||
}
|
||||
targets := []string{}
|
||||
if cfg.HasFile() {
|
||||
targets = append(targets, "file="+cfg.FileDir)
|
||||
}
|
||||
if cfg.DestDSN != "" {
|
||||
targets = append(targets, "dest")
|
||||
}
|
||||
if cfg.Purge {
|
||||
targets = append(targets, "purge")
|
||||
}
|
||||
fmt.Fprintf(w, "targets: %s\n", strings.Join(targets, ", "))
|
||||
fmt.Fprintf(w, "chunk size: %d\n", cfg.ChunkSize)
|
||||
fmt.Fprintf(w, "shard by: %s\n", cfg.ShardBy)
|
||||
if cfg.ShardBy != "none" {
|
||||
fmt.Fprintf(w, "shard rows: %d\n", cfg.ShardRows)
|
||||
}
|
||||
if a.probe.shard != nil {
|
||||
fmt.Fprintf(w, "shard column: %s (%s)\n", a.probe.shard.Column, a.probe.shard.Type)
|
||||
fmt.Fprintf(w, "shard time zone: %s\n", cfg.ShardTimeZone)
|
||||
}
|
||||
fmt.Fprintf(w, "format: %s\n", cfg.Format)
|
||||
if comp.Name == "none" {
|
||||
fmt.Fprintf(w, "compress: none\n")
|
||||
} else {
|
||||
fmt.Fprintf(w, "compress: %s (level %d)\n", comp.Name, comp.Level)
|
||||
}
|
||||
if cfg.InsertMode != "" {
|
||||
fmt.Fprintf(w, "insert mode: %s\n", cfg.InsertMode)
|
||||
}
|
||||
fmt.Fprintf(w, "max lag: %s (check interval %s)\n", cfg.MaxLag, cfg.CheckInterval)
|
||||
fmt.Fprintf(w, "retries: %d\n", cfg.Retries)
|
||||
fmt.Fprintf(w, "safe auto inc: %t\n", cfg.SafeAutoIncrement)
|
||||
fmt.Fprintf(w, "config hash: %s\n", a.cfgHash)
|
||||
|
||||
fmt.Fprintln(w, "\n== index probe ==")
|
||||
fmt.Fprintf(w, "scan key: %s (%s)\n", a.probe.key.Name, strings.Join(a.probe.key.Columns, ", "))
|
||||
if a.probe.path == "A" {
|
||||
fmt.Fprintf(w, "scan path: A (keyset chunks using index %s)\n", a.probe.joint.Name)
|
||||
} else if a.probe.path == "B" {
|
||||
if a.probe.single != nil {
|
||||
fmt.Fprintf(w, "scan path: B (day-at-a-time using index %s)\n", a.probe.single.Name)
|
||||
} else {
|
||||
fmt.Fprintln(w, "scan path: B (day-at-a-time, full table scan allowed)")
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintln(w, "scan path: primary key seek")
|
||||
}
|
||||
|
||||
fmt.Fprintln(w, "\n== first SELECT ==")
|
||||
for _, q := range a.dryRunSelects() {
|
||||
fmt.Fprintf(w, " %s\n", q)
|
||||
}
|
||||
|
||||
fmt.Fprintln(w, "\n== write statements ==")
|
||||
if cfg.DestDSN != "" {
|
||||
fmt.Fprintf(w, " %s\n", a.dryRunInsert())
|
||||
}
|
||||
if cfg.Purge {
|
||||
p := purge.New(nil, a.srcCfg.DBName, cfg.Table, a.probe.key.Columns, cfg.Where, cfg.BulkDelete, cfg.SafeAutoIncrement, false)
|
||||
for _, q := range p.DryRunSQL() {
|
||||
fmt.Fprintf(w, " %s\n", q)
|
||||
}
|
||||
}
|
||||
if cfg.HasFile() {
|
||||
fmt.Fprintf(w, " file: %s/[...]<seq>.<pk_lo>-<pk_hi>.%s%s (written as .partial until sealed)\n",
|
||||
cfg.FileDir, cfg.Format, comp.Ext)
|
||||
}
|
||||
fmt.Fprintln(w, "\nnote: file output is at-least-once; a crash and rerun may duplicate rows in shard files")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) dryRunSelects() []string {
|
||||
cols := sqlutil.Columns(a.probe.selectCols)
|
||||
table := sqlutil.Qualified(a.srcCfg.DBName, a.cfg.Table)
|
||||
where := a.cfg.Where
|
||||
key := sqlutil.Columns(a.probe.key.Columns)
|
||||
limit := fmt.Sprintf(" LIMIT %d", a.cfg.ChunkSize)
|
||||
switch a.probe.path {
|
||||
case "A":
|
||||
shard := sqlutil.Ident(a.probe.shard.Column)
|
||||
colsWithShard := sqlutil.Columns(append([]string{a.probe.shard.Column}, a.probe.key.Columns...))
|
||||
return []string{
|
||||
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s IS NULL ORDER BY %s%s", cols, table, where, shard, key, limit),
|
||||
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s IS NOT NULL AND (%s) > (?, ...) ORDER BY %s%s", cols, table, where, shard, colsWithShard, colsWithShard, limit),
|
||||
}
|
||||
case "B":
|
||||
shard := sqlutil.Ident(a.probe.shard.Column)
|
||||
colsWithShard := sqlutil.Columns(append([]string{a.probe.shard.Column}, a.probe.key.Columns...))
|
||||
return []string{
|
||||
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s IS NULL ORDER BY %s", cols, table, where, shard, key),
|
||||
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s >= ? AND %s < ? ORDER BY %s", cols, table, where, shard, shard, colsWithShard),
|
||||
}
|
||||
default:
|
||||
return []string{fmt.Sprintf("SELECT %s FROM %s WHERE (%s) ORDER BY %s%s", cols, table, where, key, limit)}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) dryRunInsert() string {
|
||||
destCfg, err := a.cfg.DestConfig()
|
||||
dbName := ""
|
||||
if err == nil && destCfg != nil {
|
||||
dbName = destCfg.DBName
|
||||
}
|
||||
verb := "INSERT INTO"
|
||||
switch a.cfg.InsertMode {
|
||||
case "ignore":
|
||||
verb = "INSERT IGNORE INTO"
|
||||
case "replace":
|
||||
verb = "REPLACE INTO"
|
||||
}
|
||||
names := make([]string, len(a.probe.outCols))
|
||||
for i, c := range a.probe.outCols {
|
||||
names[i] = c.Name
|
||||
}
|
||||
tuple := sqlutil.ParamTuple(len(names))
|
||||
return fmt.Sprintf("%s %s (%s) VALUES %s, %s, ...",
|
||||
verb, sqlutil.Qualified(dbName, a.cfg.DestTable), sqlutil.Columns(names), tuple, tuple)
|
||||
}
|
||||
|
||||
func redactDSN(dsn string) string {
|
||||
c, err := mysql.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
return "<invalid dsn>"
|
||||
}
|
||||
c.Passwd = ""
|
||||
return c.FormatDSN()
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"archer/internal/estimate"
|
||||
"archer/internal/progress"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
// estimateTimeout bounds the one-off MIN/MAX query at startup.
|
||||
const estimateTimeout = 2 * time.Second
|
||||
|
||||
// estimateSpec decides which cheap position metric this run can expose.
|
||||
func (a *App) estimateSpec() estimate.Spec {
|
||||
if a.cfg.IsDateShard() && a.probe.shard != nil {
|
||||
return estimate.Spec{
|
||||
Kind: "day",
|
||||
Column: a.probe.shard.Column,
|
||||
ShardKind: a.probe.shard.Kind,
|
||||
Unit: a.probe.shard.Unit,
|
||||
Loc: a.loc,
|
||||
}
|
||||
}
|
||||
types := make([]string, len(a.probe.key.Columns))
|
||||
for i, name := range a.probe.key.Columns {
|
||||
c, _ := a.probe.table.Column(name)
|
||||
types[i] = c.DataType
|
||||
}
|
||||
col, ok := estimate.KeyColumn(a.probe.key.Columns, types)
|
||||
if !ok {
|
||||
return estimate.Spec{}
|
||||
}
|
||||
return estimate.Spec{Kind: "key", Column: col}
|
||||
}
|
||||
|
||||
// queryEstimate runs the bounded range query on a dedicated connection and
|
||||
// silently returns nil when the metric does not apply or the query fails.
|
||||
func (a *App) queryEstimate(ctx context.Context) *progress.Estimate {
|
||||
spec := a.estimateSpec()
|
||||
if spec.Kind == "" {
|
||||
return nil
|
||||
}
|
||||
conn, err := a.srcDB.Conn(ctx)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer conn.Close()
|
||||
query := func(qctx context.Context, column string) (sql.NullString, sql.NullString, error) {
|
||||
col := sqlutil.Ident(column)
|
||||
q := fmt.Sprintf("SELECT MIN(%s), MAX(%s) FROM %s WHERE (%s)",
|
||||
col, col, sqlutil.Qualified(a.srcCfg.DBName, a.cfg.Table), a.cfg.Where)
|
||||
if spec.Kind == "day" {
|
||||
q += " AND " + col + " IS NOT NULL"
|
||||
}
|
||||
var minRaw, maxRaw sql.NullString
|
||||
err := conn.QueryRowContext(qctx, q).Scan(&minRaw, &maxRaw)
|
||||
return minRaw, maxRaw, err
|
||||
}
|
||||
return estimate.Bounded(ctx, query, spec, estimateTimeout)
|
||||
}
|
||||
@@ -0,0 +1,918 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/ulikunitz/xz"
|
||||
|
||||
"archer/internal/cli"
|
||||
)
|
||||
|
||||
func testDSN(t *testing.T) string {
|
||||
d := os.Getenv("ARCHER_TEST_DSN")
|
||||
if d == "" {
|
||||
t.Skip("ARCHER_TEST_DSN is not set; skipping MySQL integration test")
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
if dsn := os.Getenv("ARCHER_TEST_DSN"); dsn != "" {
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err == nil && cfg.DBName != "" {
|
||||
dbName := cfg.DBName
|
||||
cfg.DBName = ""
|
||||
db, err := sql.Open("mysql", cfg.FormatDSN())
|
||||
if err == nil {
|
||||
_, _ = db.Exec("CREATE DATABASE IF NOT EXISTS `" + dbName + "`")
|
||||
_, _ = db.Exec("SET GLOBAL local_infile=ON")
|
||||
_, _ = db.Exec("SET GLOBAL sql_mode=''")
|
||||
db.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func openRoot(t *testing.T, dsn string) *sql.DB {
|
||||
t.Helper()
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db, err := sql.Open("mysql", cfg.FormatDSN())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Ping(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
func execSQL(t *testing.T, db *sql.DB, q string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := db.Exec(q, args...); err != nil {
|
||||
t.Fatalf("exec %q: %v", q, err)
|
||||
}
|
||||
}
|
||||
|
||||
var stdoutMu sync.Mutex
|
||||
var stderrMu sync.Mutex
|
||||
|
||||
func captureStdout(fn func() error) (string, error) {
|
||||
stdoutMu.Lock()
|
||||
defer stdoutMu.Unlock()
|
||||
old := os.Stdout
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
os.Stdout = w
|
||||
var buf bytes.Buffer
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
_, _ = io.Copy(&buf, r)
|
||||
close(done)
|
||||
}()
|
||||
runErr := fn()
|
||||
_ = w.Close()
|
||||
os.Stdout = old
|
||||
<-done
|
||||
return buf.String(), runErr
|
||||
}
|
||||
|
||||
func captureStderr(fn func() error) (string, error) {
|
||||
stderrMu.Lock()
|
||||
defer stderrMu.Unlock()
|
||||
old := os.Stderr
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
os.Stderr = w
|
||||
var buf bytes.Buffer
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
_, _ = io.Copy(&buf, r)
|
||||
close(done)
|
||||
}()
|
||||
runErr := fn()
|
||||
_ = w.Close()
|
||||
os.Stderr = old
|
||||
<-done
|
||||
return buf.String(), runErr
|
||||
}
|
||||
|
||||
func runCLI(t *testing.T, ctx context.Context, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
cfg, err := cli.Parse(args, io.Discard)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return captureStdout(func() error { return Run(ctx, cfg) })
|
||||
}
|
||||
|
||||
func runCLIStderr(t *testing.T, ctx context.Context, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
cfg, err := cli.Parse(args, io.Discard)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return captureStderr(func() error { return Run(ctx, cfg) })
|
||||
}
|
||||
|
||||
func baseCLI(dsn, table, out string, extra ...string) []string {
|
||||
args := []string{
|
||||
"--source", dsn,
|
||||
"--table", table,
|
||||
"--where", "1=1",
|
||||
"--file", out,
|
||||
}
|
||||
return append(args, extra...)
|
||||
}
|
||||
|
||||
func listFiles(t *testing.T, dir, pattern string) []string {
|
||||
t.Helper()
|
||||
matches, err := filepath.Glob(filepath.Join(dir, pattern))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sort.Strings(matches)
|
||||
return matches
|
||||
}
|
||||
|
||||
func dumpIDs(t *testing.T, path string) []int {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ids []int
|
||||
for _, line := range strings.Split(strings.TrimRight(string(b), "\n"), "\n") {
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
id, err := strconv.Atoi(strings.SplitN(line, "\t", 2)[0])
|
||||
if err != nil {
|
||||
t.Fatalf("%s: bad line %q", path, line)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func TestIntegrationCountShards(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_count")
|
||||
execSQL(t, db, "CREATE TABLE it_count (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
||||
var b strings.Builder
|
||||
b.WriteString("INSERT INTO it_count (id, v) VALUES ")
|
||||
for i := 1; i <= 25; i++ {
|
||||
if i > 1 {
|
||||
b.WriteString(",")
|
||||
}
|
||||
fmt.Fprintf(&b, "(%d,'v%d')", i, i)
|
||||
}
|
||||
execSQL(t, db, b.String())
|
||||
|
||||
out := t.TempDir()
|
||||
_, err := runCLI(t, context.Background(), baseCLI(dsn, "it_count", out,
|
||||
"--shard-by", "count", "--shard-rows", "10", "--chunk-size", "7", "--format", "dump")...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files := listFiles(t, out, "*.dump")
|
||||
if len(files) != 3 {
|
||||
t.Fatalf("got %d shard files: %v", len(files), files)
|
||||
}
|
||||
wantNames := []string{"000001.1-10.dump", "000002.11-20.dump", "000003.21-25.dump"}
|
||||
var got []int
|
||||
for i, f := range files {
|
||||
if filepath.Base(f) != wantNames[i] {
|
||||
t.Fatalf("file %d = %s want %s", i, filepath.Base(f), wantNames[i])
|
||||
}
|
||||
got = append(got, dumpIDs(t, f)...)
|
||||
}
|
||||
if len(got) != 25 {
|
||||
t.Fatalf("got %d rows", len(got))
|
||||
}
|
||||
for i, id := range got {
|
||||
if id != i+1 {
|
||||
t.Fatalf("row %d = %d, shards are not contiguous", i, id)
|
||||
}
|
||||
}
|
||||
|
||||
entries := readManifest(t, out)
|
||||
if len(entries) != 3 {
|
||||
t.Fatalf("manifest entries %d", len(entries))
|
||||
}
|
||||
for i, e := range entries {
|
||||
if e.Seq != int64(i+1) || e.Rows != int64(len(dumpIDs(t, filepath.Join(out, e.Path)))) {
|
||||
t.Fatalf("entry %d %+v", i, e)
|
||||
}
|
||||
if e.Bytes == 0 || e.SHA == "" {
|
||||
t.Fatalf("entry %d missing bytes/hash", i)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(out, "state.json")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var complete bool
|
||||
st, err := os.ReadFile(filepath.Join(out, "state.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
complete = bytes.Contains(st, []byte(`"complete": true`))
|
||||
if !complete {
|
||||
t.Fatalf("state not complete: %s", st)
|
||||
}
|
||||
}
|
||||
|
||||
type manifestEntry struct {
|
||||
Seq int64 `json:"seq"`
|
||||
Path string `json:"path"`
|
||||
Date string `json:"date"`
|
||||
Rows int64 `json:"rows"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
SHA string `json:"sha256"`
|
||||
PKLo string `json:"pk_lo"`
|
||||
PKHi string `json:"pk_hi"`
|
||||
}
|
||||
|
||||
func readManifest(t *testing.T, dir string) []manifestEntry {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(filepath.Join(dir, "manifest.jsonl"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []manifestEntry
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(b)), "\n") {
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var e manifestEntry
|
||||
if err := json.Unmarshal([]byte(line), &e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestIntegrationDatePathsAgree(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
for _, spec := range []struct{ name, index string }{
|
||||
{"it_date_a", "KEY da (created_at, id)"},
|
||||
{"it_date_b", "KEY db (created_at)"},
|
||||
} {
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS "+spec.name)
|
||||
execSQL(t, db, "CREATE TABLE "+spec.name+` (
|
||||
id BIGINT NOT NULL,
|
||||
v VARCHAR(32),
|
||||
created_at DATETIME NULL,
|
||||
PRIMARY KEY (id),
|
||||
`+spec.index+`)`)
|
||||
execSQL(t, db, `INSERT INTO `+spec.name+` (id, v, created_at) VALUES
|
||||
(1,'a','2025-01-01 10:00:00'),
|
||||
(2,'b','2025-01-01 12:00:00'),
|
||||
(3,'c','2025-01-02 01:00:00'),
|
||||
(4,'d',NULL),
|
||||
(5,'e','2025-01-01 11:00:00'),
|
||||
(6,'f','2025-01-03 00:00:00')`)
|
||||
}
|
||||
outA := t.TempDir()
|
||||
outB := t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_a", outA,
|
||||
"--shard-by", "date", "--shard-column", "created_at", "--format", "dump", "--chunk-size", "2")...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_b", outB,
|
||||
"--shard-by", "date", "--shard-column", "created_at", "--format", "dump", "--chunk-size", "2")...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := readTree(t, outA)
|
||||
b := readTree(t, outB)
|
||||
if len(a) != len(b) {
|
||||
t.Fatalf("file counts differ: %d vs %d (%v / %v)", len(a), len(b), a, b)
|
||||
}
|
||||
for k, va := range a {
|
||||
if b[k] != va {
|
||||
t.Fatalf("%s:\nA=%q\nB=%q", k, va, b[k])
|
||||
}
|
||||
}
|
||||
if _, ok := a["date=_null/000001.4-4.dump"]; !ok {
|
||||
t.Fatalf("NULL row not routed to _null: %v", a)
|
||||
}
|
||||
if len(a) != 4 {
|
||||
t.Fatalf("expected 4 shards (null + 3 days), got %v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func readTree(t *testing.T, dir string) map[string]string {
|
||||
t.Helper()
|
||||
out := map[string]string{}
|
||||
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return err
|
||||
}
|
||||
if !strings.HasSuffix(d.Name(), ".dump") {
|
||||
return nil
|
||||
}
|
||||
rel, _ := filepath.Rel(dir, path)
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out[filepath.ToSlash(rel)] = string(b)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestIntegrationZeroDateAndNull(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_zero")
|
||||
execSQL(t, db, `CREATE TABLE it_zero (
|
||||
id BIGINT NOT NULL,
|
||||
d DATE NULL,
|
||||
PRIMARY KEY (id),
|
||||
KEY dz (d, id))`)
|
||||
execSQL(t, db, `INSERT INTO it_zero (id, d) VALUES
|
||||
(1, NULL), (2, '0000-00-00'), (3, '2024-01-01'), (4, '2024-01-01')`)
|
||||
|
||||
out := t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_zero", out,
|
||||
"--shard-by", "date", "--shard-column", "d")...); err == nil {
|
||||
t.Fatal("expected zero-date error")
|
||||
} else if !strings.Contains(err.Error(), "zero date") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
out = t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_zero", out,
|
||||
"--shard-by", "date", "--shard-column", "d", "--zero-date", "bucket")...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tree := readTree(t, out)
|
||||
for _, want := range []string{
|
||||
"date=_null/000001.1-1.dump",
|
||||
"date=0000-00-00/000001.2-2.dump",
|
||||
"date=2024-01-01/000001.3-4.dump",
|
||||
} {
|
||||
if _, ok := tree[want]; !ok {
|
||||
t.Fatalf("missing %s in %v", want, tree)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationEpochUnits(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_epoch")
|
||||
execSQL(t, db, `CREATE TABLE it_epoch (
|
||||
id BIGINT NOT NULL,
|
||||
ts BIGINT NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
KEY e (ts, id))`)
|
||||
execSQL(t, db, `INSERT INTO it_epoch (id, ts) VALUES
|
||||
(1, 1717200000), (2, 1717200001)`)
|
||||
|
||||
for _, tc := range []struct {
|
||||
unit string
|
||||
vals string
|
||||
}{
|
||||
{"epoch-seconds", "(3, 1717200000), (4, 1717200001)"},
|
||||
{"epoch-milliseconds", "(3, 1717200000000), (4, 1717200001000)"},
|
||||
{"epoch-microseconds", "(3, 1717200000000000), (4, 1717200001000000)"},
|
||||
} {
|
||||
table := "it_epoch_" + strings.Split(tc.unit, "-")[1]
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS "+table)
|
||||
execSQL(t, db, `CREATE TABLE `+table+` (
|
||||
id BIGINT NOT NULL, ts BIGINT NOT NULL,
|
||||
PRIMARY KEY (id), KEY e (ts, id))`)
|
||||
execSQL(t, db, "INSERT INTO "+table+" (id, ts) VALUES "+tc.vals)
|
||||
out := t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, table, out,
|
||||
"--shard-by", "date", "--shard-column", "ts", "--shard-column-type", tc.unit,
|
||||
"--shard-time-zone", "UTC")...); err != nil {
|
||||
t.Fatalf("%s: %v", tc.unit, err)
|
||||
}
|
||||
tree := readTree(t, out)
|
||||
if len(tree) != 1 {
|
||||
t.Fatalf("%s: expected one date bucket, got %v", tc.unit, tree)
|
||||
}
|
||||
for k := range tree {
|
||||
if !strings.Contains(k, "date=2024-06-01") {
|
||||
t.Fatalf("%s: wrong bucket %s", tc.unit, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationNoIndexFailFast(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_noindex")
|
||||
execSQL(t, db, `CREATE TABLE it_noindex (
|
||||
id BIGINT NOT NULL, d DATETIME NULL, PRIMARY KEY (id))`)
|
||||
execSQL(t, db, `INSERT INTO it_noindex (id, d) VALUES (1,'2024-01-01 00:00:00')`)
|
||||
|
||||
out := t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_noindex", out,
|
||||
"--shard-by", "date", "--shard-column", "d")...); err == nil {
|
||||
t.Fatal("expected no-index error")
|
||||
} else if !strings.Contains(err.Error(), "no usable index") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
out = t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_noindex", out,
|
||||
"--shard-by", "date", "--shard-column", "d", "--allow-full-scan")...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(out, "date=2024-01-01")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationDryRunCreatesNothing(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_dry")
|
||||
execSQL(t, db, "CREATE TABLE it_dry (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
||||
execSQL(t, db, "INSERT INTO it_dry (id, v) VALUES (1,'a')")
|
||||
|
||||
out := filepath.Join(t.TempDir(), "never-created")
|
||||
stdout, err := runCLI(t, context.Background(), baseCLI(dsn, "it_dry", out, "--dry-run")...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, statErr := os.Stat(out); !os.IsNotExist(statErr) {
|
||||
t.Fatal("dry-run created the output directory")
|
||||
}
|
||||
if !strings.Contains(stdout, "== first SELECT ==") || !strings.Contains(stdout, "config hash") {
|
||||
t.Fatalf("dry-run output incomplete:\n%s", stdout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationResumeAfterInterrupt(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_resume")
|
||||
execSQL(t, db, "CREATE TABLE it_resume (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
||||
var b strings.Builder
|
||||
b.WriteString("INSERT INTO it_resume (id, v) VALUES ")
|
||||
const total = 20000
|
||||
for i := 1; i <= total; i++ {
|
||||
if i > 1 {
|
||||
b.WriteString(",")
|
||||
}
|
||||
fmt.Fprintf(&b, "(%d,'v')", i)
|
||||
}
|
||||
execSQL(t, db, b.String())
|
||||
|
||||
out := t.TempDir()
|
||||
args := baseCLI(dsn, "it_resume", out, "--shard-by", "count", "--shard-rows", "137",
|
||||
"--chunk-size", "50", "--progress", "100")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
stderrMu.Lock()
|
||||
old := os.Stderr
|
||||
r, w, _ := os.Pipe()
|
||||
os.Stderr = w
|
||||
cfg, err := cli.Parse(args, io.Discard)
|
||||
if err != nil {
|
||||
os.Stderr = old
|
||||
stderrMu.Unlock()
|
||||
t.Fatal(err)
|
||||
}
|
||||
firstErr := make(chan error, 1)
|
||||
go func() { firstErr <- Run(ctx, cfg) }()
|
||||
go func() {
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
n, err := r.Read(buf)
|
||||
if n > 0 && strings.Contains(string(buf[:n]), "progress:") {
|
||||
cancel()
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
err = <-firstErr
|
||||
_ = w.Close()
|
||||
os.Stderr = old
|
||||
stderrMu.Unlock()
|
||||
if err == nil {
|
||||
t.Fatal("first run should have been interrupted")
|
||||
}
|
||||
|
||||
if _, err := runCLI(t, context.Background(), args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
seen := map[int]bool{}
|
||||
for _, f := range listFiles(t, out, "*.dump") {
|
||||
for _, id := range dumpIDs(t, f) {
|
||||
seen[id] = true
|
||||
}
|
||||
}
|
||||
for i := 1; i <= total; i++ {
|
||||
if !seen[i] {
|
||||
t.Fatalf("row %d missing after resume", i)
|
||||
}
|
||||
}
|
||||
st, err := os.ReadFile(filepath.Join(out, "state.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Contains(st, []byte(`"complete": true`)) {
|
||||
t.Fatalf("state not complete: %s", st)
|
||||
}
|
||||
var saved struct {
|
||||
RowsDone int64 `json:"rows_done"`
|
||||
}
|
||||
if err := json.Unmarshal(st, &saved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if saved.RowsDone != total {
|
||||
t.Fatalf("rows_done = %d after resume, want cumulative %d", saved.RowsDone, total)
|
||||
}
|
||||
if _, err := filepath.Glob(filepath.Join(out, "*.partial")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationProgressModes(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_progress")
|
||||
execSQL(t, db, "CREATE TABLE it_progress (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
||||
execSQL(t, db, "INSERT INTO it_progress (id, v) VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e')")
|
||||
|
||||
for _, extra := range [][]string{
|
||||
nil,
|
||||
{"--progress", "off"},
|
||||
{"--progress", "0"},
|
||||
} {
|
||||
out := t.TempDir()
|
||||
args := baseCLI(dsn, "it_progress", out, extra...)
|
||||
errOut, err := runCLIStderr(t, context.Background(), args...)
|
||||
if err != nil {
|
||||
t.Fatalf("%v: %v", extra, err)
|
||||
}
|
||||
if strings.Contains(errOut, "progress:") || strings.Contains(errOut, "done rows=") {
|
||||
t.Fatalf("%v: unexpected progress output %q", extra, errOut)
|
||||
}
|
||||
}
|
||||
|
||||
out := t.TempDir()
|
||||
errOut, err := runCLIStderr(t, context.Background(), baseCLI(dsn, "it_progress", out, "--progress", "on")...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(errOut, "done rows=5") {
|
||||
t.Fatalf("forced progress missing done summary: %q", errOut)
|
||||
}
|
||||
if strings.Contains(errOut, "\r") {
|
||||
t.Fatalf("redirected progress must not use carriage returns: %q", errOut)
|
||||
}
|
||||
|
||||
out = t.TempDir()
|
||||
errOut, err = runCLIStderr(t, context.Background(), baseCLI(dsn, "it_progress", out, "--progress", "1")...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(errOut, "progress: rows=5") {
|
||||
t.Fatalf("row-threshold progress line missing: %q", errOut)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationEstimateMetrics(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_estimate")
|
||||
execSQL(t, db, `CREATE TABLE it_estimate (
|
||||
id BIGINT NOT NULL,
|
||||
v VARCHAR(32),
|
||||
created_at DATETIME NULL,
|
||||
PRIMARY KEY (id),
|
||||
KEY de (created_at, id))`)
|
||||
execSQL(t, db, `INSERT INTO it_estimate (id, v, created_at) VALUES
|
||||
(1,'a','2025-01-01 01:00:00'),
|
||||
(2,'b','2025-01-01 02:00:00'),
|
||||
(3,'c','2025-01-02 01:00:00'),
|
||||
(4,'d','2025-01-03 01:00:00'),
|
||||
(5,'e','2025-01-03 02:00:00')`)
|
||||
|
||||
out := t.TempDir()
|
||||
errOut, err := runCLIStderr(t, context.Background(), baseCLI(dsn, "it_estimate", out, "--progress", "1")...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(errOut, "estimate=key") {
|
||||
t.Fatalf("primary key estimate missing: %q", errOut)
|
||||
}
|
||||
|
||||
out = t.TempDir()
|
||||
errOut, err = runCLIStderr(t, context.Background(),
|
||||
baseCLI(dsn, "it_estimate", out, "--progress", "1", "--shard-by", "date", "--shard-column", "created_at")...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(errOut, "estimate=day") {
|
||||
t.Fatalf("day estimate missing: %q", errOut)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationImportDump(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_import_src")
|
||||
execSQL(t, db, "CREATE TABLE it_import_src (id BIGINT NOT NULL, v VARCHAR(64), PRIMARY KEY (id))")
|
||||
execSQL(t, db, `INSERT INTO it_import_src (id, v) VALUES
|
||||
(1,'plain'), (2,'tab\there'), (3,'line\nbreak'), (4,'back\\slash')`)
|
||||
|
||||
for _, comp := range []string{"none", "gzip", "zstd", "xz"} {
|
||||
out := t.TempDir()
|
||||
args := baseCLI(dsn, "it_import_src", out, "--format", "dump", "--compress", comp)
|
||||
if _, err := runCLI(t, context.Background(), args...); err != nil {
|
||||
t.Fatalf("%s: %v", comp, err)
|
||||
}
|
||||
files := listFiles(t, out, "*.dump*")
|
||||
if len(files) != 1 {
|
||||
t.Fatalf("%s: files %v", comp, files)
|
||||
}
|
||||
loadPath := files[0]
|
||||
if comp != "none" {
|
||||
loadPath = filepath.Join(t.TempDir(), "decompressed.dump")
|
||||
if err := decompressFile(files[0], loadPath, comp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_import_dst")
|
||||
execSQL(t, db, "CREATE TABLE it_import_dst (id BIGINT NOT NULL, v VARCHAR(64), PRIMARY KEY (id))")
|
||||
loadLocal(t, dsn, loadPath, "it_import_dst")
|
||||
var n int
|
||||
if err := db.QueryRow("SELECT COUNT(*) FROM it_import_dst").Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 4 {
|
||||
t.Fatalf("%s: loaded %d rows", comp, n)
|
||||
}
|
||||
var v string
|
||||
if err := db.QueryRow("SELECT v FROM it_import_dst WHERE id=3").Scan(&v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v != "line\nbreak" {
|
||||
t.Fatalf("%s: round trip mismatch %q", comp, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func decompressFile(src, dst, comp string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
var r io.Reader
|
||||
switch comp {
|
||||
case "gzip":
|
||||
gr, err := gzip.NewReader(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gr.Close()
|
||||
r = gr
|
||||
case "zstd":
|
||||
zr, err := zstd.NewReader(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer zr.Close()
|
||||
r = zr
|
||||
case "xz":
|
||||
xr, err := xz.NewReader(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r = xr
|
||||
default:
|
||||
r = in
|
||||
}
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
_, err = io.Copy(out, r)
|
||||
return err
|
||||
}
|
||||
|
||||
func loadLocal(t *testing.T, dsn, path, table string) {
|
||||
t.Helper()
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.AllowAllFiles = true
|
||||
db, err := sql.Open("mysql", cfg.FormatDSN())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
p := filepath.ToSlash(path)
|
||||
p = strings.ReplaceAll(p, `'`, `''`)
|
||||
q := fmt.Sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE %s", p, table)
|
||||
if _, err := db.Exec(q); err != nil {
|
||||
t.Fatalf("load data: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationCSVAndJSONL(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_fmt")
|
||||
execSQL(t, db, `CREATE TABLE it_fmt (
|
||||
id BIGINT NOT NULL, name VARCHAR(64), price DECIMAL(10,2), created DATETIME,
|
||||
data BLOB, PRIMARY KEY (id))`)
|
||||
execSQL(t, db, `INSERT INTO it_fmt (id, name, price, created, data) VALUES
|
||||
(1,'a,\"b\"',12.30,'2024-01-02 03:04:05',X'6869'), (2,'',NULL,NULL,NULL)`)
|
||||
|
||||
// CSV
|
||||
out := t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_fmt", out, "--format", "csv")...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files := listFiles(t, out, "*.csv")
|
||||
if len(files) != 1 {
|
||||
t.Fatalf("csv files %v", files)
|
||||
}
|
||||
f, err := os.Open(files[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recs, err := csv.NewReader(f).ReadAll()
|
||||
f.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(recs) != 3 {
|
||||
t.Fatalf("csv records %d", len(recs))
|
||||
}
|
||||
if recs[0][0] != "id" || recs[0][1] != "name" {
|
||||
t.Fatalf("header %v", recs[0])
|
||||
}
|
||||
if recs[1][1] != `a,"b"` || recs[1][2] != "12.30" {
|
||||
t.Fatalf("row %v", recs[1])
|
||||
}
|
||||
if recs[2][2] != `\N` {
|
||||
t.Fatalf("NULL decimal %q", recs[2][2])
|
||||
}
|
||||
|
||||
// JSONL
|
||||
out = t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_fmt", out, "--format", "jsonl")...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files = listFiles(t, out, "*.jsonl")
|
||||
var shardFiles []string
|
||||
for _, f := range files {
|
||||
if filepath.Base(f) != "manifest.jsonl" {
|
||||
shardFiles = append(shardFiles, f)
|
||||
}
|
||||
}
|
||||
files = shardFiles
|
||||
if len(files) != 1 {
|
||||
t.Fatalf("jsonl files %v", files)
|
||||
}
|
||||
b, err := os.ReadFile(files[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
|
||||
if len(lines) != 2 {
|
||||
t.Fatalf("jsonl lines %d", len(lines))
|
||||
}
|
||||
var obj map[string]any
|
||||
if err := json.Unmarshal([]byte(lines[0]), &obj); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if obj["price"] != "12.30" {
|
||||
t.Fatalf("price %v", obj["price"])
|
||||
}
|
||||
if obj["created"] != "2024-01-02T03:04:05Z" {
|
||||
t.Fatalf("created %v", obj["created"])
|
||||
}
|
||||
if obj["data"] != "aGk=" {
|
||||
t.Fatalf("data %v", obj["data"])
|
||||
}
|
||||
var second map[string]any
|
||||
if err := json.Unmarshal([]byte(lines[1]), &second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second["price"] != nil || second["created"] != nil {
|
||||
t.Fatalf("null mapping %v", second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationPurgeAndDest(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_purge_src")
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_purge_dst")
|
||||
execSQL(t, db, "CREATE TABLE it_purge_src (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
||||
execSQL(t, db, "CREATE TABLE it_purge_dst (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
||||
execSQL(t, db, "INSERT INTO it_purge_src (id, v) VALUES (1,'a'),(2,'b'),(3,'c')")
|
||||
|
||||
out := t.TempDir()
|
||||
args := []string{
|
||||
"--source", dsn, "--table", "it_purge_src", "--where", "1=1",
|
||||
"--file", out, "--dest", dsn, "--dest-table", "it_purge_dst",
|
||||
"--purge", "--chunk-size", "2",
|
||||
}
|
||||
if _, err := runCLI(t, context.Background(), args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var srcCount, dstCount int
|
||||
db.QueryRow("SELECT COUNT(*) FROM it_purge_src").Scan(&srcCount)
|
||||
db.QueryRow("SELECT COUNT(*) FROM it_purge_dst").Scan(&dstCount)
|
||||
if srcCount != 0 || dstCount != 3 {
|
||||
t.Fatalf("src=%d dst=%d", srcCount, dstCount)
|
||||
}
|
||||
if len(listFiles(t, out, "*.dump")) != 1 {
|
||||
t.Fatal("expected one shard")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationCheckpointConfigMismatch(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_mismatch")
|
||||
execSQL(t, db, "CREATE TABLE it_mismatch (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
||||
execSQL(t, db, "INSERT INTO it_mismatch (id, v) VALUES (1,'a'),(2,'b')")
|
||||
out := t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_mismatch", out)...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Remove the complete flag so resume logic runs.
|
||||
stPath := filepath.Join(out, "state.json")
|
||||
b, _ := os.ReadFile(stPath)
|
||||
b = bytes.Replace(b, []byte(`"complete": true`), []byte(`"complete": false`), 1)
|
||||
if err := os.WriteFile(stPath, b, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
args := baseCLI(dsn, "it_mismatch", out)
|
||||
for i := range args {
|
||||
if args[i] == "1=1" {
|
||||
args[i] = "id > 0"
|
||||
}
|
||||
}
|
||||
if _, err := runCLI(t, context.Background(), args...); err == nil {
|
||||
t.Fatal("expected config mismatch error")
|
||||
} else if !strings.Contains(err.Error(), "different configuration") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
args = append(args, "--force-resume")
|
||||
if _, err := runCLI(t, context.Background(), args...); err != nil {
|
||||
t.Fatalf("force resume: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/cli"
|
||||
"archer/internal/row"
|
||||
"archer/internal/schema"
|
||||
)
|
||||
|
||||
// probeResult is everything learned about the source table needed to plan the
|
||||
// scan and the encoders.
|
||||
type probeResult struct {
|
||||
table *schema.Table
|
||||
key schema.KeyInfo
|
||||
shard *schema.ShardSpec
|
||||
path string // pk | A | B
|
||||
joint *schema.Index
|
||||
single *schema.Index
|
||||
selectCols []string
|
||||
selectMeta []row.Column
|
||||
keyIdx []int
|
||||
shardIdx int
|
||||
outCols []row.Column
|
||||
outIdx []int
|
||||
}
|
||||
|
||||
func probeTable(ctx context.Context, db *sql.DB, cfg *cli.Config, dbName string, loc *time.Location) (*probeResult, error) {
|
||||
table, err := schema.Load(ctx, db, dbName, cfg.Table)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
key, err := table.ChooseKey(cfg.Index)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
outCols, err := table.OutputColumns(cfg.Columns)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
p := &probeResult{table: table, key: key, path: "pk", shardIdx: -1, outCols: outCols}
|
||||
|
||||
if cfg.IsDateShard() {
|
||||
spec, err := schema.ResolveShardColumn(ctx, db, table, cfg.ShardColumn, cfg.ShardColumnType, cfg.Where, loc)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
p.shard = spec
|
||||
prefix := append([]string{spec.Column}, key.Columns...)
|
||||
p.joint = table.FindIndexPrefix(prefix)
|
||||
p.single = table.FindIndexPrefix([]string{spec.Column})
|
||||
switch {
|
||||
case p.joint != nil:
|
||||
p.path = "A"
|
||||
case p.single != nil:
|
||||
p.path = "B"
|
||||
case cfg.AllowFullScan:
|
||||
p.path = "B"
|
||||
default:
|
||||
return nil, archererr.Newf(archererr.KindValidation,
|
||||
"no usable index on %s for date sharding\n joint: create an index on (%s) for keyset chunks\n or: create an index on (%s) for day-at-a-time streaming\n or: pass --allow-full-scan to accept a full table scan\n%s",
|
||||
table.Name, strings.Join(prefix, ", "), spec.Column, describeIndexes(table))
|
||||
}
|
||||
}
|
||||
|
||||
needed := map[string]bool{}
|
||||
for _, c := range outCols {
|
||||
needed[strings.ToLower(c.Name)] = true
|
||||
}
|
||||
for _, c := range key.Columns {
|
||||
needed[strings.ToLower(c)] = true
|
||||
}
|
||||
if p.shard != nil {
|
||||
needed[strings.ToLower(p.shard.Column)] = true
|
||||
}
|
||||
for _, c := range table.Columns {
|
||||
if needed[strings.ToLower(c.Name)] {
|
||||
p.selectCols = append(p.selectCols, c.Name)
|
||||
p.selectMeta = append(p.selectMeta, row.Column{Name: c.Name, Type: c.DataType})
|
||||
}
|
||||
}
|
||||
pos := map[string]int{}
|
||||
for i, name := range p.selectCols {
|
||||
pos[name] = i
|
||||
}
|
||||
p.keyIdx = make([]int, len(key.Columns))
|
||||
for i, c := range key.Columns {
|
||||
idx, ok := pos[c]
|
||||
if !ok {
|
||||
return nil, archererr.Newf(archererr.KindValidation, "internal: key column %s missing from select list", c)
|
||||
}
|
||||
p.keyIdx[i] = idx
|
||||
}
|
||||
if p.shard != nil {
|
||||
idx, ok := pos[p.shard.Column]
|
||||
if !ok {
|
||||
return nil, archererr.Newf(archererr.KindValidation, "internal: shard column %s missing from select list", p.shard.Column)
|
||||
}
|
||||
p.shardIdx = idx
|
||||
}
|
||||
p.outIdx = make([]int, len(outCols))
|
||||
for i, c := range outCols {
|
||||
p.outIdx[i] = pos[c.Name]
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func describeIndexes(t *schema.Table) string {
|
||||
if len(t.Indexes) == 0 {
|
||||
return " indexes: (none)"
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString(" indexes:\n")
|
||||
for _, ix := range t.Indexes {
|
||||
kind := "index"
|
||||
if ix.Name == "PRIMARY" {
|
||||
kind = "primary key"
|
||||
} else if ix.Unique {
|
||||
kind = "unique index"
|
||||
}
|
||||
fmt.Fprintf(&b, " - %s (%s) %s\n", ix.Name, strings.Join(ix.Columns, ", "), kind)
|
||||
}
|
||||
return strings.TrimRight(b.String(), "\n")
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"archer/internal/checkpoint"
|
||||
"archer/internal/cli"
|
||||
)
|
||||
|
||||
func TestStateConfigStoresRedactedDSN(t *testing.T) {
|
||||
const source = "u:s3cr3t@tcp(h:3306)/db?charset=utf8mb4"
|
||||
cfg, err := cli.Parse([]string{"--source", source, "--table", "t", "--where", "id > 0", "--file", "out"}, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hash, fields, err := cfg.Fingerprint()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
if err := checkpoint.Save(dir, &checkpoint.State{ConfigHash: hash, Config: fields}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st, err := checkpoint.Load(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
want := redactDSN(source)
|
||||
if st.Config["source"] != want {
|
||||
t.Fatalf("config.source = %q, want %q", st.Config["source"], want)
|
||||
}
|
||||
raw, err := os.ReadFile(checkpoint.Path(dir))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bytes.Contains(raw, []byte("s3cr3t")) {
|
||||
t.Fatal("state.json must not contain the password")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
// Package archererr classifies runtime errors so the process can pick an exit
|
||||
// code and so callers can distinguish interrupted runs.
|
||||
package archererr
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type Kind int
|
||||
|
||||
const (
|
||||
KindOther Kind = iota
|
||||
KindConfig
|
||||
KindConnection
|
||||
KindWrite
|
||||
KindValidation
|
||||
KindInterrupted
|
||||
)
|
||||
|
||||
// Error wraps an error with a classification.
|
||||
type Error struct {
|
||||
Kind Kind
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *Error) Error() string { return e.Err.Error() }
|
||||
func (e *Error) Unwrap() error { return e.Err }
|
||||
|
||||
func Wrap(kind Kind, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
var existing *Error
|
||||
if errors.As(err, &existing) {
|
||||
return err
|
||||
}
|
||||
return &Error{Kind: kind, Err: err}
|
||||
}
|
||||
|
||||
func Newf(kind Kind, format string, args ...any) error {
|
||||
return &Error{Kind: kind, Err: fmt.Errorf(format, args...)}
|
||||
}
|
||||
|
||||
func KindOf(err error) Kind {
|
||||
var e *Error
|
||||
if errors.As(err, &e) {
|
||||
return e.Kind
|
||||
}
|
||||
return KindOther
|
||||
}
|
||||
|
||||
// ExitCode maps a classification to a process exit code.
|
||||
func ExitCode(err error) int {
|
||||
if err == nil {
|
||||
return 0
|
||||
}
|
||||
switch KindOf(err) {
|
||||
case KindConfig:
|
||||
return 2
|
||||
case KindConnection:
|
||||
return 3
|
||||
case KindWrite:
|
||||
return 4
|
||||
case KindValidation:
|
||||
return 5
|
||||
case KindInterrupted:
|
||||
return 130
|
||||
default:
|
||||
return 1
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
// Package atomicfile 提供 Windows 上更稳健的原子文件替换。
|
||||
//
|
||||
// 直接 os.Rename 在 Windows 上走 MoveFileEx(MOVEFILE_REPLACE_EXISTING),
|
||||
// 只要目标文件此刻被任何进程以「不共享删除」方式打开(杀毒软件实时扫描、
|
||||
// Windows Search 索引器、编辑器预览等都会这样),就会返回
|
||||
// ERROR_ACCESS_DENIED,而且这种占用通常是毫秒级的瞬时状态。
|
||||
// 因此这里对 rename 做有限次指数退避重试;非 Windows 平台无此问题,
|
||||
// 但同样行为无害(第一次就成功)。
|
||||
package atomicfile
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"time"
|
||||
)
|
||||
|
||||
// renameRetries 是 rename 的最大尝试次数(含首次)。
|
||||
const renameRetries = 12
|
||||
|
||||
// Replace 原子地用 tmp 覆盖 final。tmp 与 final 必须在同一文件系统内。
|
||||
//
|
||||
// 失败时保留 tmp 供调用者排查,并返回最后一次 rename 的原始错误。
|
||||
func Replace(tmp, final string) error {
|
||||
var err error
|
||||
delay := time.Millisecond
|
||||
for i := 0; i < renameRetries; i++ {
|
||||
if err = os.Rename(tmp, final); err == nil {
|
||||
return nil
|
||||
}
|
||||
// 非 Windows 上的失败基本都是真错误(跨设备、权限、目录不存在),
|
||||
// 重试无意义,立即返回。
|
||||
if runtime.GOOS != "windows" || !isTransient(err) {
|
||||
return err
|
||||
}
|
||||
time.Sleep(delay)
|
||||
if delay < 20*time.Millisecond {
|
||||
delay *= 2
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("%w (after %d attempts; the file is likely held open by "+
|
||||
"antivirus or the search indexer)", err, renameRetries)
|
||||
}
|
||||
|
||||
// isTransient 判断 rename 失败是否属于「目标被短暂占用」。
|
||||
//
|
||||
// 只识别 Windows 上真实的瞬时错误码。os.Rename 已经把 Win32 错误包成
|
||||
// *os.LinkError,其 Err 是 syscall.Errno,可直接与 syscall 常量比较。
|
||||
func isTransient(err error) bool {
|
||||
var le *os.LinkError
|
||||
if !errors.As(err, &le) {
|
||||
return false
|
||||
}
|
||||
return isTransientErrno(le.Err)
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build !windows
|
||||
|
||||
package atomicfile
|
||||
|
||||
// 非 Windows 平台没有「目标被短暂占用导致 rename 失败」这一语义,
|
||||
// 因此不做重试判断。
|
||||
func isTransientErrno(err error) bool { return false }
|
||||
@@ -0,0 +1,29 @@
|
||||
//go:build windows
|
||||
|
||||
package atomicfile
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// Windows 上值得重试的错误码。
|
||||
const (
|
||||
errAccessDenied syscall.Errno = 5 // ERROR_ACCESS_DENIED
|
||||
errSharingViol syscall.Errno = 32 // ERROR_SHARING_VIOLATION
|
||||
errLockViolation syscall.Errno = 33 // ERROR_LOCK_VIOLATION
|
||||
errBusy syscall.Errno = 170 // ERROR_BUSY
|
||||
errUserMapped syscall.Errno = 1224 // ERROR_USER_MAPPED_FILE
|
||||
)
|
||||
|
||||
func isTransientErrno(err error) bool {
|
||||
var errno syscall.Errno
|
||||
if !errors.As(err, &errno) {
|
||||
return false
|
||||
}
|
||||
switch errno {
|
||||
case errAccessDenied, errSharingViol, errLockViolation, errBusy, errUserMapped:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
// Package checkpoint persists and restores scan progress in state.json.
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"archer/internal/atomicfile"
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
const Version = 1
|
||||
|
||||
// Cursor identifies where the scanner should resume. Phase is one of
|
||||
// "pk", "null", "value" or "day"; the meaning of Key depends on it.
|
||||
type Cursor struct {
|
||||
Phase string `json:"phase,omitempty"`
|
||||
Key row.Key `json:"key,omitempty"`
|
||||
DayStart string `json:"day_start,omitempty"`
|
||||
}
|
||||
|
||||
// State is the entire persisted checkpoint.
|
||||
type State struct {
|
||||
Version int `json:"version"`
|
||||
RunID string `json:"run_id"`
|
||||
ConfigHash string `json:"config_hash"`
|
||||
Config map[string]string `json:"config,omitempty"`
|
||||
ShardBy string `json:"shard_by"`
|
||||
Cursor Cursor `json:"cursor"`
|
||||
LastBucket string `json:"last_bucket,omitempty"`
|
||||
LastShardSeq int64 `json:"last_shard_seq"`
|
||||
RowsDone int64 `json:"rows_done"`
|
||||
Complete bool `json:"complete"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func Path(dir string) string { return filepath.Join(dir, "state.json") }
|
||||
|
||||
// Load reads state.json. A missing file returns (nil, nil).
|
||||
func Load(dir string) (*State, error) {
|
||||
b, err := os.ReadFile(Path(dir))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read state: %w", err)
|
||||
}
|
||||
var st State
|
||||
if err := json.Unmarshal(b, &st); err != nil {
|
||||
return nil, fmt.Errorf("parse state.json: %w", err)
|
||||
}
|
||||
if st.Version != Version {
|
||||
return nil, fmt.Errorf("unsupported state.json version %d", st.Version)
|
||||
}
|
||||
return &st, nil
|
||||
}
|
||||
|
||||
// Save atomically replaces state.json using a temporary file, fsync and
|
||||
// rename.
|
||||
func Save(dir string, st *State) error {
|
||||
st.Version = Version
|
||||
st.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
|
||||
b, err := json.MarshalIndent(st, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode state: %w", err)
|
||||
}
|
||||
b = append(b, '\n')
|
||||
|
||||
final := Path(dir)
|
||||
tmp := final + ".tmp"
|
||||
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create state temp: %w", err)
|
||||
}
|
||||
if _, err := f.Write(b); err != nil {
|
||||
f.Close()
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("write state temp: %w", err)
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
f.Close()
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("sync state temp: %w", err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("close state temp: %w", err)
|
||||
}
|
||||
if err := atomicfile.Replace(tmp, final); err != nil {
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("replace state.json: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
func TestSaveLoadRoundTrip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
st := &State{
|
||||
RunID: "run-1",
|
||||
ConfigHash: "abc",
|
||||
ShardBy: "date",
|
||||
Cursor: Cursor{Phase: "value", Key: row.KeyOf("2025-01-01 00:00:00", "42")},
|
||||
LastBucket: "2025-01-01",
|
||||
RowsDone: 99,
|
||||
}
|
||||
if err := Save(dir, st); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(Path(dir) + ".tmp"); !os.IsNotExist(err) {
|
||||
t.Fatal("temp state file should not remain")
|
||||
}
|
||||
got, err := Load(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got == nil || got.RunID != "run-1" || got.RowsDone != 99 || got.LastBucket != "2025-01-01" {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
if got.Cursor.Phase != "value" || len(got.Cursor.Key) != 2 || got.Cursor.Key[1].Val != "42" {
|
||||
t.Fatalf("cursor %+v", got.Cursor)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadMissing(t *testing.T) {
|
||||
st, err := Load(t.TempDir())
|
||||
if err != nil || st != nil {
|
||||
t.Fatalf("got %v %v", st, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
func baseArgs() []string {
|
||||
return []string{
|
||||
"--source", "u:p@tcp(127.0.0.1:3306)/db",
|
||||
"--table", "t",
|
||||
"--where", "id > 0",
|
||||
"--file", "out",
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaults(t *testing.T) {
|
||||
c, err := Parse(baseArgs(), &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.ChunkSize != 2000 || c.ShardBy != "none" || c.Format != "dump" || c.Compress != "none" {
|
||||
t.Fatalf("bad defaults: %+v", c)
|
||||
}
|
||||
if c.MaxLag.String() != "1s" || c.CheckInterval.String() != "1s" || c.Retries != 1 {
|
||||
t.Fatalf("bad defaults: %+v", c)
|
||||
}
|
||||
if !c.SafeAutoIncrement {
|
||||
t.Fatal("safe auto increment must default true")
|
||||
}
|
||||
if err := c.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLimitAlias(t *testing.T) {
|
||||
args := append(baseArgs(), "--limit", "500")
|
||||
c, err := Parse(args, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.ChunkSize != 500 {
|
||||
t.Fatalf("got %d", c.ChunkSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseInsertModesMutuallyExclusive(t *testing.T) {
|
||||
args := append(baseArgs(), "--insert-ignore", "--replace")
|
||||
if _, err := Parse(args, &bytes.Buffer{}); err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRequired(t *testing.T) {
|
||||
c, _ := Parse([]string{"--source", "u:p@tcp(h:3306)/db", "--table", "t", "--file", "out"}, &bytes.Buffer{})
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--where") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateNeedsTarget(t *testing.T) {
|
||||
c, _ := Parse([]string{"--source", "u:p@tcp(h:3306)/db", "--table", "t", "--where", "1=1"}, &bytes.Buffer{})
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "output target") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateShardRules(t *testing.T) {
|
||||
args := append(baseArgs(), "--shard-by", "count", "--shard-column", "created_at", "--shard-rows", "10")
|
||||
c, err := Parse(args, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--shard-column") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
|
||||
args = append(baseArgs(), "--shard-by", "count")
|
||||
c, _ = Parse(args, &bytes.Buffer{})
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--shard-rows") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
|
||||
args = append(baseArgs(), "--shard-by", "date", "--shard-column", "c", "--bulk-delete", "--purge")
|
||||
c, _ = Parse(args, &bytes.Buffer{})
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "bulk-delete") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateBulkDeleteNeedsPurge(t *testing.T) {
|
||||
args := append(baseArgs(), "--bulk-delete")
|
||||
c, _ := Parse(args, &bytes.Buffer{})
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "requires --purge") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFingerprintExcludesPassword(t *testing.T) {
|
||||
a, _ := Parse([]string{"--source", "u:one@tcp(h:3306)/db", "--table", "t", "--where", "1=1", "--file", "out"}, &bytes.Buffer{})
|
||||
b, _ := Parse([]string{"--source", "u:two@tcp(h:3306)/db", "--table", "t", "--where", "1=1", "--file", "out"}, &bytes.Buffer{})
|
||||
ha, fa, err := a.Fingerprint()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hb, fb, err := b.Fingerprint()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ha != hb {
|
||||
t.Fatal("password must not affect the fingerprint")
|
||||
}
|
||||
if fa["source"] != fb["source"] {
|
||||
t.Fatalf("password must not affect the source field: %q vs %q", fa["source"], fb["source"])
|
||||
}
|
||||
}
|
||||
|
||||
func fingerprintFor(t *testing.T, dsn string) (string, map[string]string) {
|
||||
t.Helper()
|
||||
c, err := Parse([]string{"--source", dsn, "--table", "t", "--where", "id > 0", "--file", "out"}, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h, f, err := c.Fingerprint()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return h, f
|
||||
}
|
||||
|
||||
func TestFingerprintTracksConnectionCharset(t *testing.T) {
|
||||
hBase, _ := fingerprintFor(t, "u:p@tcp(h:3306)/db")
|
||||
hCharset, fCharset := fingerprintFor(t, "u:p@tcp(h:3306)/db?charset=utf8mb4")
|
||||
if hBase == hCharset {
|
||||
t.Fatal("charset must affect the fingerprint")
|
||||
}
|
||||
if !strings.Contains(fCharset["source"], "charset=utf8mb4") {
|
||||
t.Fatalf("source field missing charset: %q", fCharset["source"])
|
||||
}
|
||||
|
||||
hCollation, _ := fingerprintFor(t, "u:p@tcp(h:3306)/db?charset=utf8mb4&collation=utf8mb4_bin")
|
||||
if hCharset == hCollation {
|
||||
t.Fatal("collation must affect the fingerprint")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDsnFingerprintRedactsPassword(t *testing.T) {
|
||||
c, err := mysql.ParseDSN("u:s3cr3t@tcp(h:3306)/db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := dsnFingerprint(c)
|
||||
if got != "u@tcp(h:3306)/db" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if strings.Contains(got, "s3cr3t") {
|
||||
t.Fatal("password leaked into the fingerprint")
|
||||
}
|
||||
if c.Passwd != "s3cr3t" {
|
||||
t.Fatal("dsnFingerprint must not modify the input config")
|
||||
}
|
||||
if dsnFingerprint(nil) != "" {
|
||||
t.Fatal("nil config must yield an empty fragment")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFingerprintDestEmptyAndNoPassword(t *testing.T) {
|
||||
h, f := fingerprintFor(t, "u:s3cr3t@tcp(h:3306)/db")
|
||||
if f["dest"] != "" {
|
||||
t.Fatalf("dest must be empty without --dest: %q", f["dest"])
|
||||
}
|
||||
if f["source"] != "u@tcp(h:3306)/db" {
|
||||
t.Fatalf("source = %q", f["source"])
|
||||
}
|
||||
if strings.Contains(h, "s3cr3t") || strings.Contains(f["source"], "s3cr3t") {
|
||||
t.Fatal("password leaked into the fingerprint")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFingerprintDetectsTimezoneChange(t *testing.T) {
|
||||
a, _ := Parse(append(baseArgs(), "--shard-by", "date", "--shard-column", "c", "--shard-time-zone", "UTC"), &bytes.Buffer{})
|
||||
b, _ := Parse(append(baseArgs(), "--shard-by", "date", "--shard-column", "c", "--shard-time-zone", "Asia/Shanghai"), &bytes.Buffer{})
|
||||
ha, fa, _ := a.Fingerprint()
|
||||
hb, _, _ := b.Fingerprint()
|
||||
if ha == hb {
|
||||
t.Fatal("time zone must affect the fingerprint")
|
||||
}
|
||||
_ = fa
|
||||
}
|
||||
|
||||
func TestDiffFields(t *testing.T) {
|
||||
old := map[string]string{"a": "1", "b": "2"}
|
||||
cur := map[string]string{"a": "1", "b": "3", "c": "4"}
|
||||
d := DiffFields(old, cur)
|
||||
if len(d) != 2 {
|
||||
t.Fatalf("got %v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateCompressLevel(t *testing.T) {
|
||||
for _, level := range []string{"0", "10"} {
|
||||
c, err := Parse(append(baseArgs(), "--compress", "gzip", "--compress-level", level), &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "between 1 and 9") {
|
||||
t.Fatalf("level %s: got %v", level, err)
|
||||
}
|
||||
}
|
||||
|
||||
c, _ := Parse(append(baseArgs(), "--compress", "none", "--compress-level", "5"), &bytes.Buffer{})
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "only valid when compression is enabled") {
|
||||
t.Fatalf("none + level 5: got %v", err)
|
||||
}
|
||||
|
||||
c, _ = Parse(append(baseArgs(), "--compress", "none", "--compress-level", "3"), &bytes.Buffer{})
|
||||
if err := c.Validate(); err != nil {
|
||||
t.Fatalf("none + default level must be accepted: %v", err)
|
||||
}
|
||||
|
||||
for _, name := range []string{"gzip", "zstd", "xz"} {
|
||||
c, _ = Parse(append(baseArgs(), "--compress", name, "--compress-level", "9"), &bytes.Buffer{})
|
||||
if err := c.Validate(); err != nil {
|
||||
t.Fatalf("%s level 9: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressModes(t *testing.T) {
|
||||
cases := []struct {
|
||||
arg string
|
||||
mode ProgressMode
|
||||
rows int64
|
||||
valid bool
|
||||
}{
|
||||
{"", ProgressAuto, 0, true},
|
||||
{"auto", ProgressAuto, 0, true},
|
||||
{"on", ProgressOn, 0, true},
|
||||
{"off", ProgressOff, 0, true},
|
||||
{"0", ProgressOff, 0, true},
|
||||
{"100000", ProgressRows, 100000, true},
|
||||
{"1", ProgressRows, 1, true},
|
||||
{"abc", 0, 0, false},
|
||||
{"-1", 0, 0, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
args := baseArgs()
|
||||
if tc.arg != "" {
|
||||
args = append(args, "--progress", tc.arg)
|
||||
}
|
||||
c, err := Parse(args, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatalf("%q: parse: %v", tc.arg, err)
|
||||
}
|
||||
err = c.Validate()
|
||||
if !tc.valid {
|
||||
if err == nil {
|
||||
t.Fatalf("%q: expected a configuration error", tc.arg)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("%q: validate: %v", tc.arg, err)
|
||||
}
|
||||
if c.Progress.Mode != tc.mode || c.Progress.Rows != tc.rows {
|
||||
t.Fatalf("%q: got mode=%d rows=%d", tc.arg, c.Progress.Mode, c.Progress.Rows)
|
||||
}
|
||||
}
|
||||
|
||||
c, _ := Parse(baseArgs(), &bytes.Buffer{})
|
||||
if c.Progress.Mode != ProgressAuto {
|
||||
t.Fatalf("default progress mode = %d, want auto", c.Progress.Mode)
|
||||
}
|
||||
if c.ProgressInterval != time.Second {
|
||||
t.Fatalf("default progress interval = %s", c.ProgressInterval)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressEnabledDependsOnTTY(t *testing.T) {
|
||||
on, _ := Parse(append(baseArgs(), "--progress", "on"), &bytes.Buffer{})
|
||||
off, _ := Parse(append(baseArgs(), "--progress", "off"), &bytes.Buffer{})
|
||||
auto, _ := Parse(baseArgs(), &bytes.Buffer{})
|
||||
for _, c := range []*Config{on, off, auto} {
|
||||
if err := c.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if !on.Progress.Enabled(false) || !on.Progress.Enabled(true) {
|
||||
t.Fatal("on must always be enabled")
|
||||
}
|
||||
if off.Progress.Enabled(true) || off.Progress.Enabled(false) {
|
||||
t.Fatal("off must never be enabled")
|
||||
}
|
||||
if !auto.Progress.Enabled(true) {
|
||||
t.Fatal("auto must be enabled on a terminal")
|
||||
}
|
||||
if auto.Progress.Enabled(false) {
|
||||
t.Fatal("auto must be silent without a terminal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressIntervalValidation(t *testing.T) {
|
||||
c, err := Parse(append(baseArgs(), "--progress-interval", "0s"), &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--progress-interval") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFingerprintCompressLevel(t *testing.T) {
|
||||
implicit, _ := Parse(append(baseArgs(), "--compress", "gzip"), &bytes.Buffer{})
|
||||
explicit, _ := Parse(append(baseArgs(), "--compress", "gzip", "--compress-level", "3"), &bytes.Buffer{})
|
||||
hImplicit, fImplicit, err := implicit.Fingerprint()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hExplicit, _, err := explicit.Fingerprint()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hImplicit != hExplicit {
|
||||
t.Fatal("the default level must not change the fingerprint")
|
||||
}
|
||||
if _, ok := fImplicit["compress_level"]; ok {
|
||||
t.Fatal("the default level must not be fingerprinted")
|
||||
}
|
||||
|
||||
five, _ := Parse(append(baseArgs(), "--compress", "gzip", "--compress-level", "5"), &bytes.Buffer{})
|
||||
hFive, fFive, err := five.Fingerprint()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hFive == hImplicit {
|
||||
t.Fatal("a non-default level must change the fingerprint")
|
||||
}
|
||||
if fFive["compress_level"] != "5" {
|
||||
t.Fatalf("compress_level = %q", fFive["compress_level"])
|
||||
}
|
||||
|
||||
if diffs := DiffFields(fImplicit, fFive); len(diffs) != 1 || !strings.Contains(diffs[0], "compress_level") {
|
||||
t.Fatalf("raising the level: %v", diffs)
|
||||
}
|
||||
if diffs := DiffFields(fFive, fImplicit); len(diffs) != 1 || !strings.Contains(diffs[0], "compress_level") {
|
||||
t.Fatalf("lowering the level back to default: %v", diffs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
// Package cli parses and validates archer command-line configuration.
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// Config is the fully resolved runtime configuration.
|
||||
type Config struct {
|
||||
SourceDSN string
|
||||
DestDSN string
|
||||
Table string
|
||||
DestTable string
|
||||
Where string
|
||||
Columns []string
|
||||
FileDir string
|
||||
Purge bool
|
||||
BulkDelete bool
|
||||
InsertMode string // "", "ignore", "replace"
|
||||
|
||||
ChunkSize int
|
||||
|
||||
ShardBy string // none|count|date
|
||||
ShardRows int
|
||||
ShardColumn string
|
||||
ShardColumnType string
|
||||
ShardTimeZone string
|
||||
ZeroDate string
|
||||
|
||||
Format string
|
||||
Compress string
|
||||
CompressLevel int
|
||||
CSVNull string
|
||||
|
||||
MaxLag time.Duration
|
||||
CheckInterval time.Duration
|
||||
CheckReplicaLag string
|
||||
Retries int
|
||||
|
||||
SafeAutoIncrement bool
|
||||
Progress ProgressValue
|
||||
ProgressInterval time.Duration
|
||||
Statistics bool
|
||||
DryRun bool
|
||||
AllowFullScan bool
|
||||
ForceResume bool
|
||||
Index string
|
||||
Version bool
|
||||
}
|
||||
|
||||
// Defaults returns a Config with documented default values applied.
|
||||
func Defaults() *Config {
|
||||
return &Config{
|
||||
ChunkSize: 2000,
|
||||
ShardBy: "none",
|
||||
ShardRows: 0,
|
||||
ShardTimeZone: "UTC",
|
||||
Format: "dump",
|
||||
Compress: "none",
|
||||
CompressLevel: 3,
|
||||
CSVNull: `\N`,
|
||||
MaxLag: time.Second,
|
||||
CheckInterval: time.Second,
|
||||
Retries: 1,
|
||||
SafeAutoIncrement: true,
|
||||
Progress: ProgressValue{Mode: ProgressAuto},
|
||||
ProgressInterval: time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
// SourceConfig parses and normalizes the source DSN, forcing parseTime off so
|
||||
// values arrive as raw bytes.
|
||||
func (c *Config) SourceConfig() (*mysql.Config, error) {
|
||||
return normalize(c.SourceDSN)
|
||||
}
|
||||
|
||||
// DestConfig parses and normalizes the dest DSN.
|
||||
func (c *Config) DestConfig() (*mysql.Config, error) {
|
||||
if c.DestDSN == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return normalize(c.DestDSN)
|
||||
}
|
||||
|
||||
func normalize(dsn string) (*mysql.Config, error) {
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid DSN: %w", err)
|
||||
}
|
||||
cfg.ParseTime = false
|
||||
if cfg.Loc == nil {
|
||||
cfg.Loc = time.UTC
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// SameInstance reports whether two normalized DSNs point at the same MySQL
|
||||
// server, which determines whether dest writes and purges share a transaction.
|
||||
func SameInstance(src, dst *mysql.Config) bool {
|
||||
if src == nil || dst == nil {
|
||||
return false
|
||||
}
|
||||
return src.Net == dst.Net && src.Addr == dst.Addr
|
||||
}
|
||||
|
||||
// ShardMode helpers.
|
||||
func (c *Config) IsDateShard() bool { return c.ShardBy == "date" }
|
||||
func (c *Config) IsCountShard() bool { return c.ShardBy == "count" }
|
||||
func (c *Config) HasFile() bool { return c.FileDir != "" }
|
||||
@@ -0,0 +1,102 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// Fingerprint returns a stable hash of every configuration input that can
|
||||
// change scan or shard semantics, plus the normalized fields used to build it.
|
||||
// The source and dest fields are standard MySQL DSNs that include every
|
||||
// connection parameter the driver parsed (charset, collation, timeout, ...),
|
||||
// so a resume notices any of them changing. The password is the one exclusion:
|
||||
// rotating it must not break resume.
|
||||
func (c *Config) Fingerprint() (string, map[string]string, error) {
|
||||
src, err := c.SourceConfig()
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("source DSN: %w", err)
|
||||
}
|
||||
dst, err := c.DestConfig()
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("dest DSN: %w", err)
|
||||
}
|
||||
fields := map[string]string{
|
||||
"source": dsnFingerprint(src),
|
||||
"dest": dsnFingerprint(dst),
|
||||
"table": c.Table,
|
||||
"dest_table": c.DestTable,
|
||||
"where": c.Where,
|
||||
"columns": strings.Join(c.Columns, ","),
|
||||
"purge": strconv.FormatBool(c.Purge),
|
||||
"bulk_delete": strconv.FormatBool(c.BulkDelete),
|
||||
"insert_mode": c.InsertMode,
|
||||
"shard_by": c.ShardBy,
|
||||
"shard_rows": strconv.Itoa(c.ShardRows),
|
||||
"shard_column": c.ShardColumn,
|
||||
"shard_column_type": c.ShardColumnType,
|
||||
"shard_time_zone": c.ShardTimeZone,
|
||||
"zero_date": c.ZeroDate,
|
||||
"format": c.Format,
|
||||
"compress": c.Compress,
|
||||
"csv_null": c.CSVNull,
|
||||
"safe_auto_increment": strconv.FormatBool(c.SafeAutoIncrement),
|
||||
}
|
||||
// The level only changes the output when compression is on, and omitting
|
||||
// the default keeps fingerprints of tasks started before it existed
|
||||
// byte-identical so they can resume.
|
||||
if c.Compress != "" && c.Compress != "none" && c.CompressLevel != 3 {
|
||||
fields["compress_level"] = strconv.Itoa(c.CompressLevel)
|
||||
}
|
||||
keys := make([]string, 0, len(fields))
|
||||
for k := range fields {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var b strings.Builder
|
||||
for _, k := range keys {
|
||||
b.WriteString(k)
|
||||
b.WriteByte('=')
|
||||
b.WriteString(fields[k])
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
sum := sha256.Sum256([]byte(b.String()))
|
||||
return hex.EncodeToString(sum[:]), fields, nil
|
||||
}
|
||||
|
||||
// dsnFingerprint renders c as a standard MySQL DSN covering every parameter
|
||||
// the driver parsed, except the password. Copying by value keeps the driver's
|
||||
// unexported fields (charsets, timeTruncate, ...) intact, which is the only way
|
||||
// FormatDSN can serialize them.
|
||||
func dsnFingerprint(c *mysql.Config) string {
|
||||
if c == nil {
|
||||
return ""
|
||||
}
|
||||
redacted := *c
|
||||
redacted.Passwd = ""
|
||||
return redacted.FormatDSN()
|
||||
}
|
||||
|
||||
// DiffFields lists keys whose values differ between two fingerprints.
|
||||
func DiffFields(old, cur map[string]string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for k, v := range cur {
|
||||
seen[k] = true
|
||||
if old[k] != v {
|
||||
out = append(out, fmt.Sprintf("%s: %q -> %q", k, old[k], v))
|
||||
}
|
||||
}
|
||||
for k, v := range old {
|
||||
if !seen[k] {
|
||||
out = append(out, fmt.Sprintf("%s: %q -> (absent)", k, v))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Parse parses args (without the program name) into a Config. It does not
|
||||
// validate combinations; call Validate for that.
|
||||
func Parse(args []string, errOut io.Writer) (*Config, error) {
|
||||
c := Defaults()
|
||||
var (
|
||||
columnsArg string
|
||||
limitArg = -1
|
||||
insertIgnoreArg bool
|
||||
replaceArg bool
|
||||
)
|
||||
fs := flag.NewFlagSet("archer", flag.ContinueOnError)
|
||||
fs.SetOutput(errOut)
|
||||
|
||||
fs.StringVar(&c.SourceDSN, "source", "", "source MySQL DSN (required)")
|
||||
fs.StringVar(&c.DestDSN, "dest", "", "destination MySQL DSN")
|
||||
fs.StringVar(&c.Table, "table", "", "source table (required)")
|
||||
fs.StringVar(&c.DestTable, "dest-table", "", "destination table (defaults to --table)")
|
||||
fs.StringVar(&c.Where, "where", "", "row filter for the source table (required)")
|
||||
fs.StringVar(&columnsArg, "columns", "", "comma separated columns to export (default: all)")
|
||||
fs.StringVar(&c.FileDir, "file", "", "output directory for shard files")
|
||||
fs.BoolVar(&c.Purge, "purge", false, "delete archived rows from the source")
|
||||
fs.BoolVar(&c.BulkDelete, "bulk-delete", false, "delete by key range instead of IN list (requires --purge)")
|
||||
fs.BoolVar(&insertIgnoreArg, "insert-ignore", false, "use INSERT IGNORE for --dest")
|
||||
fs.BoolVar(&replaceArg, "replace", false, "use REPLACE for --dest")
|
||||
|
||||
fs.IntVar(&c.ChunkSize, "chunk-size", c.ChunkSize, "rows per SELECT / transaction / checkpoint")
|
||||
fs.IntVar(&limitArg, "limit", -1, "deprecated alias for --chunk-size")
|
||||
|
||||
fs.StringVar(&c.ShardBy, "shard-by", c.ShardBy, "shard mode: none|count|date")
|
||||
fs.IntVar(&c.ShardRows, "shard-rows", c.ShardRows, "rows per shard file (count/date modes)")
|
||||
fs.StringVar(&c.ShardColumn, "shard-column", "", "date or epoch column used for date sharding")
|
||||
fs.StringVar(&c.ShardColumnType, "shard-column-type", "", "epoch unit: epoch-seconds|epoch-milliseconds|epoch-microseconds")
|
||||
fs.StringVar(&c.ShardTimeZone, "shard-time-zone", c.ShardTimeZone, "time zone for epoch shard values")
|
||||
fs.StringVar(&c.ZeroDate, "zero-date", "", "zero date handling: bucket (default: error)")
|
||||
|
||||
fs.StringVar(&c.Format, "format", c.Format, "output format: dump|csv|jsonl")
|
||||
fs.StringVar(&c.Compress, "compress", c.Compress, "compression: none|gzip|zstd|xz")
|
||||
fs.IntVar(&c.CompressLevel, "compress-level", c.CompressLevel, "compression effort 1-9 (higher is smaller and slower)")
|
||||
fs.StringVar(&c.CSVNull, "csv-null", c.CSVNull, "string used for NULL in csv output")
|
||||
|
||||
fs.DurationVar(&c.MaxLag, "max-lag", c.MaxLag, "maximum tolerated replica lag")
|
||||
fs.DurationVar(&c.CheckInterval, "check-interval", c.CheckInterval, "wait between replica lag checks")
|
||||
fs.StringVar(&c.CheckReplicaLag, "check-replica-lag", "", "DSN of a replica whose lag is checked before each fetch")
|
||||
fs.IntVar(&c.Retries, "retries", c.Retries, "retries for cross-instance commits")
|
||||
|
||||
fs.BoolVar(&c.SafeAutoIncrement, "safe-auto-increment", c.SafeAutoIncrement, "never purge the maximum auto-increment row")
|
||||
fs.Var(&c.Progress, "progress", "progress output: auto|on|off|N (default auto; N: also every N rows)")
|
||||
fs.DurationVar(&c.ProgressInterval, "progress-interval", c.ProgressInterval, "minimum interval between progress updates")
|
||||
fs.BoolVar(&c.Statistics, "statistics", false, "print a summary when finished")
|
||||
fs.BoolVar(&c.DryRun, "dry-run", false, "print resolved config and SQL without writing")
|
||||
fs.BoolVar(&c.AllowFullScan, "allow-full-scan", false, "allow date sharding without a usable index")
|
||||
fs.BoolVar(&c.ForceResume, "force-resume", false, "resume even when the config fingerprint changed")
|
||||
fs.StringVar(&c.Index, "index", "", "force a unique index for the scan key")
|
||||
fs.BoolVar(&c.Version, "version", false, "print version and exit")
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.Version {
|
||||
return c, nil
|
||||
}
|
||||
|
||||
set := map[string]bool{}
|
||||
fs.Visit(func(f *flag.Flag) { set[f.Name] = true })
|
||||
|
||||
if set["limit"] {
|
||||
if set["chunk-size"] {
|
||||
return nil, fmt.Errorf("--limit is a deprecated alias of --chunk-size; pass only one")
|
||||
}
|
||||
c.ChunkSize = limitArg
|
||||
}
|
||||
if columnsArg != "" {
|
||||
for _, part := range strings.Split(columnsArg, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
c.Columns = append(c.Columns, part)
|
||||
}
|
||||
}
|
||||
}
|
||||
if insertIgnoreArg && replaceArg {
|
||||
return nil, fmt.Errorf("--insert-ignore and --replace are mutually exclusive")
|
||||
}
|
||||
switch {
|
||||
case insertIgnoreArg:
|
||||
c.InsertMode = "ignore"
|
||||
case replaceArg:
|
||||
c.InsertMode = "replace"
|
||||
}
|
||||
if c.DestTable == "" {
|
||||
c.DestTable = c.Table
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ProgressMode is the resolved meaning of --progress.
|
||||
type ProgressMode int
|
||||
|
||||
const (
|
||||
ProgressAuto ProgressMode = iota // live on a terminal, silent otherwise
|
||||
ProgressOff
|
||||
ProgressOn
|
||||
ProgressRows // enabled, plus a row-count threshold
|
||||
)
|
||||
|
||||
// ProgressValue is the flag.Value behind --progress. It keeps the raw
|
||||
// argument so invalid values are reported as configuration errors by Validate
|
||||
// rather than as bare flag parse errors.
|
||||
type ProgressValue struct {
|
||||
Mode ProgressMode
|
||||
Rows int64
|
||||
raw string
|
||||
}
|
||||
|
||||
// Set records the raw argument; resolve happens in Validate.
|
||||
func (p *ProgressValue) Set(s string) error {
|
||||
p.raw = s
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *ProgressValue) String() string {
|
||||
if p.raw != "" {
|
||||
return p.raw
|
||||
}
|
||||
switch p.Mode {
|
||||
case ProgressOn:
|
||||
return "on"
|
||||
case ProgressOff:
|
||||
return "off"
|
||||
case ProgressRows:
|
||||
return strconv.FormatInt(p.Rows, 10)
|
||||
default:
|
||||
return "auto"
|
||||
}
|
||||
}
|
||||
|
||||
// resolve parses the recorded argument into Mode and Rows.
|
||||
func (p *ProgressValue) resolve() error {
|
||||
switch strings.ToLower(strings.TrimSpace(p.raw)) {
|
||||
case "", "auto":
|
||||
p.Mode, p.Rows = ProgressAuto, 0
|
||||
case "on":
|
||||
p.Mode, p.Rows = ProgressOn, 0
|
||||
case "off":
|
||||
p.Mode, p.Rows = ProgressOff, 0
|
||||
default:
|
||||
n, err := strconv.ParseInt(strings.TrimSpace(p.raw), 10, 64)
|
||||
if err != nil || n < 0 {
|
||||
return fmt.Errorf("--progress must be auto, on, off or a non-negative integer, got %q", p.raw)
|
||||
}
|
||||
if n == 0 {
|
||||
p.Mode, p.Rows = ProgressOff, 0
|
||||
} else {
|
||||
p.Mode, p.Rows = ProgressRows, n
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Enabled reports whether progress output should be produced. tty describes
|
||||
// whether stderr is a terminal.
|
||||
func (p ProgressValue) Enabled(tty bool) bool {
|
||||
switch p.Mode {
|
||||
case ProgressOn, ProgressRows:
|
||||
return true
|
||||
case ProgressAuto:
|
||||
return tty
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archer/internal/archererr"
|
||||
)
|
||||
|
||||
// Validate checks required flags and the inter-parameter dependency rules.
|
||||
// The shard column's existence and type are checked later, after metadata is
|
||||
// loaded, so the error can list candidate columns.
|
||||
func (c *Config) Validate() error {
|
||||
if c.SourceDSN == "" {
|
||||
return cfgErr("--source is required")
|
||||
}
|
||||
if c.Table == "" {
|
||||
return cfgErr("--table is required")
|
||||
}
|
||||
if strings.TrimSpace(c.Where) == "" {
|
||||
return cfgErr("--where is required")
|
||||
}
|
||||
if !c.HasFile() && c.DestDSN == "" && !c.Purge {
|
||||
return cfgErr("at least one output target is required: --file, --dest or --purge")
|
||||
}
|
||||
if c.ChunkSize <= 0 {
|
||||
return cfgErr("--chunk-size must be greater than 0")
|
||||
}
|
||||
if c.Retries < 0 {
|
||||
return cfgErr("--retries must be 0 or greater")
|
||||
}
|
||||
if err := c.Progress.resolve(); err != nil {
|
||||
return cfgErr("%v", err)
|
||||
}
|
||||
if c.ProgressInterval <= 0 {
|
||||
return cfgErr("--progress-interval must be greater than 0")
|
||||
}
|
||||
|
||||
switch c.ShardBy {
|
||||
case "none", "count", "date":
|
||||
default:
|
||||
return cfgErr("--shard-by must be one of none, count, date")
|
||||
}
|
||||
|
||||
switch c.ShardBy {
|
||||
case "date":
|
||||
if c.ShardRows < 0 {
|
||||
return cfgErr("--shard-rows must be 0 or greater")
|
||||
}
|
||||
default:
|
||||
if c.ShardColumn != "" {
|
||||
return cfgErr("--shard-column is only valid with --shard-by date")
|
||||
}
|
||||
if c.ShardColumnType != "" {
|
||||
return cfgErr("--shard-column-type is only valid with --shard-by date")
|
||||
}
|
||||
if c.ZeroDate != "" {
|
||||
return cfgErr("--zero-date is only valid with --shard-by date")
|
||||
}
|
||||
}
|
||||
if c.ShardBy == "none" && c.ShardRows != 0 {
|
||||
return cfgErr("--shard-rows is only valid with --shard-by count or date")
|
||||
}
|
||||
if c.ShardBy == "count" && c.ShardRows == 0 {
|
||||
return cfgErr("--shard-rows must be greater than 0 for --shard-by count")
|
||||
}
|
||||
if c.BulkDelete && !c.Purge {
|
||||
return cfgErr("--bulk-delete requires --purge")
|
||||
}
|
||||
if c.BulkDelete && c.ShardBy == "date" {
|
||||
return cfgErr("--bulk-delete cannot be used with --shard-by date; delete by IN list instead")
|
||||
}
|
||||
|
||||
switch c.Format {
|
||||
case "dump", "csv", "jsonl":
|
||||
default:
|
||||
return cfgErr("--format must be one of dump, csv, jsonl")
|
||||
}
|
||||
switch c.Compress {
|
||||
case "none", "gzip", "zstd", "xz":
|
||||
default:
|
||||
return cfgErr("--compress must be one of none, gzip, zstd, xz")
|
||||
}
|
||||
if c.CompressLevel < 1 || c.CompressLevel > 9 {
|
||||
return cfgErr("--compress-level must be between 1 and 9")
|
||||
}
|
||||
if c.Compress == "none" && c.CompressLevel != 3 {
|
||||
return cfgErr("--compress-level is only valid when compression is enabled")
|
||||
}
|
||||
switch c.ZeroDate {
|
||||
case "", "bucket":
|
||||
default:
|
||||
return cfgErr("--zero-date must be empty or bucket")
|
||||
}
|
||||
if _, err := time.LoadLocation(c.ShardTimeZone); err != nil {
|
||||
return cfgErr("--shard-time-zone %q is not a valid location: %v", c.ShardTimeZone, err)
|
||||
}
|
||||
if _, err := c.SourceConfig(); err != nil {
|
||||
return cfgErr("--source: %v", err)
|
||||
}
|
||||
if _, err := c.DestConfig(); err != nil {
|
||||
return cfgErr("--dest: %v", err)
|
||||
}
|
||||
if c.CheckReplicaLag != "" {
|
||||
if err := validateDSN(c.CheckReplicaLag); err != nil {
|
||||
return cfgErr("--check-replica-lag: %v", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateDSN(dsn string) error {
|
||||
_, err := normalize(dsn)
|
||||
return err
|
||||
}
|
||||
|
||||
func cfgErr(format string, args ...any) error {
|
||||
return archererr.Newf(archererr.KindConfig, format, args...)
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
// Package estimate derives a cheap total-size estimate for progress display.
|
||||
//
|
||||
// Only bounded MIN/MAX queries are used; COUNT(*) is never attempted. A query
|
||||
// that fails, times out, or whose metric does not apply silently yields no
|
||||
// estimate, leaving the caller to render rows and rate only.
|
||||
package estimate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archer/internal/progress"
|
||||
"archer/internal/row"
|
||||
"archer/internal/schema"
|
||||
)
|
||||
|
||||
// Spec describes which metric a run can expose.
|
||||
type Spec struct {
|
||||
Kind string // "day", "key" or "" for none
|
||||
Column string
|
||||
ShardKind schema.ShardKind
|
||||
Unit int64
|
||||
Loc *time.Location
|
||||
}
|
||||
|
||||
// RangeQuery returns MIN and MAX of a column as raw text. Implementations must
|
||||
// honor ctx so the bounded timeout is effective.
|
||||
type RangeQuery func(ctx context.Context, column string) (minRaw, maxRaw sql.NullString, err error)
|
||||
|
||||
// Bounded runs q under a timeout and converts the result into a display
|
||||
// estimate. It returns nil instead of an error on every failure path.
|
||||
func Bounded(ctx context.Context, q RangeQuery, spec Spec, timeout time.Duration) *progress.Estimate {
|
||||
if q == nil || spec.Kind == "" || timeout <= 0 {
|
||||
return nil
|
||||
}
|
||||
qctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
minRaw, maxRaw, err := q(qctx, spec.Column)
|
||||
if err != nil || !minRaw.Valid || !maxRaw.Valid {
|
||||
return nil
|
||||
}
|
||||
|
||||
var lo, hi float64
|
||||
var ok bool
|
||||
switch spec.Kind {
|
||||
case "day":
|
||||
lo, hi, ok = dayRange(minRaw.String, maxRaw.String, spec.ShardKind, spec.Unit, spec.Loc)
|
||||
case "key":
|
||||
lo, hi, ok = keyRange(minRaw.String, maxRaw.String)
|
||||
}
|
||||
if !ok || hi <= lo {
|
||||
return nil
|
||||
}
|
||||
return &progress.Estimate{Kind: spec.Kind, Lo: lo, Hi: hi}
|
||||
}
|
||||
|
||||
// KeyColumn reports the scan key column when it is a single integer column,
|
||||
// which is the only non-date shape a cheap key-range estimate applies to.
|
||||
func KeyColumn(cols, types []string) (string, bool) {
|
||||
if len(cols) != 1 || len(types) != 1 {
|
||||
return "", false
|
||||
}
|
||||
if !row.IsInteger(types[0]) {
|
||||
return "", false
|
||||
}
|
||||
return cols[0], true
|
||||
}
|
||||
|
||||
func dayRange(minRaw, maxRaw string, kind schema.ShardKind, unit int64, loc *time.Location) (float64, float64, bool) {
|
||||
if loc == nil {
|
||||
loc = time.UTC
|
||||
}
|
||||
switch kind {
|
||||
case schema.ShardDate, schema.ShardDateTime:
|
||||
lo, ok1 := daySeconds(minRaw, loc)
|
||||
hi, ok2 := daySeconds(maxRaw, loc)
|
||||
return lo, hi, ok1 && ok2
|
||||
case schema.ShardEpoch:
|
||||
minV, err1 := strconv.ParseInt(strings.TrimSpace(minRaw), 10, 64)
|
||||
maxV, err2 := strconv.ParseInt(strings.TrimSpace(maxRaw), 10, 64)
|
||||
if err1 != nil || err2 != nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
lo := dayStart(schema.EpochTime(minV, unit, loc), loc).Unix()
|
||||
hi := dayStart(schema.EpochTime(maxV, unit, loc), loc).Unix()
|
||||
return float64(lo), float64(hi), true
|
||||
}
|
||||
return 0, 0, false
|
||||
}
|
||||
|
||||
func daySeconds(raw string, loc *time.Location) (float64, bool) {
|
||||
s := strings.TrimSpace(raw)
|
||||
if len(s) < 10 {
|
||||
return 0, false
|
||||
}
|
||||
day := s[:10]
|
||||
if strings.HasPrefix(day, "0000-") {
|
||||
return 0, false
|
||||
}
|
||||
t, err := time.ParseInLocation("2006-01-02", day, loc)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return float64(t.Unix()), true
|
||||
}
|
||||
|
||||
func keyRange(minRaw, maxRaw string) (float64, float64, bool) {
|
||||
minV, err1 := strconv.ParseInt(strings.TrimSpace(minRaw), 10, 64)
|
||||
maxV, err2 := strconv.ParseInt(strings.TrimSpace(maxRaw), 10, 64)
|
||||
if err1 != nil || err2 != nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
return float64(minV), float64(maxV), true
|
||||
}
|
||||
|
||||
func dayStart(t time.Time, loc *time.Location) time.Time {
|
||||
return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, loc)
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package estimate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"archer/internal/schema"
|
||||
)
|
||||
|
||||
func validRange(minRaw, maxRaw string) RangeQuery {
|
||||
return func(context.Context, string) (sql.NullString, sql.NullString, error) {
|
||||
return sql.NullString{String: minRaw, Valid: true}, sql.NullString{String: maxRaw, Valid: true}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyColumnApplicability(t *testing.T) {
|
||||
cases := []struct {
|
||||
cols, types []string
|
||||
want bool
|
||||
}{
|
||||
{[]string{"id"}, []string{"bigint"}, true},
|
||||
{[]string{"id"}, []string{"int"}, true},
|
||||
{[]string{"a", "b"}, []string{"bigint", "bigint"}, false},
|
||||
{[]string{"code"}, []string{"varchar"}, false},
|
||||
{[]string{"payload"}, []string{"blob"}, false},
|
||||
{[]string{"bin"}, []string{"varbinary"}, false},
|
||||
{[]string{"ts"}, []string{"datetime"}, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
col, ok := KeyColumn(tc.cols, tc.types)
|
||||
if ok != tc.want {
|
||||
t.Fatalf("%v/%v: got %v want %v", tc.cols, tc.types, ok, tc.want)
|
||||
}
|
||||
if ok && col != tc.cols[0] {
|
||||
t.Fatalf("%v: wrong column %q", tc.cols, col)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedDayRangeFromDates(t *testing.T) {
|
||||
spec := Spec{Kind: "day", Column: "created_at", ShardKind: schema.ShardDate, Loc: time.UTC}
|
||||
got := Bounded(context.Background(), validRange("2025-01-01", "2025-01-11"), spec, time.Second)
|
||||
if got == nil || got.Kind != "day" {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
if d := time.Duration(got.Hi-got.Lo) * time.Second; d != 10*24*time.Hour {
|
||||
t.Fatalf("span = %s", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedDayRangeFromDatetimeAndEpoch(t *testing.T) {
|
||||
dt := Spec{Kind: "day", Column: "c", ShardKind: schema.ShardDateTime, Loc: time.UTC}
|
||||
if got := Bounded(context.Background(), validRange("2025-01-01 10:00:00", "2025-01-03 23:00:00"), dt, time.Second); got == nil {
|
||||
t.Fatal("datetime range unavailable")
|
||||
}
|
||||
ep := Spec{Kind: "day", Column: "ts", ShardKind: schema.ShardEpoch, Unit: 1, Loc: time.UTC}
|
||||
lo := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).Unix()
|
||||
hi := time.Date(2025, 1, 4, 0, 0, 0, 0, time.UTC).Unix()
|
||||
got := Bounded(context.Background(), validRange(strconv.FormatInt(lo, 10), strconv.FormatInt(hi, 10)), ep, time.Second)
|
||||
if got == nil || got.Hi-got.Lo != 3*86400 {
|
||||
t.Fatalf("epoch range = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedKeyRange(t *testing.T) {
|
||||
spec := Spec{Kind: "key", Column: "id"}
|
||||
got := Bounded(context.Background(), validRange("100", "1100"), spec, time.Second)
|
||||
if got == nil || got.Kind != "key" || got.Lo != 100 || got.Hi != 1100 {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedSkipsUnavailableMetrics(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
q RangeQuery
|
||||
spec Spec
|
||||
}{
|
||||
{"no metric", validRange("1", "10"), Spec{}},
|
||||
{"query error", func(context.Context, string) (sql.NullString, sql.NullString, error) {
|
||||
return sql.NullString{}, sql.NullString{}, context.DeadlineExceeded
|
||||
}, Spec{Kind: "key", Column: "id"}},
|
||||
{"null bounds", func(context.Context, string) (sql.NullString, sql.NullString, error) {
|
||||
return sql.NullString{}, sql.NullString{}, nil
|
||||
}, Spec{Kind: "key", Column: "id"}},
|
||||
{"empty table", validRange("5", "5"), Spec{Kind: "key", Column: "id"}},
|
||||
{"zero date", validRange("0000-00-00", "2025-01-01"), Spec{Kind: "day", ShardKind: schema.ShardDate, Loc: time.UTC}},
|
||||
{"string key", validRange("a", "z"), Spec{Kind: "key", Column: "id"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := Bounded(context.Background(), tc.q, tc.spec, time.Second); got != nil {
|
||||
t.Fatalf("%s: expected nil, got %+v", tc.name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedTimeoutDegrades(t *testing.T) {
|
||||
blocked := func(ctx context.Context, _ string) (sql.NullString, sql.NullString, error) {
|
||||
<-ctx.Done()
|
||||
return sql.NullString{}, sql.NullString{}, ctx.Err()
|
||||
}
|
||||
start := time.Now()
|
||||
got := Bounded(context.Background(), blocked, Spec{Kind: "key", Column: "id"}, 20*time.Millisecond)
|
||||
if got != nil {
|
||||
t.Fatalf("timed-out estimate must be nil, got %+v", got)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed > time.Second {
|
||||
t.Fatalf("timeout took too long: %s", elapsed)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
// Package lag checks replica delay before each fetch.
|
||||
package lag
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// Checker waits until a replica's delay is below Max and replication is
|
||||
// running.
|
||||
type Checker struct {
|
||||
db *sql.DB
|
||||
Max time.Duration
|
||||
Interval time.Duration
|
||||
OnWait func(reason string)
|
||||
}
|
||||
|
||||
// Open connects to the replica DSN.
|
||||
func Open(dsn string, max, interval time.Duration) (*Checker, error) {
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
if err := db.Ping(); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("connect replica: %w", err)
|
||||
}
|
||||
return &Checker{db: db, Max: max, Interval: interval}, nil
|
||||
}
|
||||
|
||||
func (c *Checker) Close() error { return c.db.Close() }
|
||||
|
||||
// Wait blocks until the replica lag is acceptable or ctx is cancelled.
|
||||
func (c *Checker) Wait(ctx context.Context) error {
|
||||
for {
|
||||
reason, ok, err := c.status(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok {
|
||||
return nil
|
||||
}
|
||||
if c.OnWait != nil {
|
||||
c.OnWait(reason)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(c.Interval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Checker) status(ctx context.Context) (string, bool, error) {
|
||||
fields, err := c.showStatus(ctx, "SHOW REPLICA STATUS")
|
||||
if err != nil || len(fields) == 0 {
|
||||
fields, err = c.showStatus(ctx, "SHOW SLAVE STATUS")
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
}
|
||||
if len(fields) == 0 {
|
||||
return "replication is not configured", false, nil
|
||||
}
|
||||
sqlRunning := yes(fields["Replica_SQL_Running"], fields["Slave_SQL_Running"])
|
||||
ioRunning := yes(fields["Replica_IO_Running"], fields["Slave_IO_Running"])
|
||||
if !sqlRunning || !ioRunning {
|
||||
return "replication threads are not running", false, nil
|
||||
}
|
||||
raw := firstNonEmpty(fields["Seconds_Behind_Source"], fields["Seconds_Behind_Master"])
|
||||
if raw == "" || strings.EqualFold(raw, "NULL") {
|
||||
return "replica lag is unknown", false, nil
|
||||
}
|
||||
secs, err := strconv.ParseFloat(raw, 64)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("cannot parse replica lag %q", raw), false, nil
|
||||
}
|
||||
lag := time.Duration(secs * float64(time.Second))
|
||||
if lag > c.Max {
|
||||
return fmt.Sprintf("replica lag %s exceeds %s", lag, c.Max), false, nil
|
||||
}
|
||||
return "", true, nil
|
||||
}
|
||||
|
||||
func (c *Checker) showStatus(ctx context.Context, q string) (map[string]string, error) {
|
||||
rows, err := c.db.QueryContext(ctx, q)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
cols, err := rows.Columns()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !rows.Next() {
|
||||
if err := rows.Err(); err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, err
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
vals := make([]sql.RawBytes, len(cols))
|
||||
ptrs := make([]any, len(cols))
|
||||
for i := range vals {
|
||||
ptrs[i] = &vals[i]
|
||||
}
|
||||
if err := rows.Scan(ptrs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]string, len(cols))
|
||||
for i, name := range cols {
|
||||
if vals[i] == nil {
|
||||
out[name] = "NULL"
|
||||
} else {
|
||||
out[name] = string(vals[i])
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func yes(vals ...string) bool {
|
||||
for _, v := range vals {
|
||||
if strings.EqualFold(v, "Yes") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,707 @@
|
||||
// Package progress renders human-facing run progress for long archives.
|
||||
//
|
||||
// A Reporter receives events from the single-threaded run loop and renders
|
||||
// them from a background ticker, so blocking phases (a slow scan, fsync, lag
|
||||
// waiting) still produce a heartbeat. Rendering never touches scanner or
|
||||
// writer state directly: the run loop copies what it needs into the Reporter.
|
||||
package progress
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// rateWindow is how far back a rate sample may sit to be considered current.
|
||||
const rateWindow = 5 * time.Second
|
||||
|
||||
// Estimate describes the numeric range used for percentage and ETA.
|
||||
// Kind is "day" (position is a day boundary in unix seconds) or "key"
|
||||
// (position is a numeric scan key).
|
||||
type Estimate struct {
|
||||
Kind string
|
||||
Lo, Hi float64
|
||||
}
|
||||
|
||||
// Options configures a Reporter.
|
||||
type Options struct {
|
||||
Out io.Writer // destination, default os.Stderr
|
||||
Enabled bool // whether progress lines are produced at all
|
||||
Live bool // render in place with carriage returns
|
||||
Interval time.Duration // minimum refresh interval, default 1s
|
||||
RowEvery int64 // extra refresh every N rows; 0 disables
|
||||
Width int // live line limit, default 100
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type sample struct {
|
||||
t time.Time
|
||||
rows int64
|
||||
pos float64
|
||||
hasPos bool
|
||||
}
|
||||
|
||||
type data struct {
|
||||
started time.Time
|
||||
phase string
|
||||
reason string
|
||||
phaseT time.Time
|
||||
base int64
|
||||
read int64
|
||||
pos float64
|
||||
hasPos bool
|
||||
originP float64
|
||||
originT time.Time
|
||||
hasOrig bool
|
||||
estimate *Estimate
|
||||
shard int64
|
||||
hasShard bool
|
||||
bytes int64
|
||||
hasBytes bool
|
||||
shards int64
|
||||
hasShardN bool
|
||||
samples []sample
|
||||
rowMark int64
|
||||
lastDraw time.Time
|
||||
}
|
||||
|
||||
func (d *data) rows() int64 { return d.base + d.read }
|
||||
|
||||
func (d *data) clone() data {
|
||||
c := *d
|
||||
c.samples = append([]sample(nil), d.samples...)
|
||||
return c
|
||||
}
|
||||
|
||||
// Reporter accumulates run metrics and renders them.
|
||||
type Reporter struct {
|
||||
out io.Writer
|
||||
now func() time.Time
|
||||
enabled bool
|
||||
live bool
|
||||
interval time.Duration
|
||||
rowEvery int64
|
||||
width int
|
||||
|
||||
mu sync.Mutex
|
||||
d data
|
||||
started bool
|
||||
running bool
|
||||
stopCh chan struct{}
|
||||
doneCh chan struct{}
|
||||
|
||||
wmu sync.Mutex
|
||||
lastWidth int
|
||||
}
|
||||
|
||||
// New builds a Reporter. A disabled Reporter still carries messages (warnings)
|
||||
// to Out but emits no progress lines.
|
||||
func New(o Options) *Reporter {
|
||||
if o.Out == nil {
|
||||
o.Out = os.Stderr
|
||||
}
|
||||
if o.Now == nil {
|
||||
o.Now = time.Now
|
||||
}
|
||||
if o.Interval <= 0 {
|
||||
o.Interval = time.Second
|
||||
}
|
||||
if o.Width <= 0 {
|
||||
o.Width = 100
|
||||
}
|
||||
return &Reporter{
|
||||
out: o.Out,
|
||||
now: o.Now,
|
||||
enabled: o.Enabled,
|
||||
live: o.Live,
|
||||
interval: o.Interval,
|
||||
rowEvery: o.RowEvery,
|
||||
width: o.Width,
|
||||
stopCh: make(chan struct{}),
|
||||
doneCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins the refresh goroutine. It is a no-op for a disabled Reporter.
|
||||
func (r *Reporter) Start() {
|
||||
r.mu.Lock()
|
||||
if r.started {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
r.started = true
|
||||
r.initLocked(r.now())
|
||||
if !r.enabled {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
r.running = true
|
||||
r.mu.Unlock()
|
||||
go r.loop()
|
||||
}
|
||||
|
||||
func (r *Reporter) initLocked(now time.Time) {
|
||||
r.d.started = now
|
||||
r.d.phaseT = now
|
||||
r.d.lastDraw = time.Time{}
|
||||
if r.rowEvery > 0 {
|
||||
r.d.rowMark = r.d.read / r.rowEvery
|
||||
}
|
||||
r.pushLocked(now)
|
||||
}
|
||||
|
||||
func (r *Reporter) loop() {
|
||||
t := time.NewTicker(r.interval)
|
||||
defer t.Stop()
|
||||
defer close(r.doneCh)
|
||||
for {
|
||||
select {
|
||||
case <-r.stopCh:
|
||||
return
|
||||
case <-t.C:
|
||||
r.maybeRender(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SetBase installs the row count carried over from a previous run. Resetting
|
||||
// the sample ring keeps the rate measured from this run's work rather than
|
||||
// jumping by the inherited count on the first chunk.
|
||||
func (r *Reporter) SetBase(n int64) {
|
||||
r.mu.Lock()
|
||||
r.d.base = n
|
||||
r.d.samples = nil
|
||||
r.pushLocked(r.now())
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// AddRows accounts for rows read in this run.
|
||||
func (r *Reporter) AddRows(n int64) {
|
||||
if n == 0 {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.d.read += n
|
||||
now := r.now()
|
||||
r.pushLocked(now)
|
||||
crossed := false
|
||||
if r.rowEvery > 0 {
|
||||
mark := r.d.read / r.rowEvery
|
||||
if mark > r.d.rowMark {
|
||||
r.d.rowMark = mark
|
||||
crossed = true
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
if crossed {
|
||||
// The threshold is an extra line at a chunk boundary; draw it inline so
|
||||
// it is not lost if the run finishes before the next tick.
|
||||
r.maybeRender(true)
|
||||
}
|
||||
}
|
||||
|
||||
// SetPosition publishes the current numeric position (day seconds or key).
|
||||
func (r *Reporter) SetPosition(pos float64) {
|
||||
r.mu.Lock()
|
||||
r.d.pos = pos
|
||||
r.d.hasPos = true
|
||||
if !r.d.hasOrig {
|
||||
r.d.originP = pos
|
||||
r.d.originT = r.now()
|
||||
r.d.hasOrig = true
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetEstimate publishes or clears the range used for percentage and ETA.
|
||||
func (r *Reporter) SetEstimate(e *Estimate) {
|
||||
r.mu.Lock()
|
||||
r.d.estimate = e
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetShard publishes the current shard sequence number.
|
||||
func (r *Reporter) SetShard(seq int64) {
|
||||
r.mu.Lock()
|
||||
r.d.shard = seq
|
||||
r.d.hasShard = true
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetBytes publishes bytes written so far.
|
||||
func (r *Reporter) SetBytes(n int64) {
|
||||
r.mu.Lock()
|
||||
r.d.bytes = n
|
||||
r.d.hasBytes = true
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetShards publishes the sealed shard count for the done summary.
|
||||
func (r *Reporter) SetShards(n int64) {
|
||||
r.mu.Lock()
|
||||
r.d.shards = n
|
||||
r.d.hasShardN = true
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// Phase records the current phase and an optional waiting reason. Repeating
|
||||
// the same phase (for example lag-wait reported on every poll) keeps the phase
|
||||
// clock running so its duration is visible.
|
||||
func (r *Reporter) Phase(name, reason string) {
|
||||
r.mu.Lock()
|
||||
if r.d.phase != name {
|
||||
r.d.phase = name
|
||||
r.d.phaseT = r.now()
|
||||
}
|
||||
r.d.reason = reason
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// Message writes a complete diagnostic line, first clearing any live line.
|
||||
func (r *Reporter) Message(format string, args ...any) {
|
||||
line := strings.TrimRight(fmt.Sprintf(format, args...), "\n")
|
||||
r.wmu.Lock()
|
||||
defer r.wmu.Unlock()
|
||||
if r.live {
|
||||
r.clearLocked()
|
||||
}
|
||||
fmt.Fprintln(r.out, line)
|
||||
}
|
||||
|
||||
// Stop ends refreshing and clears the live line without printing a summary.
|
||||
func (r *Reporter) Stop() {
|
||||
r.stopTicker()
|
||||
r.wmu.Lock()
|
||||
r.clearLocked()
|
||||
r.wmu.Unlock()
|
||||
}
|
||||
|
||||
// Done prints the final summary line on normal completion.
|
||||
func (r *Reporter) Done() {
|
||||
r.stopTicker()
|
||||
now := r.now()
|
||||
r.mu.Lock()
|
||||
s := r.d.clone()
|
||||
r.mu.Unlock()
|
||||
r.wmu.Lock()
|
||||
defer r.wmu.Unlock()
|
||||
r.clearLocked()
|
||||
if r.enabled {
|
||||
fmt.Fprintln(r.out, buildDone(s, now))
|
||||
}
|
||||
}
|
||||
|
||||
// Interrupted prints the final snapshot line after a signal.
|
||||
func (r *Reporter) Interrupted() {
|
||||
r.stopTicker()
|
||||
now := r.now()
|
||||
r.mu.Lock()
|
||||
s := r.d.clone()
|
||||
r.mu.Unlock()
|
||||
r.wmu.Lock()
|
||||
defer r.wmu.Unlock()
|
||||
r.clearLocked()
|
||||
if r.enabled {
|
||||
fmt.Fprintln(r.out, buildLine(s, now))
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reporter) stopTicker() {
|
||||
r.mu.Lock()
|
||||
if !r.running {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
r.running = false
|
||||
close(r.stopCh)
|
||||
done := r.doneCh
|
||||
r.mu.Unlock()
|
||||
<-done
|
||||
}
|
||||
|
||||
func (r *Reporter) pushLocked(now time.Time) {
|
||||
r.d.samples = append(r.d.samples, sample{
|
||||
t: now,
|
||||
rows: r.d.rows(),
|
||||
pos: r.d.pos,
|
||||
hasPos: r.d.hasPos,
|
||||
})
|
||||
cut := now.Add(-2 * rateWindow)
|
||||
i := 0
|
||||
for i < len(r.d.samples) && r.d.samples[i].t.Before(cut) {
|
||||
i++
|
||||
}
|
||||
if i > 0 {
|
||||
r.d.samples = append(r.d.samples[:0], r.d.samples[i:]...)
|
||||
}
|
||||
if len(r.d.samples) > 256 {
|
||||
r.d.samples = r.d.samples[len(r.d.samples)-256:]
|
||||
}
|
||||
}
|
||||
|
||||
// maybeRender draws a frame. force bypasses the interval throttle so a row
|
||||
// threshold can produce an extra line.
|
||||
func (r *Reporter) maybeRender(force bool) {
|
||||
if !r.enabled {
|
||||
return
|
||||
}
|
||||
now := r.now()
|
||||
r.mu.Lock()
|
||||
if !force && !r.d.lastDraw.IsZero() && now.Sub(r.d.lastDraw) < r.interval {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
r.d.lastDraw = now
|
||||
s := r.d.clone()
|
||||
r.mu.Unlock()
|
||||
|
||||
var line string
|
||||
if r.live {
|
||||
line = r.buildLive(s, now)
|
||||
} else {
|
||||
line = buildLine(s, now)
|
||||
}
|
||||
|
||||
r.wmu.Lock()
|
||||
defer r.wmu.Unlock()
|
||||
if r.live {
|
||||
if line == "" {
|
||||
line = " "
|
||||
}
|
||||
pad := 0
|
||||
if r.lastWidth > len(line) {
|
||||
pad = r.lastWidth - len(line)
|
||||
}
|
||||
fmt.Fprintf(r.out, "\r%s%s", line, strings.Repeat(" ", pad))
|
||||
r.lastWidth = len(line)
|
||||
return
|
||||
}
|
||||
fmt.Fprintln(r.out, line)
|
||||
}
|
||||
|
||||
func (r *Reporter) clearLocked() {
|
||||
if r.live && r.lastWidth > 0 {
|
||||
fmt.Fprintf(r.out, "\r%s\r", strings.Repeat(" ", r.lastWidth))
|
||||
r.lastWidth = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reporter) buildLive(s data, now time.Time) string {
|
||||
// Field priority when narrowing: drop bytes, then the shard number, then
|
||||
// shrink the bar. The line never wraps.
|
||||
type shape struct {
|
||||
bar int
|
||||
bytes bool
|
||||
shard bool
|
||||
}
|
||||
shapes := []shape{
|
||||
{20, true, true},
|
||||
{20, false, true},
|
||||
{20, false, false},
|
||||
{12, false, false},
|
||||
{8, false, false},
|
||||
{0, false, false},
|
||||
}
|
||||
var last string
|
||||
for _, sh := range shapes {
|
||||
last = liveLine(s, now, sh.bar, sh.bytes, sh.shard)
|
||||
if len(last) <= r.width {
|
||||
return last
|
||||
}
|
||||
}
|
||||
return truncate(last, r.width)
|
||||
}
|
||||
|
||||
func liveLine(s data, now time.Time, barWidth int, showBytes, showShard bool) string {
|
||||
var parts []string
|
||||
pct, eta, hasEta, hasEst := calc(s, now)
|
||||
if hasEst {
|
||||
if barWidth > 0 {
|
||||
filled := int(pct*float64(barWidth) + 0.5)
|
||||
parts = append(parts, "["+strings.Repeat("#", filled)+strings.Repeat("-", barWidth-filled)+"]")
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("~%.0f%%", pct*100))
|
||||
if s.estimate.Kind == "day" {
|
||||
cur, total := dayProgress(s)
|
||||
parts = append(parts, fmt.Sprintf("day %d/%d", cur, total))
|
||||
}
|
||||
}
|
||||
parts = append(parts, humanRows(s.rows())+" rows")
|
||||
if rr, ok := rate(s.samples, now, false); ok && rr > 0 {
|
||||
parts = append(parts, humanRate(rr))
|
||||
}
|
||||
if s.phase != "" {
|
||||
parts = append(parts, fmt.Sprintf("%s %s", s.phase, humanDuration(now.Sub(s.phaseT))))
|
||||
}
|
||||
if s.reason != "" {
|
||||
parts = append(parts, "("+s.reason+")")
|
||||
}
|
||||
if showShard && s.hasShard {
|
||||
parts = append(parts, fmt.Sprintf("shard %d", s.shard))
|
||||
}
|
||||
if showBytes && s.hasBytes {
|
||||
parts = append(parts, humanBytes(s.bytes))
|
||||
}
|
||||
if hasEta {
|
||||
parts = append(parts, "eta ~"+humanDuration(eta))
|
||||
}
|
||||
parts = append(parts, "elapsed "+humanDuration(now.Sub(s.started)))
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func buildLine(s data, now time.Time) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("progress: rows=")
|
||||
b.WriteString(strconv.FormatInt(s.rows(), 10))
|
||||
b.WriteString(" rate=")
|
||||
if rr, ok := rate(s.samples, now, false); ok && rr > 0 {
|
||||
b.WriteString(humanRate(rr))
|
||||
} else {
|
||||
b.WriteString(humanRate(cumulativeRows(s, now)))
|
||||
}
|
||||
b.WriteString(" phase=")
|
||||
if s.phase == "" {
|
||||
b.WriteString("start")
|
||||
} else {
|
||||
b.WriteString(s.phase)
|
||||
}
|
||||
b.WriteString(" phase_elapsed=")
|
||||
b.WriteString(humanDuration(now.Sub(s.phaseT)))
|
||||
b.WriteString(" elapsed=")
|
||||
b.WriteString(humanDuration(now.Sub(s.started)))
|
||||
if s.reason != "" {
|
||||
b.WriteString(" reason=")
|
||||
b.WriteString(strings.Join(strings.Fields(s.reason), "_"))
|
||||
}
|
||||
if s.hasShard {
|
||||
fmt.Fprintf(&b, " shard=%d", s.shard)
|
||||
}
|
||||
if s.hasBytes {
|
||||
fmt.Fprintf(&b, " bytes=%d", s.bytes)
|
||||
}
|
||||
pct, eta, hasEta, hasEst := calc(s, now)
|
||||
if !hasEst {
|
||||
b.WriteString(" estimate=none")
|
||||
return b.String()
|
||||
}
|
||||
fmt.Fprintf(&b, " estimate=%s pct=~%.1f", s.estimate.Kind, pct*100)
|
||||
if s.estimate.Kind == "day" {
|
||||
cur, total := dayProgress(s)
|
||||
fmt.Fprintf(&b, " day=%d/%d", cur, total)
|
||||
}
|
||||
if hasEta {
|
||||
b.WriteString(" eta=~")
|
||||
b.WriteString(humanDuration(eta))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func buildDone(s data, now time.Time) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("done rows=")
|
||||
b.WriteString(strconv.FormatInt(s.rows(), 10))
|
||||
b.WriteString(" elapsed=")
|
||||
b.WriteString(humanDuration(now.Sub(s.started)))
|
||||
if s.hasShardN {
|
||||
fmt.Fprintf(&b, " shards=%d", s.shards)
|
||||
}
|
||||
if s.hasBytes {
|
||||
fmt.Fprintf(&b, " bytes=%d", s.bytes)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// calc returns percentage, ETA and whether an estimate is displayable.
|
||||
func calc(s data, now time.Time) (pct float64, eta time.Duration, hasEta, ok bool) {
|
||||
if s.estimate == nil || !s.hasPos {
|
||||
return 0, 0, false, false
|
||||
}
|
||||
span := s.estimate.Hi - s.estimate.Lo
|
||||
if span <= 0 {
|
||||
return 0, 0, false, false
|
||||
}
|
||||
pct = (s.pos - s.estimate.Lo) / span
|
||||
if pct < 0 {
|
||||
pct = 0
|
||||
}
|
||||
if pct > 1 {
|
||||
pct = 1
|
||||
}
|
||||
if pr, rok := positionRate(s, now); rok && pr > 0 {
|
||||
remaining := s.estimate.Hi - s.pos
|
||||
if remaining < 0 {
|
||||
remaining = 0
|
||||
}
|
||||
secs := remaining / pr
|
||||
if secs >= 0 && secs < 9e9 {
|
||||
eta = time.Duration(secs * float64(time.Second))
|
||||
hasEta = true
|
||||
}
|
||||
}
|
||||
return pct, eta, hasEta, true
|
||||
}
|
||||
|
||||
func dayProgress(s data) (int, int) {
|
||||
const day = 86400.0
|
||||
total := int((s.estimate.Hi-s.estimate.Lo)/day) + 1
|
||||
cur := int((s.pos-s.estimate.Lo)/day) + 1
|
||||
if cur < 1 {
|
||||
cur = 1
|
||||
}
|
||||
if cur > total {
|
||||
cur = total
|
||||
}
|
||||
return cur, total
|
||||
}
|
||||
|
||||
func rate(samples []sample, now time.Time, usePos bool) (float64, bool) {
|
||||
cut := now.Add(-rateWindow)
|
||||
var first *sample
|
||||
for i := range samples {
|
||||
if usePos && !samples[i].hasPos {
|
||||
continue
|
||||
}
|
||||
if !samples[i].t.Before(cut) {
|
||||
first = &samples[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if first == nil {
|
||||
for i := len(samples) - 1; i >= 0; i-- {
|
||||
if !usePos || samples[i].hasPos {
|
||||
first = &samples[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if first == nil || len(samples) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
last := &samples[len(samples)-1]
|
||||
if usePos && !last.hasPos {
|
||||
return 0, false
|
||||
}
|
||||
if first == last {
|
||||
return 0, false
|
||||
}
|
||||
dt := last.t.Sub(first.t).Seconds()
|
||||
if dt <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
var d float64
|
||||
if usePos {
|
||||
d = last.pos - first.pos
|
||||
} else {
|
||||
d = float64(last.rows - first.rows)
|
||||
}
|
||||
if d < 0 {
|
||||
return 0, false
|
||||
}
|
||||
return d / dt, true
|
||||
}
|
||||
|
||||
func positionRate(s data, now time.Time) (float64, bool) {
|
||||
if v, ok := rate(s.samples, now, true); ok {
|
||||
return v, true
|
||||
}
|
||||
if s.hasOrig && now.After(s.originT) {
|
||||
return (s.pos - s.originP) / now.Sub(s.originT).Seconds(), true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func cumulativeRows(s data, now time.Time) float64 {
|
||||
el := now.Sub(s.started).Seconds()
|
||||
if el <= 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(s.read) / el
|
||||
}
|
||||
|
||||
func humanRows(n int64) string {
|
||||
switch {
|
||||
case n >= 1_000_000_000:
|
||||
return fmt.Sprintf("%.2fG", float64(n)/1e9)
|
||||
case n >= 1_000_000:
|
||||
return fmt.Sprintf("%.2fM", float64(n)/1e6)
|
||||
case n >= 10_000:
|
||||
return fmt.Sprintf("%.1fk", float64(n)/1e3)
|
||||
default:
|
||||
return strconv.FormatInt(n, 10)
|
||||
}
|
||||
}
|
||||
|
||||
func humanRate(v float64) string {
|
||||
switch {
|
||||
case v >= 1_000_000:
|
||||
return fmt.Sprintf("%.2fM/s", v/1e6)
|
||||
case v >= 10_000:
|
||||
return fmt.Sprintf("%.1fk/s", v/1e3)
|
||||
case v >= 1:
|
||||
return fmt.Sprintf("%.0f/s", v)
|
||||
default:
|
||||
return "0/s"
|
||||
}
|
||||
}
|
||||
|
||||
func humanBytes(n int64) string {
|
||||
const unit = 1024
|
||||
if n < unit {
|
||||
return fmt.Sprintf("%dB", n)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for v := n / unit; v >= unit; v /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f%ciB", float64(n)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
||||
func humanDuration(d time.Duration) string {
|
||||
if d < 0 {
|
||||
d = 0
|
||||
}
|
||||
if d < time.Second {
|
||||
return fmt.Sprintf("%dms", d.Milliseconds())
|
||||
}
|
||||
secs := int(d.Seconds())
|
||||
switch {
|
||||
case secs < 60:
|
||||
return fmt.Sprintf("%ds", secs)
|
||||
case secs < 3600:
|
||||
return fmt.Sprintf("%dm%02ds", secs/60, secs%60)
|
||||
default:
|
||||
return fmt.Sprintf("%dh%02dm", secs/3600, (secs%3600)/60)
|
||||
}
|
||||
}
|
||||
|
||||
func truncate(s string, max int) string {
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
if max <= 0 {
|
||||
return ""
|
||||
}
|
||||
runes := []rune(s)
|
||||
if len(runes) <= max {
|
||||
return s
|
||||
}
|
||||
return string(runes[:max])
|
||||
}
|
||||
|
||||
// IsTerminal reports whether w is a character device (a terminal).
|
||||
func IsTerminal(w io.Writer) bool {
|
||||
f, ok := w.(*os.File)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
st, err := f.Stat()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return st.Mode()&os.ModeCharDevice != 0
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package progress
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type clock struct{ t time.Time }
|
||||
|
||||
func (c *clock) now() time.Time { return c.t }
|
||||
func (c *clock) add(d time.Duration) { c.t = c.t.Add(d) }
|
||||
|
||||
func newTestReporter(t *testing.T, live, enabled bool, opts ...func(*Options)) (*Reporter, *bytes.Buffer, *clock) {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
clk := &clock{t: time.Unix(1_700_000_000, 0)}
|
||||
o := Options{Out: &buf, Enabled: enabled, Live: live, Interval: time.Second, Now: clk.now}
|
||||
for _, fn := range opts {
|
||||
fn(&o)
|
||||
}
|
||||
r := New(o)
|
||||
r.mu.Lock()
|
||||
r.started = true
|
||||
r.initLocked(clk.t)
|
||||
r.mu.Unlock()
|
||||
return r, &buf, clk
|
||||
}
|
||||
|
||||
func TestLiveLineHasRequiredFields(t *testing.T) {
|
||||
r, buf, clk := newTestReporter(t, true, true)
|
||||
r.SetEstimate(&Estimate{Kind: "day", Lo: 0, Hi: 10 * 86400})
|
||||
r.Phase("scan", "")
|
||||
r.SetShard(12)
|
||||
r.SetPosition(3 * 86400)
|
||||
r.AddRows(1000)
|
||||
clk.add(time.Second)
|
||||
r.SetPosition(4 * 86400)
|
||||
r.AddRows(1000)
|
||||
r.maybeRender(true)
|
||||
|
||||
out := buf.String()
|
||||
if !strings.HasPrefix(out, "\r") {
|
||||
t.Fatalf("live line must start with carriage return: %q", out)
|
||||
}
|
||||
if strings.Contains(out, "\n") {
|
||||
t.Fatalf("live line must not emit a newline: %q", out)
|
||||
}
|
||||
for _, want := range []string{"rows", "~40%", "day 5/11", "shard 12", "scan", "eta ~", "elapsed"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("live line %q missing %q", out, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveLineOmitsEstimateWhenUnavailable(t *testing.T) {
|
||||
r, buf, _ := newTestReporter(t, true, true)
|
||||
r.Phase("scan", "")
|
||||
r.AddRows(10)
|
||||
r.maybeRender(true)
|
||||
out := buf.String()
|
||||
if strings.Contains(out, "%") || strings.Contains(out, "eta") {
|
||||
t.Fatalf("unexpected estimate in %q", out)
|
||||
}
|
||||
if !strings.Contains(out, "rows") {
|
||||
t.Fatalf("missing rows in %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLineRenderingIsStableKeyValue(t *testing.T) {
|
||||
r, buf, clk := newTestReporter(t, false, true)
|
||||
r.SetEstimate(&Estimate{Kind: "key", Lo: 0, Hi: 1000})
|
||||
r.Phase("db", "")
|
||||
r.SetPosition(100)
|
||||
r.AddRows(500)
|
||||
clk.add(time.Second)
|
||||
r.SetPosition(500)
|
||||
r.AddRows(500)
|
||||
r.maybeRender(true)
|
||||
|
||||
out := buf.String()
|
||||
if strings.Contains(out, "\r") {
|
||||
t.Fatalf("line mode must not use carriage returns: %q", out)
|
||||
}
|
||||
if !strings.HasSuffix(out, "\n") {
|
||||
t.Fatalf("line mode must end with a newline: %q", out)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"progress: rows=1000", "rate=", "phase=db", "phase_elapsed=", "elapsed=",
|
||||
"estimate=key", "pct=~50.0", "eta=~",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("line %q missing %q", out, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLineRenderingMarksEstimateNone(t *testing.T) {
|
||||
r, buf, _ := newTestReporter(t, false, true)
|
||||
r.Phase("scan", "")
|
||||
r.AddRows(100)
|
||||
r.maybeRender(true)
|
||||
if out := buf.String(); !strings.Contains(out, "estimate=none") {
|
||||
t.Fatalf("want estimate=none, got %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntervalThrottle(t *testing.T) {
|
||||
r, buf, clk := newTestReporter(t, false, true)
|
||||
r.maybeRender(true)
|
||||
if n := strings.Count(buf.String(), "\n"); n != 1 {
|
||||
t.Fatalf("first frame not drawn: %q", buf.String())
|
||||
}
|
||||
clk.add(200 * time.Millisecond)
|
||||
r.maybeRender(false)
|
||||
if n := strings.Count(buf.String(), "\n"); n != 1 {
|
||||
t.Fatalf("throttled frame drawn early: %q", buf.String())
|
||||
}
|
||||
clk.add(time.Second)
|
||||
r.maybeRender(false)
|
||||
if n := strings.Count(buf.String(), "\n"); n != 2 {
|
||||
t.Fatalf("frame after interval not drawn: %q", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRowThresholdRenders(t *testing.T) {
|
||||
r, buf, _ := newTestReporter(t, false, true, func(o *Options) { o.RowEvery = 100 })
|
||||
r.AddRows(100)
|
||||
if n := strings.Count(buf.String(), "\n"); n != 1 {
|
||||
t.Fatalf("crossing the first threshold did not render: %q", buf.String())
|
||||
}
|
||||
r.AddRows(50)
|
||||
if n := strings.Count(buf.String(), "\n"); n != 1 {
|
||||
t.Fatalf("sub-threshold rows must not render: %q", buf.String())
|
||||
}
|
||||
r.AddRows(50)
|
||||
if n := strings.Count(buf.String(), "\n"); n != 2 {
|
||||
t.Fatalf("crossing the next threshold did not render: %q", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageClearsLiveLine(t *testing.T) {
|
||||
r, buf, _ := newTestReporter(t, true, true)
|
||||
r.AddRows(1000)
|
||||
r.maybeRender(true)
|
||||
width := r.lastWidth
|
||||
buf.Reset()
|
||||
r.Message("warning: %s", "something")
|
||||
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, "\r"+strings.Repeat(" ", width)+"\r") {
|
||||
t.Fatalf("message did not clear the live line: %q", out)
|
||||
}
|
||||
if !strings.Contains(out, "warning: something\n") {
|
||||
t.Fatalf("message missing: %q", out)
|
||||
}
|
||||
if strings.Contains(out, "rows=") {
|
||||
t.Fatalf("live line leaked into message: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShortLineIsPadded(t *testing.T) {
|
||||
r, buf, clk := newTestReporter(t, true, true)
|
||||
r.SetEstimate(&Estimate{Kind: "day", Lo: 0, Hi: 100 * 86400})
|
||||
r.SetPosition(50 * 86400)
|
||||
r.AddRows(1000)
|
||||
r.maybeRender(true)
|
||||
first := r.lastWidth
|
||||
|
||||
buf.Reset()
|
||||
r.SetEstimate(nil)
|
||||
clk.add(2 * time.Second)
|
||||
r.AddRows(1000)
|
||||
r.maybeRender(true)
|
||||
second := buf.String()
|
||||
if r.lastWidth >= first {
|
||||
t.Fatalf("second frame should be shorter: %d -> %d (%q)", first, r.lastWidth, second)
|
||||
}
|
||||
if !strings.HasSuffix(second, strings.Repeat(" ", first-r.lastWidth)) {
|
||||
t.Fatalf("short frame not padded to previous width: %q", second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoneSummary(t *testing.T) {
|
||||
r, buf, clk := newTestReporter(t, false, true)
|
||||
r.SetShard(3)
|
||||
r.SetShards(3)
|
||||
r.SetBytes(4096)
|
||||
r.AddRows(1500)
|
||||
clk.add(3 * time.Second)
|
||||
r.Done()
|
||||
out := buf.String()
|
||||
for _, want := range []string{"done rows=1500", "elapsed=3s", "shards=3", "bytes=4096"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("done line %q missing %q", out, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInterruptedSnapshot(t *testing.T) {
|
||||
r, buf, _ := newTestReporter(t, true, true)
|
||||
r.Phase("file", "")
|
||||
r.AddRows(42)
|
||||
r.maybeRender(true)
|
||||
buf.Reset()
|
||||
r.Interrupted()
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, "progress: rows=42") {
|
||||
t.Fatalf("final snapshot missing: %q", out)
|
||||
}
|
||||
if !strings.Contains(out, "phase=file") {
|
||||
t.Fatalf("final snapshot missing phase: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDisabledReporterIsSilentButMessagesWork(t *testing.T) {
|
||||
r, buf, _ := newTestReporter(t, false, false)
|
||||
r.AddRows(10)
|
||||
r.maybeRender(true)
|
||||
r.Done()
|
||||
if buf.Len() != 0 {
|
||||
t.Fatalf("disabled reporter emitted progress: %q", buf.String())
|
||||
}
|
||||
r.Message("warning: still visible")
|
||||
if !strings.Contains(buf.String(), "warning: still visible") {
|
||||
t.Fatalf("messages must survive disabled progress: %q", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepeatedPhaseKeepsClock(t *testing.T) {
|
||||
r, buf, clk := newTestReporter(t, false, true)
|
||||
r.Phase("lag-wait", "reason a")
|
||||
clk.add(3 * time.Second)
|
||||
r.Phase("lag-wait", "reason b")
|
||||
r.maybeRender(true)
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, "phase=lag-wait") || !strings.Contains(out, "phase_elapsed=3s") {
|
||||
t.Fatalf("repeated phase reset its clock: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLagWaitShowsReason(t *testing.T) {
|
||||
r, buf, _ := newTestReporter(t, false, true)
|
||||
r.Phase("lag-wait", "replica lag 3s exceeds 1s")
|
||||
r.AddRows(10)
|
||||
r.maybeRender(true)
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, "phase=lag-wait") || !strings.Contains(out, "reason=replica_lag_3s_exceeds_1s") {
|
||||
t.Fatalf("lag reason missing: %q", out)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
// Package purge deletes archived rows from the source table, either by an
|
||||
// explicit key IN list or by key range.
|
||||
package purge
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"archer/internal/row"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
// Execer is satisfied by *sql.DB, *sql.Tx and *sql.Conn.
|
||||
type Execer interface {
|
||||
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
|
||||
}
|
||||
|
||||
const maxParams = 1000
|
||||
|
||||
// Purger deletes rows chunk by chunk.
|
||||
type Purger struct {
|
||||
DB *sql.DB
|
||||
DBName string
|
||||
Table string
|
||||
KeyCols []string
|
||||
Where string
|
||||
Bulk bool
|
||||
SafeAutoInc bool
|
||||
AutoIncKey bool
|
||||
KeyIdx []int
|
||||
}
|
||||
|
||||
// New builds a Purger. keyIdx is unused for deletion but kept for symmetry
|
||||
// with the scan configuration.
|
||||
func New(db *sql.DB, dbName, table string, keyCols []string, where string, bulk, safe, autoInc bool) *Purger {
|
||||
return &Purger{
|
||||
DB: db,
|
||||
DBName: dbName,
|
||||
Table: table,
|
||||
KeyCols: keyCols,
|
||||
Where: where,
|
||||
Bulk: bulk,
|
||||
SafeAutoInc: safe,
|
||||
AutoIncKey: autoInc,
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteChunk removes the rows read in this chunk using exec. It returns the
|
||||
// number of affected rows.
|
||||
func (p *Purger) DeleteChunk(ctx context.Context, exec Execer, c *row.Chunk) (int64, error) {
|
||||
if len(c.Keys) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
keys := c.Keys
|
||||
var maxKey *row.KeyPart
|
||||
if p.SafeAutoInc && p.AutoIncKey && len(p.KeyCols) == 1 {
|
||||
v, err := p.currentMax(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
maxKey = v
|
||||
}
|
||||
|
||||
if p.Bulk && maxKey == nil {
|
||||
return p.deleteRange(ctx, exec, c)
|
||||
}
|
||||
return p.deleteInList(ctx, exec, keys, maxKey)
|
||||
}
|
||||
|
||||
func (p *Purger) table() string {
|
||||
return sqlutil.Qualified(p.DBName, p.Table)
|
||||
}
|
||||
|
||||
func (p *Purger) currentMax(ctx context.Context) (*row.KeyPart, error) {
|
||||
q := fmt.Sprintf("SELECT MAX(%s) FROM %s", sqlutil.Ident(p.KeyCols[0]), p.table())
|
||||
var v sql.NullString
|
||||
if err := p.DB.QueryRowContext(ctx, q).Scan(&v); err != nil {
|
||||
return nil, fmt.Errorf("read max auto-increment key: %w", err)
|
||||
}
|
||||
if !v.Valid {
|
||||
return nil, nil
|
||||
}
|
||||
return &row.KeyPart{Val: v.String}, nil
|
||||
}
|
||||
|
||||
func (p *Purger) deleteRange(ctx context.Context, exec Execer, c *row.Chunk) (int64, error) {
|
||||
lo, hi := c.FirstKey, c.LastKey
|
||||
if lo.IsZero() || hi.IsZero() {
|
||||
return 0, nil
|
||||
}
|
||||
var (
|
||||
q string
|
||||
args []any
|
||||
)
|
||||
if len(p.KeyCols) == 1 {
|
||||
q = fmt.Sprintf("DELETE FROM %s WHERE %s BETWEEN ? AND ? AND (%s)",
|
||||
p.table(), sqlutil.Ident(p.KeyCols[0]), p.Where)
|
||||
args = []any{lo[0].DriverValue(), hi[0].DriverValue()}
|
||||
} else {
|
||||
q = fmt.Sprintf("DELETE FROM %s WHERE %s >= %s AND %s <= %s AND (%s)",
|
||||
p.table(), sqlutil.RowTuple(p.KeyCols), sqlutil.ParamTuple(len(p.KeyCols)),
|
||||
sqlutil.RowTuple(p.KeyCols), sqlutil.ParamTuple(len(p.KeyCols)), p.Where)
|
||||
args = append(args, lo.Args()...)
|
||||
args = append(args, hi.Args()...)
|
||||
}
|
||||
res, err := exec.ExecContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("bulk delete: %w", err)
|
||||
}
|
||||
return res.RowsAffected()
|
||||
}
|
||||
|
||||
func (p *Purger) deleteInList(ctx context.Context, exec Execer, keys []row.Key, maxKey *row.KeyPart) (int64, error) {
|
||||
batch := maxParams / len(p.KeyCols)
|
||||
if batch < 1 {
|
||||
batch = 1
|
||||
}
|
||||
var total int64
|
||||
for start := 0; start < len(keys); start += batch {
|
||||
end := start + batch
|
||||
if end > len(keys) {
|
||||
end = len(keys)
|
||||
}
|
||||
var args [][]any
|
||||
for _, k := range keys[start:end] {
|
||||
if maxKey != nil && keyEquals(k[len(k)-1], *maxKey) {
|
||||
continue
|
||||
}
|
||||
args = append(args, k.Args())
|
||||
}
|
||||
if len(args) == 0 {
|
||||
continue
|
||||
}
|
||||
q := fmt.Sprintf("DELETE FROM %s WHERE %s AND (%s)",
|
||||
p.table(), sqlutil.InList(p.KeyCols, len(args)), p.Where)
|
||||
flat := make([]any, 0, len(args)*len(p.KeyCols))
|
||||
for _, a := range args {
|
||||
flat = append(flat, a...)
|
||||
}
|
||||
res, err := exec.ExecContext(ctx, q, flat...)
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("delete chunk: %w", err)
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
total += n
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func keyEquals(a, b row.KeyPart) bool {
|
||||
if a.Null || b.Null {
|
||||
return a.Null && b.Null
|
||||
}
|
||||
if a.Val == b.Val {
|
||||
return true
|
||||
}
|
||||
ai, err1 := strconv.ParseInt(a.Val, 10, 64)
|
||||
bi, err2 := strconv.ParseInt(b.Val, 10, 64)
|
||||
return err1 == nil && err2 == nil && ai == bi
|
||||
}
|
||||
|
||||
// DryRunSQL returns representative delete statements.
|
||||
func (p *Purger) DryRunSQL() []string {
|
||||
in := fmt.Sprintf("DELETE FROM %s WHERE %s IN (?, ?) AND (%s)",
|
||||
p.table(), sqlutil.RowTuple(p.KeyCols), p.Where)
|
||||
if len(p.KeyCols) == 1 {
|
||||
in = fmt.Sprintf("DELETE FROM %s WHERE %s IN (?, ?) AND (%s)",
|
||||
p.table(), sqlutil.Ident(p.KeyCols[0]), p.Where)
|
||||
}
|
||||
out := []string{in}
|
||||
if p.Bulk {
|
||||
if len(p.KeyCols) == 1 {
|
||||
out = append(out, fmt.Sprintf("DELETE FROM %s WHERE %s BETWEEN ? AND ? AND (%s) -- --bulk-delete",
|
||||
p.table(), sqlutil.Ident(p.KeyCols[0]), p.Where))
|
||||
} else {
|
||||
out = append(out, fmt.Sprintf("DELETE FROM %s WHERE %s >= %s AND %s <= %s AND (%s) -- --bulk-delete",
|
||||
p.table(), sqlutil.RowTuple(p.KeyCols), sqlutil.ParamTuple(len(p.KeyCols)),
|
||||
sqlutil.RowTuple(p.KeyCols), sqlutil.ParamTuple(len(p.KeyCols)), p.Where))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package row
|
||||
|
||||
import "bytes"
|
||||
|
||||
// csvEncoder emits RFC4180 CSV with a column-name header and CRLF line
|
||||
// endings. NULL is written as the configured token and empty strings as empty
|
||||
// fields.
|
||||
type csvEncoder struct {
|
||||
cols []Column
|
||||
idx []int
|
||||
null string
|
||||
}
|
||||
|
||||
func (e *csvEncoder) Header() []byte {
|
||||
var b bytes.Buffer
|
||||
for n, i := range e.idx {
|
||||
if n > 0 {
|
||||
b.WriteByte(',')
|
||||
}
|
||||
writeCSVField(&b, []byte(e.cols[i].Name))
|
||||
}
|
||||
b.WriteString("\r\n")
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func (e *csvEncoder) Encode(dst []byte, r Row) ([]byte, error) {
|
||||
for n, i := range e.idx {
|
||||
if n > 0 {
|
||||
dst = append(dst, ',')
|
||||
}
|
||||
v := r.Values[i]
|
||||
if v.IsNull() {
|
||||
dst = append(dst, e.null...)
|
||||
continue
|
||||
}
|
||||
dst = appendCSVField(dst, v.Raw)
|
||||
}
|
||||
return append(dst, '\r', '\n'), nil
|
||||
}
|
||||
|
||||
func needsCSVQuote(b []byte) bool {
|
||||
for _, c := range b {
|
||||
switch c {
|
||||
case ',', '"', '\r', '\n':
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func appendCSVField(dst, field []byte) []byte {
|
||||
if !needsCSVQuote(field) {
|
||||
return append(dst, field...)
|
||||
}
|
||||
dst = append(dst, '"')
|
||||
for _, c := range field {
|
||||
if c == '"' {
|
||||
dst = append(dst, '"', '"')
|
||||
continue
|
||||
}
|
||||
dst = append(dst, c)
|
||||
}
|
||||
return append(dst, '"')
|
||||
}
|
||||
|
||||
func writeCSVField(b *bytes.Buffer, field []byte) {
|
||||
b.Write(appendCSVField(nil, field))
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package row
|
||||
|
||||
// dumpEncoder emits MySQL LOAD DATA compatible TSV. Fields are separated by
|
||||
// TAB, rows by LF, NULL is \N and the escape table matches
|
||||
// SELECT ... INTO OUTFILE with the default FIELDS ESCAPED BY '\\'.
|
||||
type dumpEncoder struct {
|
||||
idx []int
|
||||
}
|
||||
|
||||
func (e *dumpEncoder) Header() []byte { return nil }
|
||||
|
||||
func (e *dumpEncoder) Encode(dst []byte, r Row) ([]byte, error) {
|
||||
for n, i := range e.idx {
|
||||
if n > 0 {
|
||||
dst = append(dst, '\t')
|
||||
}
|
||||
v := r.Values[i]
|
||||
if v.IsNull() {
|
||||
dst = append(dst, '\\', 'N')
|
||||
continue
|
||||
}
|
||||
for _, b := range v.Raw {
|
||||
switch b {
|
||||
case '\\':
|
||||
dst = append(dst, '\\', '\\')
|
||||
case '\t':
|
||||
dst = append(dst, '\\', 't')
|
||||
case '\n':
|
||||
dst = append(dst, '\\', 'n')
|
||||
case '\r':
|
||||
dst = append(dst, '\\', 'r')
|
||||
case 0:
|
||||
dst = append(dst, '\\', '0')
|
||||
case '\b':
|
||||
dst = append(dst, '\\', 'b')
|
||||
case 0x1a:
|
||||
dst = append(dst, '\\', 'Z')
|
||||
default:
|
||||
dst = append(dst, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
return append(dst, '\n'), nil
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package row
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Options carries encoder-level knobs shared by all formats.
|
||||
type Options struct {
|
||||
CSVNull string
|
||||
}
|
||||
|
||||
// Encoder appends the textual representation of a row to dst. It is not safe
|
||||
// for concurrent use; each shard owns its own encoder.
|
||||
type Encoder interface {
|
||||
// Header returns bytes that must be written at the start of every shard.
|
||||
Header() []byte
|
||||
// Encode appends one row to dst.
|
||||
Encode(dst []byte, r Row) ([]byte, error)
|
||||
}
|
||||
|
||||
// NewEncoder builds an encoder for the given format. cols describes every
|
||||
// column in the row (select order) and idx selects which of them are emitted,
|
||||
// in output order.
|
||||
func NewEncoder(format string, cols []Column, idx []int, opts Options) (Encoder, error) {
|
||||
if len(idx) == 0 {
|
||||
idx = make([]int, len(cols))
|
||||
for i := range cols {
|
||||
idx[i] = i
|
||||
}
|
||||
}
|
||||
switch format {
|
||||
case "dump":
|
||||
return &dumpEncoder{idx: idx}, nil
|
||||
case "csv":
|
||||
return &csvEncoder{cols: cols, idx: idx, null: opts.CSVNull}, nil
|
||||
case "jsonl":
|
||||
return &jsonlEncoder{cols: cols, idx: idx}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported format %q", format)
|
||||
}
|
||||
}
|
||||
|
||||
// FormatExt returns the file extension for a format.
|
||||
func FormatExt(format string) string {
|
||||
switch format {
|
||||
case "dump":
|
||||
return "dump"
|
||||
case "csv":
|
||||
return "csv"
|
||||
case "jsonl":
|
||||
return "jsonl"
|
||||
}
|
||||
return format
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package row
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func vals(raw ...any) Row {
|
||||
r := Row{}
|
||||
for _, v := range raw {
|
||||
switch x := v.(type) {
|
||||
case nil:
|
||||
r.Values = append(r.Values, Value{})
|
||||
case string:
|
||||
r.Values = append(r.Values, Value{Raw: []byte(x)})
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func cols(names ...string) []Column {
|
||||
out := make([]Column, len(names))
|
||||
for i, n := range names {
|
||||
out[i] = Column{Name: n, Type: "varchar"}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestDumpEncoding(t *testing.T) {
|
||||
enc, err := NewEncoder("dump", cols("a", "b", "c"), nil, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := vals("x\ty", nil, `a\nb`)
|
||||
got, err := enc.Encode(nil, r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "x\\ty\t\\N\ta\\\\nb\n"
|
||||
if string(got) != want {
|
||||
t.Fatalf("got %q want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDumpEscapesControlBytes(t *testing.T) {
|
||||
enc, _ := NewEncoder("dump", cols("a"), nil, Options{})
|
||||
r := Row{Values: []Value{{Raw: []byte{'a', 0, '\b', 0x1a, '\r'}}}}
|
||||
got, _ := enc.Encode(nil, r)
|
||||
want := "a\\0\\b\\Z\\r\n"
|
||||
if string(got) != want {
|
||||
t.Fatalf("got %q want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVEncoding(t *testing.T) {
|
||||
enc, err := NewEncoder("csv", cols("a", "b", "c", "d"), nil, Options{CSVNull: `\N`})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h := string(enc.Header()); h != "a,b,c,d\r\n" {
|
||||
t.Fatalf("header %q", h)
|
||||
}
|
||||
r := vals(`a,"b"`, "", nil, "plain")
|
||||
got, _ := enc.Encode(nil, r)
|
||||
want := "\"a,\"\"b\"\"\",,\\N,plain\r\n"
|
||||
if string(got) != want {
|
||||
t.Fatalf("got %q want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVNullToken(t *testing.T) {
|
||||
enc, _ := NewEncoder("csv", cols("a"), nil, Options{CSVNull: "NULL"})
|
||||
got, _ := enc.Encode(nil, vals(nil))
|
||||
if string(got) != "NULL\r\n" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONLTypes(t *testing.T) {
|
||||
columns := []Column{
|
||||
{Name: "id", Type: "bigint"},
|
||||
{Name: "price", Type: "decimal"},
|
||||
{Name: "created", Type: "datetime"},
|
||||
{Name: "day", Type: "date"},
|
||||
{Name: "blob", Type: "blob"},
|
||||
{Name: "note", Type: "varchar"},
|
||||
{Name: "missing", Type: "varchar"},
|
||||
}
|
||||
enc, _ := NewEncoder("jsonl", columns, nil, Options{})
|
||||
r := vals("42", "12.30", "2024-01-02 03:04:05", "2024-01-02", "hi", `line "one"`, nil)
|
||||
got, _ := enc.Encode(nil, r)
|
||||
want := `{"id":42,"price":"12.30","created":"2024-01-02T03:04:05Z","day":"2024-01-02","blob":"aGk=","note":"line \"one\"","missing":null}` + "\n"
|
||||
if string(got) != want {
|
||||
t.Fatalf("got %s\nwant %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONLColumnProjection(t *testing.T) {
|
||||
columns := []Column{{Name: "a", Type: "int"}, {Name: "b", Type: "int"}, {Name: "c", Type: "int"}}
|
||||
enc, _ := NewEncoder("jsonl", columns, []int{2, 0}, Options{})
|
||||
got, _ := enc.Encode(nil, vals("1", "2", "3"))
|
||||
if strings.TrimSpace(string(got)) != `{"c":3,"a":1}` {
|
||||
t.Fatalf("got %s", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package row
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"time"
|
||||
)
|
||||
|
||||
// jsonlEncoder emits one JSON object per line. Types are mapped deliberately
|
||||
// so downstream tools see useful values: decimals become strings to preserve
|
||||
// precision, binary columns become base64, date/time columns become strings
|
||||
// and NULL becomes JSON null.
|
||||
type jsonlEncoder struct {
|
||||
cols []Column
|
||||
idx []int
|
||||
}
|
||||
|
||||
func (e *jsonlEncoder) Header() []byte { return nil }
|
||||
|
||||
func (e *jsonlEncoder) Encode(dst []byte, r Row) ([]byte, error) {
|
||||
dst = append(dst, '{')
|
||||
for n, i := range e.idx {
|
||||
if n > 0 {
|
||||
dst = append(dst, ',')
|
||||
}
|
||||
dst = appendJSONString(dst, []byte(e.cols[i].Name))
|
||||
dst = append(dst, ':')
|
||||
dst = appendJSONValue(dst, e.cols[i], r.Values[i])
|
||||
}
|
||||
return append(dst, '}', '\n'), nil
|
||||
}
|
||||
|
||||
func appendJSONValue(dst []byte, col Column, v Value) []byte {
|
||||
if v.IsNull() {
|
||||
return append(dst, "null"...)
|
||||
}
|
||||
switch {
|
||||
case IsInteger(col.Type), IsFloat(col.Type):
|
||||
if looksNumeric(v.Raw) {
|
||||
return append(dst, v.Raw...)
|
||||
}
|
||||
return appendJSONString(dst, v.Raw)
|
||||
case IsDecimal(col.Type):
|
||||
return appendJSONString(dst, v.Raw)
|
||||
case IsBinary(col.Type):
|
||||
dst = append(dst, '"')
|
||||
dst = append(dst, base64.StdEncoding.EncodeToString(v.Raw)...)
|
||||
return append(dst, '"')
|
||||
case IsDate(col.Type):
|
||||
return appendJSONString(dst, v.Raw)
|
||||
case IsDateTime(col.Type):
|
||||
if s, ok := formatRFC3339(v.Raw); ok {
|
||||
return appendJSONString(dst, []byte(s))
|
||||
}
|
||||
return appendJSONString(dst, v.Raw)
|
||||
default:
|
||||
return appendJSONString(dst, v.Raw)
|
||||
}
|
||||
}
|
||||
|
||||
func looksNumeric(b []byte) bool {
|
||||
if len(b) == 0 {
|
||||
return false
|
||||
}
|
||||
for i, c := range b {
|
||||
if (c >= '0' && c <= '9') || c == '-' || c == '+' || c == '.' || c == 'e' || c == 'E' {
|
||||
continue
|
||||
}
|
||||
if i == 0 && c == ' ' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var dateTimeLayouts = []string{
|
||||
"2006-01-02 15:04:05.999999999",
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02T15:04:05Z07:00",
|
||||
"2006-01-02T15:04:05.999999999Z07:00",
|
||||
}
|
||||
|
||||
func formatRFC3339(raw []byte) (string, bool) {
|
||||
s := string(raw)
|
||||
for _, layout := range dateTimeLayouts {
|
||||
t, err := time.ParseInLocation(layout, s, time.UTC)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if t.Nanosecond() != 0 {
|
||||
return t.UTC().Format(time.RFC3339Nano), true
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
const hexDigits = "0123456789abcdef"
|
||||
|
||||
func appendJSONString(dst, s []byte) []byte {
|
||||
dst = append(dst, '"')
|
||||
for _, c := range s {
|
||||
switch c {
|
||||
case '"', '\\':
|
||||
dst = append(dst, '\\', c)
|
||||
case '\n':
|
||||
dst = append(dst, '\\', 'n')
|
||||
case '\r':
|
||||
dst = append(dst, '\\', 'r')
|
||||
case '\t':
|
||||
dst = append(dst, '\\', 't')
|
||||
case '\b':
|
||||
dst = append(dst, '\\', 'b')
|
||||
case '\f':
|
||||
dst = append(dst, '\\', 'f')
|
||||
default:
|
||||
if c < 0x20 {
|
||||
dst = append(dst, '\\', 'u', '0', '0', hexDigits[c>>4], hexDigits[c&0xf])
|
||||
} else {
|
||||
dst = append(dst, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
return append(dst, '"')
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
// Package row defines scanned values, key tuples and the line encoders used
|
||||
// to serialize rows into dump, csv and jsonl shards.
|
||||
package row
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Value is one column value as returned by the MySQL driver with
|
||||
// parseTime disabled. A nil Raw means SQL NULL; an empty non-nil Raw means
|
||||
// the empty string.
|
||||
type Value struct {
|
||||
Raw []byte
|
||||
}
|
||||
|
||||
func (v Value) IsNull() bool { return v.Raw == nil }
|
||||
func (v Value) Bytes() []byte { return v.Raw }
|
||||
func (v Value) String() string { return string(v.Raw) }
|
||||
|
||||
// DriverValue returns the value as a bind parameter.
|
||||
func (v Value) DriverValue() any {
|
||||
if v.Raw == nil {
|
||||
return nil
|
||||
}
|
||||
return v.Raw
|
||||
}
|
||||
|
||||
// KeyPart is a single component of a key tuple. Values are held as text so a
|
||||
// key can be serialized into state.json and rebound to the driver without
|
||||
// carrying driver-specific types.
|
||||
type KeyPart struct {
|
||||
Null bool `json:"null,omitempty"`
|
||||
Val string `json:"val,omitempty"`
|
||||
}
|
||||
|
||||
// DriverValue converts the part into a value accepted by database/sql.
|
||||
func (p KeyPart) DriverValue() any {
|
||||
if p.Null {
|
||||
return nil
|
||||
}
|
||||
return p.Val
|
||||
}
|
||||
|
||||
// Key is a tuple of values identifying a row position in a unique index.
|
||||
type Key []KeyPart
|
||||
|
||||
func KeyOf(values ...string) Key {
|
||||
parts := make(Key, len(values))
|
||||
for i, v := range values {
|
||||
parts[i] = KeyPart{Val: v}
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
// Args returns the key as driver bind parameters.
|
||||
func (k Key) Args() []any {
|
||||
out := make([]any, len(k))
|
||||
for i, p := range k {
|
||||
out[i] = p.DriverValue()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (k Key) IsZero() bool { return len(k) == 0 }
|
||||
|
||||
func (k Key) Equal(o Key) bool {
|
||||
if len(k) != len(o) {
|
||||
return false
|
||||
}
|
||||
for i := range k {
|
||||
if k[i] != o[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (k Key) String() string {
|
||||
if len(k) == 0 {
|
||||
return "()"
|
||||
}
|
||||
parts := make([]string, len(k))
|
||||
for i, p := range k {
|
||||
if p.Null {
|
||||
parts[i] = "NULL"
|
||||
} else {
|
||||
parts[i] = p.Val
|
||||
}
|
||||
}
|
||||
return "(" + strings.Join(parts, ",") + ")"
|
||||
}
|
||||
|
||||
// FirstLast returns the two parts of a composite (or single) key.
|
||||
func (k Key) FirstLast() (KeyPart, KeyPart) {
|
||||
if len(k) == 0 {
|
||||
return KeyPart{Null: true}, KeyPart{Null: true}
|
||||
}
|
||||
return k[0], k[len(k)-1]
|
||||
}
|
||||
|
||||
// Row holds a scanned row in select-list order.
|
||||
type Row struct {
|
||||
Values []Value
|
||||
}
|
||||
|
||||
// Chunk is a batch of rows produced by one scan step. Keys is parallel to
|
||||
// Rows and holds the unique-index key of each row; Buckets is parallel to
|
||||
// Rows and is populated only in date sharding mode.
|
||||
type Chunk struct {
|
||||
Rows []Row
|
||||
Keys []Key
|
||||
Buckets []string
|
||||
FirstKey Key
|
||||
LastKey Key
|
||||
}
|
||||
|
||||
func (c *Chunk) Len() int { return len(c.Rows) }
|
||||
|
||||
// Column describes one exported column for encoding purposes.
|
||||
type Column struct {
|
||||
Name string
|
||||
Type string // lowercase information_schema DATA_TYPE
|
||||
}
|
||||
|
||||
func isOneOf(t string, set ...string) bool {
|
||||
for _, s := range set {
|
||||
if t == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func IsInteger(t string) bool {
|
||||
return isOneOf(t, "tinyint", "smallint", "mediumint", "int", "integer", "bigint", "year")
|
||||
}
|
||||
|
||||
func IsFloat(t string) bool {
|
||||
return isOneOf(t, "float", "double", "real")
|
||||
}
|
||||
|
||||
func IsDecimal(t string) bool { return isOneOf(t, "decimal", "numeric") }
|
||||
|
||||
func IsBinary(t string) bool {
|
||||
return isOneOf(t, "binary", "varbinary", "tinyblob", "blob", "mediumblob", "longblob", "bit", "geometry", "geometrycollection", "point", "linestring", "polygon", "multipoint", "multilinestring", "multipolygon")
|
||||
}
|
||||
|
||||
func IsDate(t string) bool { return t == "date" }
|
||||
|
||||
func IsDateTime(t string) bool { return isOneOf(t, "datetime", "timestamp") }
|
||||
|
||||
func IsTimeType(t string) bool { return isOneOf(t, "date", "datetime", "timestamp") }
|
||||
@@ -0,0 +1,148 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"archer/internal/row"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
// DateAScanner uses an index that starts with (shard_column, key...) and
|
||||
// performs keyset pagination over it. NULL rows are scanned in a separate
|
||||
// first phase because row-constructor comparisons do not advance past NULL.
|
||||
type DateAScanner struct {
|
||||
o *Options
|
||||
phase string // null | value
|
||||
cursor row.Key
|
||||
done bool
|
||||
}
|
||||
|
||||
func NewDateA(o Options) *DateAScanner {
|
||||
s := &DateAScanner{o: &o, phase: "null"}
|
||||
if o.Resume != nil {
|
||||
switch o.Resume.Phase {
|
||||
case "null":
|
||||
s.phase = "null"
|
||||
s.cursor = o.Resume.Key
|
||||
case "value":
|
||||
s.phase = "value"
|
||||
s.cursor = o.Resume.Key
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *DateAScanner) Next(ctx context.Context) (*row.Chunk, error) {
|
||||
for {
|
||||
if s.done {
|
||||
return nil, io.EOF
|
||||
}
|
||||
var (
|
||||
c *row.Chunk
|
||||
exhausted bool
|
||||
err error
|
||||
)
|
||||
if s.phase == "null" {
|
||||
c, exhausted, err = s.queryNull(ctx)
|
||||
} else {
|
||||
c, exhausted, err = s.queryValue(ctx)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.Len() == 0 {
|
||||
if s.phase == "null" {
|
||||
s.phase = "value"
|
||||
s.cursor = nil
|
||||
continue
|
||||
}
|
||||
s.done = true
|
||||
return nil, io.EOF
|
||||
}
|
||||
if s.phase == "null" {
|
||||
if exhausted {
|
||||
s.phase = "value"
|
||||
s.cursor = nil
|
||||
} else {
|
||||
s.cursor = c.LastKey
|
||||
}
|
||||
} else {
|
||||
s.cursor = s.o.compositeCursor(c.Rows[len(c.Rows)-1])
|
||||
c.LastKey = s.cursor
|
||||
if exhausted {
|
||||
s.done = true
|
||||
}
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DateAScanner) queryNull(ctx context.Context) (*row.Chunk, bool, error) {
|
||||
var b strings.Builder
|
||||
b.WriteString(s.o.selectSQL())
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.Ident(s.o.Shard.Column))
|
||||
b.WriteString(" IS NULL")
|
||||
var args []any
|
||||
if len(s.cursor) > 0 {
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.GreaterThan(s.o.KeyCols))
|
||||
args = append(args, s.cursor.Args()...)
|
||||
}
|
||||
b.WriteString(" ORDER BY ")
|
||||
b.WriteString(sqlutil.Columns(s.o.KeyCols))
|
||||
b.WriteString(" LIMIT ?")
|
||||
args = append(args, s.o.ChunkSize)
|
||||
return s.run(ctx, b.String(), args)
|
||||
}
|
||||
|
||||
func (s *DateAScanner) queryValue(ctx context.Context) (*row.Chunk, bool, error) {
|
||||
var b strings.Builder
|
||||
b.WriteString(s.o.selectSQL())
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.Ident(s.o.Shard.Column))
|
||||
b.WriteString(" IS NOT NULL")
|
||||
cols := append([]string{s.o.Shard.Column}, s.o.KeyCols...)
|
||||
var args []any
|
||||
if len(s.cursor) > 0 {
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.GreaterThan(cols))
|
||||
args = append(args, s.cursor.Args()...)
|
||||
}
|
||||
b.WriteString(" ORDER BY ")
|
||||
b.WriteString(sqlutil.Columns(cols))
|
||||
b.WriteString(" LIMIT ?")
|
||||
args = append(args, s.o.ChunkSize)
|
||||
c, exhausted, err := s.run(ctx, b.String(), args)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if c.Len() > 0 {
|
||||
c.LastKey = s.o.compositeCursor(c.Rows[len(c.Rows)-1])
|
||||
}
|
||||
return c, exhausted, nil
|
||||
}
|
||||
|
||||
func (s *DateAScanner) run(ctx context.Context, q string, args []any) (*row.Chunk, bool, error) {
|
||||
rows, err := s.o.DB.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
c, exhausted, err := s.o.readChunk(rows, s.o.ChunkSize)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if err := s.o.fillBuckets(c); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return c, exhausted, nil
|
||||
}
|
||||
|
||||
func (s *DateAScanner) ResumeState() CursorState {
|
||||
return CursorState{Phase: s.phase, Key: s.cursor}
|
||||
}
|
||||
|
||||
func (s *DateAScanner) Close() error { return nil }
|
||||
@@ -0,0 +1,350 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archer/internal/row"
|
||||
"archer/internal/schema"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
const zeroDay = "0000-00-00"
|
||||
|
||||
// DateBScanner enumerates days one at a time and streams each day's rows over
|
||||
// a dedicated connection. It is used when only a single-column index on the
|
||||
// shard column exists.
|
||||
type DateBScanner struct {
|
||||
o *Options
|
||||
conn *sql.Conn
|
||||
|
||||
phase string // null | day
|
||||
rows *sql.Rows
|
||||
cursor row.Key
|
||||
done bool
|
||||
|
||||
daysInited bool
|
||||
hasDays bool
|
||||
zeroPending bool
|
||||
zeroDone bool
|
||||
nextDay time.Time
|
||||
maxDay time.Time
|
||||
curDay string
|
||||
nextDayStr string
|
||||
|
||||
resumeDay string
|
||||
resumeKey row.Key
|
||||
resumeUse bool
|
||||
}
|
||||
|
||||
func NewDateB(ctx context.Context, o Options) (*DateBScanner, error) {
|
||||
conn, err := o.DB.Conn(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("acquire streaming connection: %w", err)
|
||||
}
|
||||
s := &DateBScanner{o: &o, conn: conn, phase: "null"}
|
||||
if o.Resume != nil {
|
||||
switch o.Resume.Phase {
|
||||
case "null":
|
||||
s.phase = "null"
|
||||
s.cursor = o.Resume.Key
|
||||
case "day":
|
||||
s.phase = "day"
|
||||
s.resumeDay = o.Resume.DayStart
|
||||
s.resumeKey = o.Resume.Key
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *DateBScanner) Next(ctx context.Context) (*row.Chunk, error) {
|
||||
for {
|
||||
if s.done {
|
||||
return nil, io.EOF
|
||||
}
|
||||
if s.rows == nil {
|
||||
if err := s.openNext(ctx); err != nil {
|
||||
if err == io.EOF {
|
||||
s.done = true
|
||||
return nil, io.EOF
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
c, exhausted, err := s.o.readChunk(s.rows, s.o.ChunkSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.o.fillBuckets(c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.Len() > 0 {
|
||||
if s.phase == "day" {
|
||||
s.cursor = s.o.compositeCursor(c.Rows[len(c.Rows)-1])
|
||||
} else {
|
||||
s.cursor = c.Keys[len(c.Keys)-1]
|
||||
}
|
||||
c.LastKey = s.cursor
|
||||
}
|
||||
if exhausted {
|
||||
if err := s.rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.rows = nil
|
||||
if s.phase == "null" {
|
||||
s.phase = "day"
|
||||
s.cursor = nil
|
||||
}
|
||||
}
|
||||
if c.Len() == 0 {
|
||||
continue
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DateBScanner) openNext(ctx context.Context) error {
|
||||
if s.phase == "null" {
|
||||
return s.openNull(ctx)
|
||||
}
|
||||
if !s.daysInited {
|
||||
if err := s.initDays(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if s.zeroPending && !s.zeroDone {
|
||||
s.zeroPending = false
|
||||
s.zeroDone = true
|
||||
s.curDay = zeroDay
|
||||
s.nextDayStr = zeroDay
|
||||
return s.openZero(ctx)
|
||||
}
|
||||
if !s.advanceDay() {
|
||||
return io.EOF
|
||||
}
|
||||
return s.openDay(ctx)
|
||||
}
|
||||
|
||||
func (s *DateBScanner) openNull(ctx context.Context) error {
|
||||
var b strings.Builder
|
||||
b.WriteString(s.o.selectSQL())
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.Ident(s.o.Shard.Column))
|
||||
b.WriteString(" IS NULL")
|
||||
var args []any
|
||||
if len(s.cursor) > 0 {
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.GreaterThan(s.o.KeyCols))
|
||||
args = append(args, s.cursor.Args()...)
|
||||
}
|
||||
b.WriteString(" ORDER BY ")
|
||||
b.WriteString(sqlutil.Columns(s.o.KeyCols))
|
||||
return s.query(ctx, b.String(), args)
|
||||
}
|
||||
|
||||
func (s *DateBScanner) openZero(ctx context.Context) error {
|
||||
var b strings.Builder
|
||||
b.WriteString(s.o.selectSQL())
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.Ident(s.o.Shard.Column))
|
||||
b.WriteString(" < ?")
|
||||
upper := "0001-01-01"
|
||||
if s.o.Shard.Kind == schema.ShardDateTime {
|
||||
upper = "0001-01-01 00:00:00"
|
||||
}
|
||||
args := []any{upper}
|
||||
cols := append([]string{s.o.Shard.Column}, s.o.KeyCols...)
|
||||
if len(s.cursor) > 0 {
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.GreaterThan(cols))
|
||||
args = append(args, s.cursor.Args()...)
|
||||
}
|
||||
b.WriteString(" ORDER BY ")
|
||||
b.WriteString(sqlutil.Columns(cols))
|
||||
s.nextDayStr = zeroDay
|
||||
return s.query(ctx, b.String(), args)
|
||||
}
|
||||
|
||||
func (s *DateBScanner) openDay(ctx context.Context) error {
|
||||
var b strings.Builder
|
||||
b.WriteString(s.o.selectSQL())
|
||||
b.WriteString(" AND ")
|
||||
col := sqlutil.Ident(s.o.Shard.Column)
|
||||
b.WriteString(col)
|
||||
b.WriteString(" >= ? AND ")
|
||||
b.WriteString(col)
|
||||
b.WriteString(" < ?")
|
||||
|
||||
var lo, hi any
|
||||
switch s.o.Shard.Kind {
|
||||
case schema.ShardDate:
|
||||
lo = s.curDay
|
||||
hi = s.nextDayStr
|
||||
case schema.ShardDateTime:
|
||||
lo = s.curDay + " 00:00:00"
|
||||
hi = s.nextDayStr + " 00:00:00"
|
||||
case schema.ShardEpoch:
|
||||
lo = schema.TimeToEpoch(s.dayTime(s.curDay), s.o.Shard.Unit)
|
||||
hi = schema.TimeToEpoch(s.dayTime(s.curDay).AddDate(0, 0, 1), s.o.Shard.Unit)
|
||||
}
|
||||
args := []any{lo, hi}
|
||||
cols := append([]string{s.o.Shard.Column}, s.o.KeyCols...)
|
||||
if len(s.cursor) > 0 {
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.GreaterThan(cols))
|
||||
args = append(args, s.cursor.Args()...)
|
||||
}
|
||||
b.WriteString(" ORDER BY ")
|
||||
b.WriteString(sqlutil.Columns(cols))
|
||||
return s.query(ctx, b.String(), args)
|
||||
}
|
||||
|
||||
func (s *DateBScanner) query(ctx context.Context, q string, args []any) error {
|
||||
rows, err := s.conn.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.rows = rows
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DateBScanner) dayTime(day string) time.Time {
|
||||
t, _ := time.ParseInLocation("2006-01-02", day, s.o.ShardLoc)
|
||||
return t
|
||||
}
|
||||
|
||||
func (s *DateBScanner) initDays(ctx context.Context) error {
|
||||
s.daysInited = true
|
||||
q := fmt.Sprintf("SELECT MIN(%s), MAX(%s) FROM %s WHERE (%s) AND %s IS NOT NULL",
|
||||
sqlutil.Ident(s.o.Shard.Column), sqlutil.Ident(s.o.Shard.Column),
|
||||
s.o.qualified(), s.o.Where, sqlutil.Ident(s.o.Shard.Column))
|
||||
var minRaw, maxRaw sql.NullString
|
||||
if err := s.conn.QueryRowContext(ctx, q).Scan(&minRaw, &maxRaw); err != nil {
|
||||
return fmt.Errorf("read shard column range: %w", err)
|
||||
}
|
||||
if !minRaw.Valid || !maxRaw.Valid {
|
||||
return nil
|
||||
}
|
||||
switch s.o.Shard.Kind {
|
||||
case schema.ShardDate, schema.ShardDateTime:
|
||||
minDay := s.first10(minRaw.String)
|
||||
maxDay := s.first10(maxRaw.String)
|
||||
zeroExists := strings.HasPrefix(minDay, "0000-")
|
||||
if zeroExists {
|
||||
minDay = "0001-01-01"
|
||||
}
|
||||
s.zeroPending = s.wantZero(zeroExists)
|
||||
if strings.HasPrefix(maxDay, "0000-") {
|
||||
return nil
|
||||
}
|
||||
minT, err := time.ParseInLocation("2006-01-02", minDay, s.o.ShardLoc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse shard range minimum %q: %w", minRaw.String, err)
|
||||
}
|
||||
maxT, err := time.ParseInLocation("2006-01-02", maxDay, s.o.ShardLoc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse shard range maximum %q: %w", maxRaw.String, err)
|
||||
}
|
||||
s.minMax(minT, maxT)
|
||||
case schema.ShardEpoch:
|
||||
minV, err := strconv.ParseInt(strings.TrimSpace(minRaw.String), 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse shard range minimum: %w", err)
|
||||
}
|
||||
maxV, err := strconv.ParseInt(strings.TrimSpace(maxRaw.String), 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse shard range maximum: %w", err)
|
||||
}
|
||||
minT := schema.EpochTime(minV, s.o.Shard.Unit, s.o.ShardLoc)
|
||||
maxT := schema.EpochTime(maxV, s.o.Shard.Unit, s.o.ShardLoc)
|
||||
s.minMax(dayStart(minT, s.o.ShardLoc), dayStart(maxT, s.o.ShardLoc))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DateBScanner) wantZero(zeroExists bool) bool {
|
||||
if s.resumeDay == "" {
|
||||
return zeroExists
|
||||
}
|
||||
return s.resumeDay == zeroDay
|
||||
}
|
||||
|
||||
func (s *DateBScanner) minMax(minT, maxT time.Time) {
|
||||
s.hasDays = true
|
||||
s.nextDay = minT
|
||||
s.maxDay = maxT
|
||||
if s.resumeDay != "" && s.resumeDay != zeroDay {
|
||||
if d, err := time.ParseInLocation("2006-01-02", s.resumeDay, s.o.ShardLoc); err == nil && d.After(minT) {
|
||||
s.nextDay = d
|
||||
}
|
||||
}
|
||||
if s.resumeDay == zeroDay {
|
||||
s.nextDay = minT
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DateBScanner) advanceDay() bool {
|
||||
if !s.hasDays {
|
||||
return false
|
||||
}
|
||||
for !s.nextDay.After(s.maxDay) {
|
||||
day := s.nextDay.Format("2006-01-02")
|
||||
if s.resumeDay != "" && s.resumeDay != zeroDay && day < s.resumeDay && !s.resumeUse {
|
||||
s.nextDay = s.nextDay.AddDate(0, 0, 1)
|
||||
continue
|
||||
}
|
||||
s.curDay = day
|
||||
s.nextDayStr = s.nextDay.AddDate(0, 0, 1).Format("2006-01-02")
|
||||
s.nextDay = s.nextDay.AddDate(0, 0, 1)
|
||||
if s.resumeDay != "" && day == s.resumeDay && !s.resumeUse {
|
||||
s.cursor = s.resumeKey
|
||||
s.resumeUse = true
|
||||
} else {
|
||||
s.cursor = nil
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *DateBScanner) first10(v string) string {
|
||||
if len(v) < 10 {
|
||||
return v
|
||||
}
|
||||
return v[:10]
|
||||
}
|
||||
|
||||
func dayStart(t time.Time, loc *time.Location) time.Time {
|
||||
return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, loc)
|
||||
}
|
||||
|
||||
func (s *DateBScanner) ResumeState() CursorState {
|
||||
phase := s.phase
|
||||
day := ""
|
||||
if phase == "day" {
|
||||
day = s.curDay
|
||||
}
|
||||
return CursorState{Phase: phase, Key: s.cursor, DayStart: day}
|
||||
}
|
||||
|
||||
func (s *DateBScanner) Close() error {
|
||||
var firstErr error
|
||||
if s.rows != nil {
|
||||
if err := s.rows.Close(); err != nil {
|
||||
firstErr = err
|
||||
}
|
||||
s.rows = nil
|
||||
}
|
||||
if s.conn != nil {
|
||||
if err := s.conn.Close(); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
s.conn = nil
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"archer/internal/row"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
// PKScanner walks the table by its chosen unique key using
|
||||
// key > :last ORDER BY key LIMIT :chunk_size.
|
||||
type PKScanner struct {
|
||||
o *Options
|
||||
cursor row.Key
|
||||
done bool
|
||||
}
|
||||
|
||||
func NewPK(o Options) *PKScanner {
|
||||
s := &PKScanner{o: &o}
|
||||
if o.Resume != nil && (o.Resume.Phase == "" || o.Resume.Phase == "pk") {
|
||||
s.cursor = o.Resume.Key
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *PKScanner) Next(ctx context.Context) (*row.Chunk, error) {
|
||||
if s.done {
|
||||
return nil, io.EOF
|
||||
}
|
||||
q, args := s.query()
|
||||
rows, err := s.o.DB.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
chunk, exhausted, err := s.o.readChunk(rows, s.o.ChunkSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(chunk.Rows) == 0 {
|
||||
s.done = true
|
||||
return nil, io.EOF
|
||||
}
|
||||
if exhausted {
|
||||
s.done = true
|
||||
}
|
||||
s.cursor = chunk.LastKey
|
||||
return chunk, nil
|
||||
}
|
||||
|
||||
func (s *PKScanner) query() (string, []any) {
|
||||
var b strings.Builder
|
||||
b.WriteString(s.o.selectSQL())
|
||||
var args []any
|
||||
if len(s.cursor) > 0 {
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.GreaterThan(s.o.KeyCols))
|
||||
args = append(args, s.cursor.Args()...)
|
||||
}
|
||||
b.WriteString(" ORDER BY ")
|
||||
b.WriteString(sqlutil.Columns(s.o.KeyCols))
|
||||
b.WriteString(" LIMIT ?")
|
||||
args = append(args, s.o.ChunkSize)
|
||||
return b.String(), args
|
||||
}
|
||||
|
||||
func (s *PKScanner) Close() error { return nil }
|
||||
|
||||
func (s *PKScanner) ResumeState() CursorState {
|
||||
return CursorState{Phase: "pk", Key: s.cursor}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
// Package scan implements keyset (seek) chunked scanning of a source table.
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"archer/internal/row"
|
||||
"archer/internal/schema"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
// CursorState is the resumable position of a scanner.
|
||||
type CursorState struct {
|
||||
Phase string
|
||||
Key row.Key
|
||||
DayStart string
|
||||
}
|
||||
|
||||
// Scanner produces chunks in scan order.
|
||||
type Scanner interface {
|
||||
Next(ctx context.Context) (*row.Chunk, error) // io.EOF when exhausted
|
||||
ResumeState() CursorState
|
||||
Close() error
|
||||
}
|
||||
|
||||
// Resume is the persisted scan position.
|
||||
type Resume struct {
|
||||
Phase string // pk|null|value|day
|
||||
Key row.Key
|
||||
DayStart string
|
||||
}
|
||||
|
||||
// Options configures a Scanner.
|
||||
type Options struct {
|
||||
DB *sql.DB
|
||||
DBName string
|
||||
Table string
|
||||
Where string
|
||||
SelectCols []string
|
||||
KeyCols []string
|
||||
KeyIdx []int
|
||||
ChunkSize int
|
||||
|
||||
Shard *schema.ShardSpec // nil outside date mode
|
||||
ShardIdx int // -1 outside date mode
|
||||
ShardLoc *time.Location
|
||||
ZeroDate string
|
||||
Resume *Resume
|
||||
}
|
||||
|
||||
func (o *Options) qualified() string { return sqlutil.Qualified(o.DBName, o.Table) }
|
||||
|
||||
func (o *Options) selectSQL() string {
|
||||
return "SELECT " + sqlutil.Columns(o.SelectCols) +
|
||||
" FROM " + o.qualified() +
|
||||
" WHERE (" + o.Where + ")"
|
||||
}
|
||||
|
||||
func (o *Options) keyOf(r row.Row) row.Key {
|
||||
k := make(row.Key, len(o.KeyIdx))
|
||||
for i, idx := range o.KeyIdx {
|
||||
v := r.Values[idx]
|
||||
if v.IsNull() {
|
||||
k[i] = row.KeyPart{Null: true}
|
||||
} else {
|
||||
k[i] = row.KeyPart{Val: string(v.Raw)}
|
||||
}
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
func (o *Options) compositeCursor(r row.Row) row.Key {
|
||||
k := o.keyOf(r)
|
||||
out := make(row.Key, 0, 1+len(k))
|
||||
v := r.Values[o.ShardIdx]
|
||||
if v.IsNull() {
|
||||
out = append(out, row.KeyPart{Null: true})
|
||||
} else {
|
||||
out = append(out, row.KeyPart{Val: string(v.Raw)})
|
||||
}
|
||||
return append(out, k...)
|
||||
}
|
||||
|
||||
func (o *Options) bucket(r row.Row) (string, error) {
|
||||
return schema.BucketOf(r.Values[o.ShardIdx], o.Shard, o.ShardLoc, o.ZeroDate, o.keyOf(r))
|
||||
}
|
||||
|
||||
// readChunk consumes up to limit rows from an open result set.
|
||||
func (o *Options) readChunk(rows *sql.Rows, limit int) (*row.Chunk, bool, error) {
|
||||
chunk := &row.Chunk{}
|
||||
raw := make([]sql.RawBytes, len(o.SelectCols))
|
||||
ptrs := make([]any, len(raw))
|
||||
for i := range raw {
|
||||
ptrs[i] = &raw[i]
|
||||
}
|
||||
exhausted := false
|
||||
for len(chunk.Rows) < limit {
|
||||
if !rows.Next() {
|
||||
exhausted = true
|
||||
break
|
||||
}
|
||||
if err := rows.Scan(ptrs...); err != nil {
|
||||
return nil, false, fmt.Errorf("scan row: %w", err)
|
||||
}
|
||||
vals := make([]row.Value, len(raw))
|
||||
for i, b := range raw {
|
||||
if b == nil {
|
||||
continue
|
||||
}
|
||||
c := make([]byte, len(b))
|
||||
copy(c, b)
|
||||
vals[i] = row.Value{Raw: c}
|
||||
}
|
||||
chunk.Rows = append(chunk.Rows, row.Row{Values: vals})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, false, fmt.Errorf("read rows: %w", err)
|
||||
}
|
||||
for i := range chunk.Rows {
|
||||
chunk.Keys = append(chunk.Keys, o.keyOf(chunk.Rows[i]))
|
||||
}
|
||||
if len(chunk.Rows) > 0 {
|
||||
chunk.FirstKey = chunk.Keys[0]
|
||||
chunk.LastKey = chunk.Keys[len(chunk.Keys)-1]
|
||||
}
|
||||
return chunk, exhausted, nil
|
||||
}
|
||||
|
||||
func (o *Options) fillBuckets(c *row.Chunk) error {
|
||||
c.Buckets = make([]string, len(c.Rows))
|
||||
for i := range c.Rows {
|
||||
b, err := o.bucket(c.Rows[i])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.Buckets[i] = b
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
package schema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archer/internal/row"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
// ShardKind is how a shard column's date is derived from its raw value.
|
||||
type ShardKind int
|
||||
|
||||
const (
|
||||
ShardNone ShardKind = iota
|
||||
ShardDate
|
||||
ShardDateTime
|
||||
ShardEpoch
|
||||
)
|
||||
|
||||
// ShardSpec is the resolved shard column and how to interpret it.
|
||||
type ShardSpec struct {
|
||||
Column string
|
||||
Kind ShardKind
|
||||
Unit int64 // epoch multiplier to nanoseconds: 1e9, 1e6, 1e3
|
||||
Type string
|
||||
}
|
||||
|
||||
// ResolveShardColumn validates --shard-column against the table and the unit
|
||||
// flag. It returns a descriptive error listing candidates when the column is
|
||||
// missing or the type is unsupported.
|
||||
func ResolveShardColumn(ctx context.Context, db *sql.DB, t *Table, column, unitFlag, where string, loc *time.Location) (*ShardSpec, error) {
|
||||
if column == "" {
|
||||
return nil, fmt.Errorf("--shard-column is required for --shard-by date\ncandidate columns:\n%s", describeTimeColumns(t))
|
||||
}
|
||||
c, ok := t.Column(column)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("shard column %q does not exist; candidate columns:\n%s", column, describeTimeColumns(t))
|
||||
}
|
||||
switch c.DataType {
|
||||
case "date":
|
||||
return &ShardSpec{Column: c.Name, Kind: ShardDate, Unit: 1, Type: c.DataType}, nil
|
||||
case "datetime", "timestamp":
|
||||
return &ShardSpec{Column: c.Name, Kind: ShardDateTime, Unit: 1, Type: c.DataType}, nil
|
||||
}
|
||||
if row.IsInteger(c.DataType) {
|
||||
unit, err := ParseEpochUnit(unitFlag)
|
||||
if err != nil {
|
||||
minV, maxV, ok, serr := SampleEpochRange(ctx, db, t, c.Name, where)
|
||||
if serr != nil {
|
||||
return nil, fmt.Errorf("%w (also failed to sample %s: %v)", err, c.Name, serr)
|
||||
}
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w; column %s has no non-NULL values to sample", err, c.Name)
|
||||
}
|
||||
return nil, fmt.Errorf("%w\nsampled %s range: %d .. %d\n seconds: %s .. %s\n milliseconds: %s .. %s\n microseconds: %s .. %s",
|
||||
err, c.Name, minV, maxV,
|
||||
epochString(minV, 1, loc), epochString(maxV, 1, loc),
|
||||
epochString(minV, 1_000, loc), epochString(maxV, 1_000, loc),
|
||||
epochString(minV, 1_000_000, loc), epochString(maxV, 1_000_000, loc))
|
||||
}
|
||||
return &ShardSpec{Column: c.Name, Kind: ShardEpoch, Unit: unit, Type: c.DataType}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("column %q has unsupported type %s; v1 supports DATE, DATETIME, TIMESTAMP and integer epoch columns with --shard-column-type\ncandidate columns:\n%s",
|
||||
c.Name, c.DataType, describeTimeColumns(t))
|
||||
}
|
||||
|
||||
func describeTimeColumns(t *Table) string {
|
||||
var b strings.Builder
|
||||
found := false
|
||||
for _, c := range t.TimeColumns() {
|
||||
found = true
|
||||
fmt.Fprintf(&b, " - %s %s%s\n", c.Name, c.ColumnType, nullableSuffix(c))
|
||||
}
|
||||
for _, c := range t.Columns {
|
||||
if row.IsInteger(c.DataType) {
|
||||
found = true
|
||||
fmt.Fprintf(&b, " - %s %s (integer; requires --shard-column-type)%s\n", c.Name, c.ColumnType, nullableSuffix(c))
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return " (none)"
|
||||
}
|
||||
return strings.TrimRight(b.String(), "\n")
|
||||
}
|
||||
|
||||
func nullableSuffix(c Column) string {
|
||||
if c.Nullable {
|
||||
return " NULL"
|
||||
}
|
||||
return " NOT NULL"
|
||||
}
|
||||
|
||||
// ParseEpochUnit maps --shard-column-type to raw units per second: 1 for
|
||||
// seconds, 1000 for milliseconds, 1000000 for microseconds.
|
||||
func ParseEpochUnit(flag string) (int64, error) {
|
||||
switch strings.ToLower(strings.TrimSpace(flag)) {
|
||||
case "epoch-seconds", "seconds", "second", "s", "1":
|
||||
return 1, nil
|
||||
case "epoch-milliseconds", "epoch-millis", "milliseconds", "millisecond", "ms", "1000":
|
||||
return 1_000, nil
|
||||
case "epoch-microseconds", "epoch-micros", "microseconds", "microsecond", "us", "1000000":
|
||||
return 1_000_000, nil
|
||||
case "":
|
||||
return 0, fmt.Errorf("integer shard column requires --shard-column-type (epoch-seconds, epoch-milliseconds or epoch-microseconds)")
|
||||
default:
|
||||
return 0, fmt.Errorf("unsupported --shard-column-type %q; use epoch-seconds, epoch-milliseconds or epoch-microseconds", flag)
|
||||
}
|
||||
}
|
||||
|
||||
// SampleEpochRange returns MIN/MAX of an integer column under the base where.
|
||||
func SampleEpochRange(ctx context.Context, db *sql.DB, t *Table, col, where string) (int64, int64, bool, error) {
|
||||
q := fmt.Sprintf("SELECT MIN(%s), MAX(%s) FROM %s", sqlutil.Ident(col), sqlutil.Ident(col), sqlutil.Qualified(t.DB, t.Name))
|
||||
if strings.TrimSpace(where) != "" {
|
||||
q += " WHERE (" + where + ")"
|
||||
}
|
||||
var minRaw, maxRaw sql.NullString
|
||||
if err := db.QueryRowContext(ctx, q).Scan(&minRaw, &maxRaw); err != nil {
|
||||
return 0, 0, false, err
|
||||
}
|
||||
if !minRaw.Valid || !maxRaw.Valid {
|
||||
return 0, 0, false, nil
|
||||
}
|
||||
minV, err := strconv.ParseInt(strings.TrimSpace(minRaw.String), 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, false, fmt.Errorf("parse min %q: %w", minRaw.String, err)
|
||||
}
|
||||
maxV, err := strconv.ParseInt(strings.TrimSpace(maxRaw.String), 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, false, fmt.Errorf("parse max %q: %w", maxRaw.String, err)
|
||||
}
|
||||
return minV, maxV, true, nil
|
||||
}
|
||||
|
||||
func epochString(v, unit int64, loc *time.Location) string {
|
||||
return EpochTime(v, unit, loc).Format("2006-01-02 15:04:05")
|
||||
}
|
||||
|
||||
// EpochTime converts an epoch value with the given nanoseconds-per-unit to a
|
||||
// time in loc.
|
||||
func EpochTime(v, unit int64, loc *time.Location) time.Time {
|
||||
sec := v / unit
|
||||
frac := v % unit
|
||||
if frac < 0 {
|
||||
sec--
|
||||
frac += unit
|
||||
}
|
||||
nsec := frac * (1_000_000_000 / unit)
|
||||
return time.Unix(sec, nsec).In(loc)
|
||||
}
|
||||
|
||||
// TimeToEpoch is the inverse of EpochTime.
|
||||
func TimeToEpoch(t time.Time, unit int64) int64 {
|
||||
return t.Unix()*unit + int64(t.Nanosecond())/(1_000_000_000/unit)
|
||||
}
|
||||
|
||||
const nullBucket = "_null"
|
||||
const zeroBucket = "0000-00-00"
|
||||
|
||||
// BucketOf maps a raw shard value to its date bucket. key is used only in
|
||||
// error messages.
|
||||
func BucketOf(v row.Value, spec *ShardSpec, loc *time.Location, zeroDate string, key row.Key) (string, error) {
|
||||
if v.IsNull() {
|
||||
return nullBucket, nil
|
||||
}
|
||||
switch spec.Kind {
|
||||
case ShardEpoch:
|
||||
n, err := strconv.ParseInt(strings.TrimSpace(string(v.Raw)), 10, 64)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("row %s: cannot parse epoch value %q in column %s: %w", key, string(v.Raw), spec.Column, err)
|
||||
}
|
||||
return EpochTime(n, spec.Unit, loc).Format("2006-01-02"), nil
|
||||
case ShardDate, ShardDateTime:
|
||||
s := string(v.Raw)
|
||||
if len(s) < 10 {
|
||||
return "", fmt.Errorf("row %s: cannot parse date value %q in column %s", key, s, spec.Column)
|
||||
}
|
||||
day := s[:10]
|
||||
if strings.HasPrefix(day, "0000-") {
|
||||
if zeroDate == "bucket" {
|
||||
return zeroBucket, nil
|
||||
}
|
||||
return "", fmt.Errorf("row %s has zero date %q in column %s; pass --zero-date bucket to route it to date=0000-00-00", key, s, spec.Column)
|
||||
}
|
||||
if _, err := time.Parse("2006-01-02", day); err != nil {
|
||||
return "", fmt.Errorf("row %s: invalid date %q in column %s", key, s, spec.Column)
|
||||
}
|
||||
return day, nil
|
||||
}
|
||||
return "", fmt.Errorf("internal: shard kind not resolved")
|
||||
}
|
||||
|
||||
// BucketLess orders buckets in scan order: NULL first, then zero dates, then
|
||||
// real dates.
|
||||
func BucketLess(a, b string) bool {
|
||||
ca, ka := bucketClass(a)
|
||||
cb, kb := bucketClass(b)
|
||||
if ca != cb {
|
||||
return ca < cb
|
||||
}
|
||||
return ka < kb
|
||||
}
|
||||
|
||||
func bucketClass(b string) (int, string) {
|
||||
switch b {
|
||||
case "":
|
||||
return -1, ""
|
||||
case nullBucket:
|
||||
return 0, ""
|
||||
case zeroBucket:
|
||||
return 1, ""
|
||||
default:
|
||||
return 2, b
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package schema
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
func TestBucketOfDateTime(t *testing.T) {
|
||||
spec := &ShardSpec{Column: "created_at", Kind: ShardDateTime}
|
||||
loc := time.UTC
|
||||
cases := []struct {
|
||||
raw []byte
|
||||
want string
|
||||
}{
|
||||
{[]byte("2025-06-01 13:45:00"), "2025-06-01"},
|
||||
{[]byte("2025-12-31 00:00:00"), "2025-12-31"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, err := BucketOf(row.Value{Raw: c.raw}, spec, loc, "", row.KeyOf("1"))
|
||||
if err != nil || got != c.want {
|
||||
t.Fatalf("%s: got %q err %v want %q", c.raw, got, err, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketOfNull(t *testing.T) {
|
||||
spec := &ShardSpec{Column: "c", Kind: ShardDateTime}
|
||||
got, err := BucketOf(row.Value{}, spec, time.UTC, "", row.KeyOf("1"))
|
||||
if err != nil || got != "_null" {
|
||||
t.Fatalf("got %q err %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketOfZeroDate(t *testing.T) {
|
||||
spec := &ShardSpec{Column: "c", Kind: ShardDateTime}
|
||||
if _, err := BucketOf(row.Value{Raw: []byte("0000-00-00 00:00:00")}, spec, time.UTC, "", row.KeyOf("7")); err == nil {
|
||||
t.Fatal("expected zero-date error")
|
||||
}
|
||||
got, err := BucketOf(row.Value{Raw: []byte("0000-00-00 00:00:00")}, spec, time.UTC, "bucket", row.KeyOf("7"))
|
||||
if err != nil || got != "0000-00-00" {
|
||||
t.Fatalf("got %q err %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketOfEpoch(t *testing.T) {
|
||||
loc, err := time.LoadLocation("UTC")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
unit int64
|
||||
raw string
|
||||
want string
|
||||
}{
|
||||
{1, "1717200000", "2024-06-01"},
|
||||
{1_000, "1717200000000", "2024-06-01"},
|
||||
{1_000_000, "1717200000000000", "2024-06-01"},
|
||||
} {
|
||||
spec := &ShardSpec{Column: "ts", Kind: ShardEpoch, Unit: tc.unit}
|
||||
got, err := BucketOf(row.Value{Raw: []byte(tc.raw)}, spec, loc, "", row.KeyOf("1"))
|
||||
if err != nil || got != tc.want {
|
||||
t.Fatalf("unit %d: got %q err %v want %q", tc.unit, got, err, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketLess(t *testing.T) {
|
||||
order := []string{"_null", "0000-00-00", "2024-01-01", "2024-01-02", "2025-01-01"}
|
||||
for i := 0; i+1 < len(order); i++ {
|
||||
if !BucketLess(order[i], order[i+1]) {
|
||||
t.Fatalf("expected %q < %q", order[i], order[i+1])
|
||||
}
|
||||
if BucketLess(order[i+1], order[i]) {
|
||||
t.Fatalf("expected %q not < %q", order[i+1], order[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEpochUnit(t *testing.T) {
|
||||
cases := map[string]int64{
|
||||
"epoch-seconds": 1,
|
||||
"seconds": 1,
|
||||
"epoch-milliseconds": 1_000,
|
||||
"ms": 1_000,
|
||||
"epoch-microseconds": 1_000_000,
|
||||
"us": 1_000_000,
|
||||
}
|
||||
for in, want := range cases {
|
||||
got, err := ParseEpochUnit(in)
|
||||
if err != nil || got != want {
|
||||
t.Fatalf("%s: got %d err %v want %d", in, got, err, want)
|
||||
}
|
||||
}
|
||||
if _, err := ParseEpochUnit(""); err == nil {
|
||||
t.Fatal("empty unit should error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEpochRoundTrip(t *testing.T) {
|
||||
loc := time.UTC
|
||||
for _, unit := range []int64{1, 1_000, 1_000_000} {
|
||||
now := time.Date(2024, 6, 1, 12, 30, 15, 0, loc)
|
||||
v := TimeToEpoch(now, unit)
|
||||
back := EpochTime(v, unit, loc)
|
||||
if !back.Equal(now) {
|
||||
t.Fatalf("unit %d: got %s want %s", unit, back, now)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
// Package schema loads MySQL table metadata and answers questions about
|
||||
// columns and indexes needed to choose a scan strategy.
|
||||
package schema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
// Column is one column of the source table.
|
||||
type Column struct {
|
||||
Name string
|
||||
Ordinal int
|
||||
DataType string
|
||||
ColumnType string
|
||||
Nullable bool
|
||||
Extra string
|
||||
Key string
|
||||
}
|
||||
|
||||
func (c Column) IsAutoIncrement() bool {
|
||||
return strings.Contains(strings.ToLower(c.Extra), "auto_increment")
|
||||
}
|
||||
|
||||
// Index is one index of the source table, in index-column order.
|
||||
type Index struct {
|
||||
Name string
|
||||
Unique bool
|
||||
Columns []string
|
||||
}
|
||||
|
||||
// Table is the loaded metadata of the source table.
|
||||
type Table struct {
|
||||
DB string
|
||||
Name string
|
||||
Columns []Column
|
||||
byName map[string]Column
|
||||
Indexes []Index
|
||||
}
|
||||
|
||||
func (t *Table) Column(name string) (Column, bool) {
|
||||
c, ok := t.byName[strings.ToLower(name)]
|
||||
return c, ok
|
||||
}
|
||||
|
||||
func (t *Table) ColumnNames() []string {
|
||||
out := make([]string, len(t.Columns))
|
||||
for i, c := range t.Columns {
|
||||
out[i] = c.Name
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TimeColumns returns every DATE/DATETIME/TIMESTAMP/IPv4-ish candidate used to
|
||||
// explain to the user which shard columns are available.
|
||||
func (t *Table) TimeColumns() []Column {
|
||||
var out []Column
|
||||
for _, c := range t.Columns {
|
||||
if c.DataType == "date" || c.DataType == "datetime" || c.DataType == "timestamp" {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Load reads column and index metadata from information_schema.
|
||||
func Load(ctx context.Context, db *sql.DB, dbName, table string) (*Table, error) {
|
||||
t := &Table{DB: dbName, Name: table, byName: map[string]Column{}}
|
||||
|
||||
rows, err := db.QueryContext(ctx, `
|
||||
SELECT COLUMN_NAME, ORDINAL_POSITION, LOWER(DATA_TYPE), COLUMN_TYPE, IS_NULLABLE, EXTRA, COLUMN_KEY
|
||||
FROM information_schema.COLUMNS
|
||||
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
|
||||
ORDER BY ORDINAL_POSITION`, dbName, table)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load columns: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var c Column
|
||||
var nullable string
|
||||
if err := rows.Scan(&c.Name, &c.Ordinal, &c.DataType, &c.ColumnType, &nullable, &c.Extra, &c.Key); err != nil {
|
||||
return nil, fmt.Errorf("scan column: %w", err)
|
||||
}
|
||||
c.Nullable = strings.EqualFold(nullable, "YES")
|
||||
t.Columns = append(t.Columns, c)
|
||||
t.byName[strings.ToLower(c.Name)] = c
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("load columns: %w", err)
|
||||
}
|
||||
if len(t.Columns) == 0 {
|
||||
return nil, fmt.Errorf("table %s.%s not found or has no columns", dbName, table)
|
||||
}
|
||||
|
||||
idxRows, err := db.QueryContext(ctx, `
|
||||
SELECT INDEX_NAME, NON_UNIQUE, SEQ_IN_INDEX, COLUMN_NAME
|
||||
FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
|
||||
ORDER BY INDEX_NAME, SEQ_IN_INDEX`, dbName, table)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load indexes: %w", err)
|
||||
}
|
||||
defer idxRows.Close()
|
||||
|
||||
byIdx := map[string]*Index{}
|
||||
for idxRows.Next() {
|
||||
var name, col string
|
||||
var nonUnique, seq int
|
||||
if err := idxRows.Scan(&name, &nonUnique, &seq, &col); err != nil {
|
||||
return nil, fmt.Errorf("scan index: %w", err)
|
||||
}
|
||||
if col == "" {
|
||||
continue
|
||||
}
|
||||
ix := byIdx[name]
|
||||
if ix == nil {
|
||||
ix = &Index{Name: name, Unique: nonUnique == 0}
|
||||
byIdx[name] = ix
|
||||
t.Indexes = append(t.Indexes, *ix)
|
||||
}
|
||||
ix.Columns = append(ix.Columns, col)
|
||||
}
|
||||
if err := idxRows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("load indexes: %w", err)
|
||||
}
|
||||
// Rebuild pointers into the slice since append may have copied structs.
|
||||
for i := range t.Indexes {
|
||||
t.Indexes[i] = *byIdx[t.Indexes[i].Name]
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// KeyInfo describes the unique index chosen as scan key.
|
||||
type KeyInfo struct {
|
||||
Name string
|
||||
Columns []string
|
||||
IsPrimary bool
|
||||
}
|
||||
|
||||
func (t *Table) PrimaryKey() *Index {
|
||||
for i := range t.Indexes {
|
||||
if t.Indexes[i].Name == "PRIMARY" {
|
||||
return &t.Indexes[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ChooseKey selects the scan key: an explicitly requested index, else the
|
||||
// primary key, else the first unique index. Every key column must be NOT NULL.
|
||||
func (t *Table) ChooseKey(forced string) (KeyInfo, error) {
|
||||
var chosen *Index
|
||||
switch {
|
||||
case forced != "":
|
||||
for i := range t.Indexes {
|
||||
if strings.EqualFold(t.Indexes[i].Name, forced) {
|
||||
chosen = &t.Indexes[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if chosen == nil {
|
||||
return KeyInfo{}, fmt.Errorf("index %q does not exist on %s", forced, t.Name)
|
||||
}
|
||||
if !chosen.Unique {
|
||||
return KeyInfo{}, fmt.Errorf("index %q is not unique; a unique index is required for seek pagination", forced)
|
||||
}
|
||||
case t.PrimaryKey() != nil:
|
||||
chosen = t.PrimaryKey()
|
||||
default:
|
||||
for i := range t.Indexes {
|
||||
if t.Indexes[i].Unique {
|
||||
chosen = &t.Indexes[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if chosen == nil {
|
||||
return KeyInfo{}, fmt.Errorf("table %s has no primary key or unique index; create one or pass --index", t.Name)
|
||||
}
|
||||
}
|
||||
for _, col := range chosen.Columns {
|
||||
c, ok := t.Column(col)
|
||||
if !ok {
|
||||
return KeyInfo{}, fmt.Errorf("index %q references unknown column %q", chosen.Name, col)
|
||||
}
|
||||
if c.Nullable {
|
||||
return KeyInfo{}, fmt.Errorf("index %q column %q is nullable; seek pagination needs a NOT NULL unique key", chosen.Name, col)
|
||||
}
|
||||
}
|
||||
return KeyInfo{Name: chosen.Name, Columns: chosen.Columns, IsPrimary: chosen.Name == "PRIMARY"}, nil
|
||||
}
|
||||
|
||||
// FindIndexExact returns the index whose columns are exactly cols.
|
||||
func (t *Table) FindIndexExact(cols []string) *Index {
|
||||
for i := range t.Indexes {
|
||||
if equalFoldSlice(t.Indexes[i].Columns, cols) {
|
||||
return &t.Indexes[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FindIndexPrefix returns the first index whose leading columns are exactly
|
||||
// prefix, and its full column list.
|
||||
func (t *Table) FindIndexPrefix(prefix []string) *Index {
|
||||
for i := range t.Indexes {
|
||||
ix := &t.Indexes[i]
|
||||
if len(ix.Columns) < len(prefix) {
|
||||
continue
|
||||
}
|
||||
if equalFoldSlice(ix.Columns[:len(prefix)], prefix) {
|
||||
return ix
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HasColumnInIndex reports whether col is the first column of some index.
|
||||
func (t *Table) HasColumnPrefixIndex(col string) *Index {
|
||||
return t.FindIndexPrefix([]string{col})
|
||||
}
|
||||
|
||||
func equalFoldSlice(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if !strings.EqualFold(a[i], b[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// OutputColumns resolves the requested column list (empty means all) and
|
||||
// validates names against the table.
|
||||
func (t *Table) OutputColumns(requested []string) ([]row.Column, error) {
|
||||
if len(requested) == 0 {
|
||||
out := make([]row.Column, len(t.Columns))
|
||||
for i, c := range t.Columns {
|
||||
out[i] = row.Column{Name: c.Name, Type: c.DataType}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
out := make([]row.Column, 0, len(requested))
|
||||
seen := map[string]bool{}
|
||||
for _, name := range requested {
|
||||
c, ok := t.Column(name)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown column %q", name)
|
||||
}
|
||||
if seen[c.Name] {
|
||||
continue
|
||||
}
|
||||
seen[c.Name] = true
|
||||
out = append(out, row.Column{Name: c.Name, Type: c.DataType})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
// DefaultLevel is the compression level used when none is specified. It
|
||||
// matches the byte output of archer before --compress-level existed.
|
||||
const DefaultLevel = 3
|
||||
|
||||
// xzReadDictCap is the largest dictionary xz can produce here (level 9).
|
||||
// Readers always allow it so lowering the level with --force-resume does not
|
||||
// make previously written shards unreadable.
|
||||
const xzReadDictCap = 64 << 20
|
||||
|
||||
// Compressor wraps shard byte streams. Every shard is compressed on its own
|
||||
// so a single file can be decompressed and imported without its siblings.
|
||||
// Level is a format-neutral 1..9 scale where 9 is the strongest setting.
|
||||
type Compressor struct {
|
||||
Name string // "none", "gzip", "zstd", "xz"
|
||||
Ext string // "", ".gz", ".zst", ".xz"
|
||||
Level int
|
||||
}
|
||||
|
||||
func NewCompressor(name string, level int) (*Compressor, error) {
|
||||
if level < 1 || level > 9 {
|
||||
return nil, fmt.Errorf("compression level %d out of range 1-9", level)
|
||||
}
|
||||
switch name {
|
||||
case "", "none":
|
||||
return &Compressor{Name: "none", Level: level}, nil
|
||||
case "gzip":
|
||||
return &Compressor{Name: "gzip", Ext: ".gz", Level: level}, nil
|
||||
case "zstd":
|
||||
return &Compressor{Name: "zstd", Ext: ".zst", Level: level}, nil
|
||||
case "xz":
|
||||
return &Compressor{Name: "xz", Ext: ".xz", Level: level}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported compression %q", name)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Compressor) Wrap(w io.Writer) (io.WriteCloser, error) {
|
||||
switch c.Name {
|
||||
case "gzip":
|
||||
return gzip.NewWriterLevel(w, gzipLevel(c.Level))
|
||||
case "zstd":
|
||||
return zstd.NewWriter(w, zstd.WithEncoderLevel(zstdLevel(c.Level)))
|
||||
case "xz":
|
||||
xw, err := xz.WriterConfig{DictCap: xzDictCap(c.Level)}.NewWriter(w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return xw, nil
|
||||
default:
|
||||
return nopCloser{w}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Reader returns a decompressing reader for an existing shard.
|
||||
func (c *Compressor) Reader(r io.Reader) (io.ReadCloser, error) {
|
||||
switch c.Name {
|
||||
case "gzip":
|
||||
return gzip.NewReader(r)
|
||||
case "zstd":
|
||||
zr, err := zstd.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return zr.IOReadCloser(), nil
|
||||
case "xz":
|
||||
xr, err := xz.ReaderConfig{DictCap: xzReadDictCap}.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.NopCloser(xr), nil
|
||||
default:
|
||||
return io.NopCloser(r), nil
|
||||
}
|
||||
}
|
||||
|
||||
// gzipLevel passes the neutral level straight through.
|
||||
func gzipLevel(level int) int {
|
||||
return level
|
||||
}
|
||||
|
||||
func zstdLevel(level int) zstd.EncoderLevel {
|
||||
switch {
|
||||
case level <= 2:
|
||||
return zstd.SpeedFastest
|
||||
case level <= 5:
|
||||
return zstd.SpeedDefault
|
||||
case level <= 8:
|
||||
return zstd.SpeedBetterCompression
|
||||
default:
|
||||
return zstd.SpeedBestCompression
|
||||
}
|
||||
}
|
||||
|
||||
// xzDictCap maps the neutral level onto xz dictionary sizes; the library has
|
||||
// no preset parameter, so the dictionary is the only strength knob.
|
||||
func xzDictCap(level int) int {
|
||||
switch level {
|
||||
case 1:
|
||||
return 1 << 20
|
||||
case 2:
|
||||
return 2 << 20
|
||||
case 3, 4:
|
||||
return 4 << 20
|
||||
case 5, 6:
|
||||
return 8 << 20
|
||||
case 7:
|
||||
return 16 << 20
|
||||
case 8:
|
||||
return 32 << 20
|
||||
default:
|
||||
return 64 << 20
|
||||
}
|
||||
}
|
||||
|
||||
type nopCloser struct{ io.Writer }
|
||||
|
||||
func (nopCloser) Close() error { return nil }
|
||||
|
||||
type flushCloser interface {
|
||||
Flush() error
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Entry is one line of manifest.jsonl.
|
||||
type Entry struct {
|
||||
Seq int64 `json:"seq"`
|
||||
Path string `json:"path"`
|
||||
Date string `json:"date,omitempty"`
|
||||
PKLo string `json:"pk_lo"`
|
||||
PKHi string `json:"pk_hi"`
|
||||
Rows int64 `json:"rows"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
ClosedAt string `json:"closed_at"`
|
||||
}
|
||||
|
||||
// Manifest appends sealed-shard records to manifest.jsonl and knows which
|
||||
// shards already exist.
|
||||
type Manifest struct {
|
||||
path string
|
||||
f *os.File
|
||||
seen map[string]bool
|
||||
entries []Entry
|
||||
}
|
||||
|
||||
const manifestName = "manifest.jsonl"
|
||||
|
||||
func OpenManifest(root string) (*Manifest, error) {
|
||||
path := filepath.Join(root, manifestName)
|
||||
m := &Manifest{path: path, seen: map[string]bool{}}
|
||||
if err := m.readExisting(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open manifest: %w", err)
|
||||
}
|
||||
m.f = f
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *Manifest) readExisting() error {
|
||||
f, err := os.Open(m.path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read manifest: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
sc := bufio.NewScanner(f)
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var e Entry
|
||||
if err := json.Unmarshal([]byte(line), &e); err != nil {
|
||||
continue
|
||||
}
|
||||
m.entries = append(m.entries, e)
|
||||
m.seen[e.Path] = true
|
||||
}
|
||||
return sc.Err()
|
||||
}
|
||||
|
||||
func (m *Manifest) Has(rel string) bool { return m.seen[rel] }
|
||||
|
||||
func (m *Manifest) Entries() []Entry { return m.entries }
|
||||
|
||||
// Append writes one record and fsyncs it. A sealed shard is only considered
|
||||
// complete once its manifest line is durable.
|
||||
func (m *Manifest) Append(e Entry) error {
|
||||
if e.ClosedAt == "" {
|
||||
e.ClosedAt = time.Now().UTC().Format(time.RFC3339)
|
||||
}
|
||||
b, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b = append(b, '\n')
|
||||
if _, err := m.f.Write(b); err != nil {
|
||||
return fmt.Errorf("append manifest: %w", err)
|
||||
}
|
||||
if err := m.f.Sync(); err != nil {
|
||||
return fmt.Errorf("sync manifest: %w", err)
|
||||
}
|
||||
m.seen[e.Path] = true
|
||||
m.entries = append(m.entries, e)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manifest) Close() error {
|
||||
if m.f == nil {
|
||||
return nil
|
||||
}
|
||||
return m.f.Close()
|
||||
}
|
||||
|
||||
var shardNameRE = regexp.MustCompile(`^(\d+)\..*?\.([A-Za-z0-9]+?)(\.gz|\.zst|\.xz)?$`)
|
||||
|
||||
// Adopt scans the output tree for sealed shard files that have no manifest
|
||||
// record (a crash after rename but before the manifest append) and records
|
||||
// them, so resume never overwrites durable data. It returns the maximum seq
|
||||
// seen per bucket.
|
||||
func (m *Manifest) Adopt(root, format string, comp *Compressor) (map[string]int64, error) {
|
||||
maxSeq := map[string]int64{}
|
||||
root = filepath.Clean(root)
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
if name == manifestName || name == "state.json" || name == "state.json.tmp" ||
|
||||
strings.HasSuffix(name, ".partial") || strings.HasSuffix(name, ".partial.meta") {
|
||||
return nil
|
||||
}
|
||||
mt := shardNameRE.FindStringSubmatch(name)
|
||||
if mt == nil {
|
||||
return nil
|
||||
}
|
||||
if mt[2] != format || mt[3] != comp.Ext {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
bucket := bucketOfRel(rel)
|
||||
var seq int64
|
||||
fmt.Sscanf(mt[1], "%d", &seq)
|
||||
if seq > maxSeq[bucket] {
|
||||
maxSeq[bucket] = seq
|
||||
}
|
||||
if m.seen[rel] {
|
||||
return nil
|
||||
}
|
||||
rows, err := countRows(path, comp, format)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sum, size, err := hashFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entry := Entry{
|
||||
Seq: seq,
|
||||
Path: rel,
|
||||
Date: bucket,
|
||||
Rows: rows,
|
||||
Bytes: size,
|
||||
SHA256: sum,
|
||||
}
|
||||
return m.Append(entry)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("adopt sealed shards: %w", err)
|
||||
}
|
||||
return maxSeq, nil
|
||||
}
|
||||
|
||||
func bucketOfRel(rel string) string {
|
||||
dir := filepath.ToSlash(filepath.Dir(rel))
|
||||
if dir == "." {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(dir, "/")
|
||||
for _, p := range parts {
|
||||
if strings.HasPrefix(p, "date=") {
|
||||
return strings.TrimPrefix(p, "date=")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// MaxSeqByBucket returns the highest seq recorded per bucket.
|
||||
func (m *Manifest) MaxSeqByBucket() map[string]int64 {
|
||||
out := map[string]int64{}
|
||||
for _, e := range m.entries {
|
||||
if e.Seq > out[e.Date] {
|
||||
out[e.Date] = e.Seq
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func hashFile(path string) (string, int64, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(h, f)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), n, nil
|
||||
}
|
||||
|
||||
func countRows(path string, comp *Compressor, format string) (int64, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
r, err := comp.Reader(f)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer r.Close()
|
||||
br := bufio.NewReaderSize(r, 256*1024)
|
||||
var n int64
|
||||
for {
|
||||
_, err := br.ReadBytes('\n')
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n++
|
||||
}
|
||||
if format == "csv" && n > 0 {
|
||||
n--
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// CleanupPartials removes leftover .partial files and their metadata, keeping
|
||||
// keepPartial (if non-empty) and its sidecar.
|
||||
func CleanupPartials(root, keepPartial string) error {
|
||||
var paths []string
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if keepPartial != "" && (path == keepPartial || path == keepPartial+".meta") {
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
if strings.HasSuffix(name, ".partial") || strings.HasSuffix(name, ".partial.meta") {
|
||||
paths = append(paths, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Strings(paths)
|
||||
for _, p := range paths {
|
||||
if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
// FinalName builds the sealed shard filename:
|
||||
// <seq:06d>.<lo>-<hi>.<format>[.<comp>]
|
||||
func FinalName(seq int64, lo, hi, format, compExt string) string {
|
||||
return fmt.Sprintf("%06d.%s-%s.%s%s", seq, safeNamePart(lo), safeNamePart(hi), format, compExt)
|
||||
}
|
||||
|
||||
// KeyFilePart renders a key tuple into a filename-safe fragment.
|
||||
func KeyFilePart(k row.Key) string {
|
||||
if len(k) == 0 {
|
||||
return "NULL"
|
||||
}
|
||||
parts := make([]string, len(k))
|
||||
for i, p := range k {
|
||||
if p.Null {
|
||||
parts[i] = "NULL"
|
||||
} else {
|
||||
parts[i] = p.Val
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
func safeNamePart(s string) string {
|
||||
if s == "" {
|
||||
return "_"
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == '.', r == '_', r == '-':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteByte('_')
|
||||
}
|
||||
if b.Len() >= 96 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if b.Len() == 0 {
|
||||
return "_"
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// BucketDir returns the directory that holds shards for a bucket.
|
||||
func BucketDir(root, bucket string) string {
|
||||
if bucket == "" {
|
||||
return root
|
||||
}
|
||||
return filepath.Join(root, "date="+safeNamePart(bucket))
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"archer/internal/atomicfile"
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
// PartialMeta describes the durable prefix of an in-progress shard. It is
|
||||
// written before each chunk is committed, so a resume can truncate the file
|
||||
// back to a consistent offset and continue appending.
|
||||
type PartialMeta struct {
|
||||
Seq int64 `json:"seq"`
|
||||
Bucket string `json:"bucket,omitempty"`
|
||||
Rows int64 `json:"rows"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
FirstKey row.Key `json:"first_key"`
|
||||
LastKey row.Key `json:"last_key"`
|
||||
}
|
||||
|
||||
func partialPath(dir string, seq int64) string {
|
||||
return filepath.Join(dir, fmt.Sprintf("%06d.partial", seq))
|
||||
}
|
||||
|
||||
func partialMetaPath(dir string, seq int64) string {
|
||||
return partialPath(dir, seq) + ".meta"
|
||||
}
|
||||
|
||||
// PartialPath exposes the partial path for tests and diagnostics.
|
||||
func PartialPath(root, bucket string, seq int64) string {
|
||||
return partialPath(BucketDir(root, bucket), seq)
|
||||
}
|
||||
|
||||
// LoadPartialMeta reads the sidecar for an in-progress shard. A missing file
|
||||
// returns (nil, nil).
|
||||
func LoadPartialMeta(root, bucket string, seq int64) (*PartialMeta, error) {
|
||||
path := partialMetaPath(BucketDir(root, bucket), seq)
|
||||
b, err := os.ReadFile(path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read partial meta: %w", err)
|
||||
}
|
||||
var m PartialMeta
|
||||
if err := json.Unmarshal(b, &m); err != nil {
|
||||
return nil, fmt.Errorf("parse partial meta: %w", err)
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
func savePartialMeta(dir string, m *PartialMeta) error {
|
||||
path := partialMetaPath(dir, m.Seq)
|
||||
b, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := f.Write(b); err != nil {
|
||||
f.Close()
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
f.Close()
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := atomicfile.Replace(tmp, path); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func removePartialMeta(dir string, seq int64) {
|
||||
os.Remove(partialMetaPath(dir, seq))
|
||||
}
|
||||
@@ -0,0 +1,426 @@
|
||||
// Package shard writes row streams into ordered, independently importable
|
||||
// shard files and records them in manifest.jsonl.
|
||||
package shard
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"archer/internal/atomicfile"
|
||||
"archer/internal/row"
|
||||
"archer/internal/schema"
|
||||
)
|
||||
|
||||
// Options configures a Writer.
|
||||
type Options struct {
|
||||
Dir string
|
||||
Mode string // none|count|date
|
||||
ShardRows int64
|
||||
Format string
|
||||
Compressor *Compressor
|
||||
Columns []row.Column
|
||||
OutIdx []int
|
||||
CSVNull string
|
||||
StartBucket string
|
||||
StartSeq int64
|
||||
}
|
||||
|
||||
// Writer owns the single active shard file.
|
||||
type Writer struct {
|
||||
opt Options
|
||||
man *Manifest
|
||||
seq int64
|
||||
curBucket string
|
||||
cur *shardFile
|
||||
|
||||
totalRows int64
|
||||
shards int64
|
||||
bytes int64
|
||||
}
|
||||
|
||||
type shardFile struct {
|
||||
bucket string
|
||||
seq int64
|
||||
dir string
|
||||
partial string
|
||||
file *os.File
|
||||
cw *countWriter
|
||||
compW io.WriteCloser
|
||||
bw *bufio.Writer
|
||||
enc row.Encoder
|
||||
buf []byte
|
||||
rows int64
|
||||
memberRows int64
|
||||
firstKey row.Key
|
||||
lastKey row.Key
|
||||
}
|
||||
|
||||
type countWriter struct {
|
||||
w io.Writer
|
||||
n int64
|
||||
}
|
||||
|
||||
func (c *countWriter) Write(p []byte) (int, error) {
|
||||
n, err := c.w.Write(p)
|
||||
c.n += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Open creates a writer. It does not open a shard until the first row arrives.
|
||||
func Open(opt Options, man *Manifest) (*Writer, error) {
|
||||
w := &Writer{opt: opt, man: man, seq: opt.StartSeq, curBucket: opt.StartBucket}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// Resume reopens an existing partial shard and continues appending to it.
|
||||
func (w *Writer) Resume(m *PartialMeta) error {
|
||||
dir := BucketDir(w.opt.Dir, m.Bucket)
|
||||
path := partialPath(dir, m.Seq)
|
||||
f, err := os.OpenFile(path, os.O_RDWR, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resume partial shard: %w", err)
|
||||
}
|
||||
st, err := f.Stat()
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if st.Size() < m.Bytes {
|
||||
f.Close()
|
||||
return fmt.Errorf("partial shard %s is shorter (%d bytes) than its metadata (%d bytes); refusing to resume", path, st.Size(), m.Bytes)
|
||||
}
|
||||
if st.Size() > m.Bytes {
|
||||
if err := f.Truncate(m.Bytes); err != nil {
|
||||
f.Close()
|
||||
return fmt.Errorf("truncate partial shard: %w", err)
|
||||
}
|
||||
}
|
||||
if _, err := f.Seek(m.Bytes, io.SeekStart); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
enc, err := row.NewEncoder(w.opt.Format, w.opt.Columns, w.opt.OutIdx, row.Options{CSVNull: w.opt.CSVNull})
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
sf := &shardFile{
|
||||
bucket: m.Bucket,
|
||||
seq: m.Seq,
|
||||
dir: dir,
|
||||
partial: path,
|
||||
file: f,
|
||||
enc: enc,
|
||||
rows: m.Rows,
|
||||
memberRows: m.Rows,
|
||||
firstKey: m.FirstKey,
|
||||
lastKey: m.LastKey,
|
||||
}
|
||||
if err := sf.startChain(m.Bytes); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
w.cur = sf
|
||||
w.seq = m.Seq
|
||||
w.curBucket = m.Bucket
|
||||
w.totalRows = m.Rows
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sf *shardFile) startChain(offset int64) error {
|
||||
sf.cw = &countWriter{w: sf.file, n: offset}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sf *shardFile) openChain(comp *Compressor, bufSize int) error {
|
||||
cw, err := comp.Wrap(sf.cw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sf.compW = cw
|
||||
sf.bw = bufio.NewWriterSize(cw, bufSize)
|
||||
sf.memberRows = sf.rows
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) openShard(bucket string) error {
|
||||
if w.cur != nil {
|
||||
return fmt.Errorf("internal: shard already open")
|
||||
}
|
||||
if bucket != w.curBucket {
|
||||
w.seq = 0
|
||||
w.curBucket = bucket
|
||||
}
|
||||
w.seq++
|
||||
dir := BucketDir(w.opt.Dir, bucket)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("create shard directory: %w", err)
|
||||
}
|
||||
path := partialPath(dir, w.seq)
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create partial shard: %w", err)
|
||||
}
|
||||
enc, err := row.NewEncoder(w.opt.Format, w.opt.Columns, w.opt.OutIdx, row.Options{CSVNull: w.opt.CSVNull})
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
sf := &shardFile{bucket: bucket, seq: w.seq, dir: dir, partial: path, file: f, enc: enc}
|
||||
if err := sf.startChain(0); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if hdr := enc.Header(); len(hdr) > 0 {
|
||||
if _, err := sf.bw.Write(hdr); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
}
|
||||
w.cur = sf
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteChunk routes chunk rows into the active shard, sealing on bucket change
|
||||
// or when --shard-rows is reached. A chunk may span multiple shards.
|
||||
func (w *Writer) WriteChunk(c *row.Chunk) error {
|
||||
for i := range c.Rows {
|
||||
bucket := ""
|
||||
if w.opt.Mode == "date" {
|
||||
bucket = c.Buckets[i]
|
||||
}
|
||||
if w.cur != nil {
|
||||
needSeal := false
|
||||
if bucket != w.cur.bucket {
|
||||
if !schema.BucketLess(w.cur.bucket, bucket) {
|
||||
return fmt.Errorf("rows arrived out of shard order: %q after %q", bucket, w.cur.bucket)
|
||||
}
|
||||
needSeal = true
|
||||
} else if w.opt.ShardRows > 0 && w.cur.rows >= w.opt.ShardRows {
|
||||
needSeal = true
|
||||
}
|
||||
if needSeal {
|
||||
if err := w.seal(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if w.cur == nil {
|
||||
if err := w.openShard(bucket); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := w.cur.writeRow(c.Rows[i]); err != nil {
|
||||
return err
|
||||
}
|
||||
w.cur.trackKey(c.Keys[i])
|
||||
w.totalRows++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sf *shardFile) writeRow(r row.Row) error {
|
||||
var err error
|
||||
sf.buf, err = sf.enc.Encode(sf.buf[:0], r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := sf.bw.Write(sf.buf); err != nil {
|
||||
return fmt.Errorf("write shard row: %w", err)
|
||||
}
|
||||
sf.rows++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sf *shardFile) trackKey(k row.Key) {
|
||||
if sf.firstKey.IsZero() {
|
||||
sf.firstKey = k
|
||||
}
|
||||
sf.lastKey = k
|
||||
}
|
||||
|
||||
// Checkpoint makes the current shard bytes and their metadata durable. It is
|
||||
// called before destructive database work when --purge is enabled.
|
||||
func (w *Writer) Checkpoint() error {
|
||||
if w.cur == nil {
|
||||
return nil
|
||||
}
|
||||
sf := w.cur
|
||||
if sf.rows > sf.memberRows {
|
||||
if err := sf.closeMember(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
meta := &PartialMeta{
|
||||
Seq: sf.seq,
|
||||
Bucket: sf.bucket,
|
||||
Rows: sf.rows,
|
||||
Bytes: sf.cw.n,
|
||||
FirstKey: sf.firstKey,
|
||||
LastKey: sf.lastKey,
|
||||
}
|
||||
if err := savePartialMeta(sf.dir, meta); err != nil {
|
||||
return fmt.Errorf("write partial metadata: %w", err)
|
||||
}
|
||||
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sf *shardFile) closeMember() error {
|
||||
if err := sf.bw.Flush(); err != nil {
|
||||
return fmt.Errorf("flush shard: %w", err)
|
||||
}
|
||||
if err := sf.compW.Close(); err != nil {
|
||||
return fmt.Errorf("close shard compressor: %w", err)
|
||||
}
|
||||
if err := sf.file.Sync(); err != nil {
|
||||
return fmt.Errorf("sync shard: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Flush pushes buffered rows to the OS without sealing the shard.
|
||||
func (w *Writer) Flush() error {
|
||||
if w.cur == nil || w.cur.bw == nil {
|
||||
return nil
|
||||
}
|
||||
return w.cur.bw.Flush()
|
||||
}
|
||||
|
||||
// SealCurrent closes and publishes the active shard even if it is not full.
|
||||
func (w *Writer) SealCurrent() error {
|
||||
if w.cur == nil {
|
||||
return nil
|
||||
}
|
||||
if w.cur.rows == 0 {
|
||||
return w.discardCurrent()
|
||||
}
|
||||
return w.seal()
|
||||
}
|
||||
|
||||
func (w *Writer) discardCurrent() error {
|
||||
sf := w.cur
|
||||
sf.file.Close()
|
||||
os.Remove(sf.partial)
|
||||
removePartialMeta(sf.dir, sf.seq)
|
||||
w.cur = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) seal() error {
|
||||
sf := w.cur
|
||||
if sf.rows == 0 {
|
||||
return w.discardCurrent()
|
||||
}
|
||||
if sf.rows > sf.memberRows || sf.compW != nil {
|
||||
if err := sf.closeMember(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := sf.file.Close(); err != nil {
|
||||
return fmt.Errorf("close partial shard: %w", err)
|
||||
}
|
||||
lo := KeyFilePart(sf.firstKey)
|
||||
hi := KeyFilePart(sf.lastKey)
|
||||
name := FinalName(sf.seq, lo, hi, w.opt.Format, w.opt.Compressor.Ext)
|
||||
finalPath := filepath.Join(sf.dir, name)
|
||||
if err := os.Remove(finalPath); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("remove stale shard: %w", err)
|
||||
}
|
||||
if err := atomicfile.Replace(sf.partial, finalPath); err != nil {
|
||||
return fmt.Errorf("publish shard: %w", err)
|
||||
}
|
||||
sum, size, err := hashFile(finalPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash shard: %w", err)
|
||||
}
|
||||
rel, err := filepath.Rel(w.opt.Dir, finalPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entry := Entry{
|
||||
Seq: sf.seq,
|
||||
Path: filepath.ToSlash(rel),
|
||||
Date: sf.bucket,
|
||||
PKLo: lo,
|
||||
PKHi: hi,
|
||||
Rows: sf.rows,
|
||||
Bytes: size,
|
||||
SHA256: sum,
|
||||
}
|
||||
if err := w.man.Append(entry); err != nil {
|
||||
return err
|
||||
}
|
||||
removePartialMeta(sf.dir, sf.seq)
|
||||
w.shards++
|
||||
w.bytes += size
|
||||
w.cur = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close publishes any in-progress shard and closes the manifest.
|
||||
func (w *Writer) Close() error {
|
||||
if err := w.SealCurrent(); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.man.Close()
|
||||
}
|
||||
|
||||
// StatePosition reports the bucket and the last sealed seq within that bucket,
|
||||
// which is what a checkpoint must record for resume.
|
||||
func (w *Writer) StatePosition() (bucket string, sealedSeq int64) {
|
||||
if w.cur != nil {
|
||||
return w.cur.bucket, w.cur.seq - 1
|
||||
}
|
||||
return w.curBucket, w.seq
|
||||
}
|
||||
|
||||
// Abandon closes file handles without sealing the active shard, leaving the
|
||||
// partial and its metadata for a later resume.
|
||||
func (w *Writer) Abandon() error {
|
||||
if w.cur != nil {
|
||||
if w.cur.bw != nil {
|
||||
_ = w.cur.bw.Flush()
|
||||
}
|
||||
_ = w.cur.file.Close()
|
||||
w.cur = nil
|
||||
}
|
||||
return w.man.Close()
|
||||
}
|
||||
|
||||
// Stats returns rows written, shards sealed and bytes sealed.
|
||||
func (w *Writer) Stats() (rows, shards, bytes int64) {
|
||||
return w.totalRows, w.shards, w.bytes
|
||||
}
|
||||
|
||||
// CurrentShard returns the sequence number of the active shard, or the last
|
||||
// one when no shard is currently open.
|
||||
func (w *Writer) CurrentShard() int64 {
|
||||
if w.cur != nil {
|
||||
return w.cur.seq
|
||||
}
|
||||
return w.seq
|
||||
}
|
||||
|
||||
// WrittenBytes returns sealed bytes plus the bytes already written to the
|
||||
// active partial shard.
|
||||
func (w *Writer) WrittenBytes() int64 {
|
||||
n := w.bytes
|
||||
if w.cur != nil && w.cur.cw != nil {
|
||||
n += w.cur.cw.n
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,459 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
func testOpts(dir, mode string) Options {
|
||||
return Options{
|
||||
Dir: dir,
|
||||
Mode: mode,
|
||||
Format: "dump",
|
||||
Compressor: &Compressor{Name: "none"},
|
||||
Columns: []row.Column{{Name: "id", Type: "int"}, {Name: "v", Type: "varchar"}},
|
||||
}
|
||||
}
|
||||
|
||||
func chunk(start int, values ...string) *row.Chunk {
|
||||
c := &row.Chunk{}
|
||||
for i, v := range values {
|
||||
id := start + i
|
||||
c.Rows = append(c.Rows, row.Row{Values: []row.Value{
|
||||
{Raw: []byte(itoa(id))},
|
||||
{Raw: []byte(v)},
|
||||
}})
|
||||
c.Keys = append(c.Keys, row.KeyOf(itoa(id)))
|
||||
}
|
||||
if len(c.Keys) > 0 {
|
||||
c.FirstKey = c.Keys[0]
|
||||
c.LastKey = c.Keys[len(c.Keys)-1]
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func itoa(i int) string {
|
||||
if i == 0 {
|
||||
return "0"
|
||||
}
|
||||
var b [20]byte
|
||||
p := len(b)
|
||||
neg := i < 0
|
||||
if neg {
|
||||
i = -i
|
||||
}
|
||||
for i > 0 {
|
||||
p--
|
||||
b[p] = byte('0' + i%10)
|
||||
i /= 10
|
||||
}
|
||||
if neg {
|
||||
p--
|
||||
b[p] = '-'
|
||||
}
|
||||
return string(b[p:])
|
||||
}
|
||||
|
||||
func TestCountShardsOrderAndNames(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, err := OpenManifest(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opt := testOpts(dir, "count")
|
||||
opt.ShardRows = 2
|
||||
w, err := Open(opt, man)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A chunk of 3 rows spans two shards.
|
||||
if err := w.WriteChunk(chunk(1, "a", "b", "c")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.WriteChunk(chunk(4, "d", "e")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
want := map[string]string{
|
||||
"000001.1-2.dump": "1\ta\n2\tb\n",
|
||||
"000002.3-4.dump": "3\tc\n4\td\n",
|
||||
"000003.5-5.dump": "5\te\n",
|
||||
}
|
||||
for name, content := range want {
|
||||
b, err := os.ReadFile(filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if string(b) != content {
|
||||
t.Fatalf("%s: got %q want %q", name, b, content)
|
||||
}
|
||||
}
|
||||
entries := man.Entries()
|
||||
if len(entries) != 3 {
|
||||
t.Fatalf("manifest entries: %d", len(entries))
|
||||
}
|
||||
if entries[0].PKLo != "1" || entries[0].PKHi != "2" || entries[0].Rows != 2 {
|
||||
t.Fatalf("entry0 %+v", entries[0])
|
||||
}
|
||||
if entries[2].Path != "000003.5-5.dump" || entries[2].SHA256 == "" {
|
||||
t.Fatalf("entry2 %+v", entries[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDateShardsDirectories(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
opt := testOpts(dir, "date")
|
||||
w, _ := Open(opt, man)
|
||||
c := chunk(1, "a", "b", "c")
|
||||
c.Buckets = []string{"2025-06-01", "2025-06-01", "2025-06-02"}
|
||||
if err := w.WriteChunk(c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "date=2025-06-01", "000001.1-2.dump")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "date=2025-06-02", "000001.3-3.dump")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDateOrderViolation(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
w, _ := Open(testOpts(dir, "date"), man)
|
||||
c := chunk(1, "a", "b")
|
||||
c.Buckets = []string{"2025-06-02", "2025-06-01"}
|
||||
if err := w.WriteChunk(c); err == nil {
|
||||
t.Fatal("expected ordering error")
|
||||
}
|
||||
_ = w.Abandon()
|
||||
}
|
||||
|
||||
func TestResumeAppendsAfterTruncatingGarbage(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
opt := testOpts(dir, "count")
|
||||
opt.ShardRows = 10
|
||||
w, _ := Open(opt, man)
|
||||
if err := w.WriteChunk(chunk(1, "a", "b")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Checkpoint(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := LoadPartialMeta(dir, "", 1)
|
||||
if err != nil || meta == nil {
|
||||
t.Fatalf("meta %v %v", meta, err)
|
||||
}
|
||||
if err := w.Abandon(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Simulate bytes from an uncommitted chunk that were flushed but never
|
||||
// checkpointed: resume must truncate them.
|
||||
partial := PartialPath(dir, "", 1)
|
||||
f, err := os.OpenFile(partial, os.O_APPEND|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.WriteString("garbage\n")
|
||||
f.Close()
|
||||
|
||||
man2, _ := OpenManifest(dir)
|
||||
w2, err := Open(opt, man2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w2.Resume(meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w2.WriteChunk(chunk(3, "c", "d")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w2.Checkpoint(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w2.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(dir, "000001.1-4.dump"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(b) != "1\ta\n2\tb\n3\tc\n4\td\n" {
|
||||
t.Fatalf("got %q", b)
|
||||
}
|
||||
if _, err := os.Stat(partial); !os.IsNotExist(err) {
|
||||
t.Fatal("partial should be gone after seal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestAdoptOrphan(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
orphan := filepath.Join(dir, "000007.9-12.dump")
|
||||
if err := os.WriteFile(orphan, []byte("9\tx\n10\ty\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
comp := &Compressor{Name: "none"}
|
||||
maxSeq, err := man.Adopt(dir, "dump", comp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if maxSeq[""] != 7 {
|
||||
t.Fatalf("maxSeq %+v", maxSeq)
|
||||
}
|
||||
if len(man.Entries()) != 1 || man.Entries()[0].Rows != 2 {
|
||||
t.Fatalf("entries %+v", man.Entries())
|
||||
}
|
||||
// A second adopt must not duplicate the entry.
|
||||
if _, err := man.Adopt(dir, "dump", comp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(man.Entries()) != 1 {
|
||||
t.Fatalf("entries %+v", man.Entries())
|
||||
}
|
||||
man.Close()
|
||||
}
|
||||
|
||||
func TestSafeNamePart(t *testing.T) {
|
||||
got := safeNamePart("2024-01-01 00:00:00")
|
||||
if strings.ContainsAny(got, `:/\*?"<>|`) {
|
||||
t.Fatalf("unsafe characters kept: %q", got)
|
||||
}
|
||||
if got != "2024-01-01_00_00_00" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if safeNamePart("") != "_" {
|
||||
t.Fatal("empty name part")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressedShardIsIndependent(t *testing.T) {
|
||||
for _, compName := range []string{"gzip", "zstd", "xz"} {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
comp, err := NewCompressor(compName, DefaultLevel)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opt := testOpts(dir, "none")
|
||||
opt.Compressor = comp
|
||||
w, _ := Open(opt, man)
|
||||
if err := w.WriteChunk(chunk(1, "a", "b")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var path string
|
||||
entries := man.Entries()
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("%s: entries %d", compName, len(entries))
|
||||
}
|
||||
path = filepath.Join(dir, entries[0].Path)
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := comp.Reader(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var b strings.Builder
|
||||
buf := make([]byte, 1024)
|
||||
for {
|
||||
n, err := r.Read(buf)
|
||||
b.Write(buf[:n])
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
r.Close()
|
||||
f.Close()
|
||||
if b.String() != "1\ta\n2\tb\n" {
|
||||
t.Fatalf("%s: got %q", compName, b.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressionLevelMapping(t *testing.T) {
|
||||
xzDictMiB := []int{1, 2, 4, 4, 8, 8, 16, 32, 64}
|
||||
for level := 1; level <= 9; level++ {
|
||||
c, err := NewCompressor("gzip", level)
|
||||
if err != nil {
|
||||
t.Fatalf("level %d: %v", level, err)
|
||||
}
|
||||
if c.Level != level || c.Name != "gzip" || c.Ext != ".gz" {
|
||||
t.Fatalf("level %d: %+v", level, c)
|
||||
}
|
||||
if got := gzipLevel(level); got != level {
|
||||
t.Fatalf("gzip level %d -> %d", level, got)
|
||||
}
|
||||
|
||||
var wantZstd zstd.EncoderLevel
|
||||
switch {
|
||||
case level <= 2:
|
||||
wantZstd = zstd.SpeedFastest
|
||||
case level <= 5:
|
||||
wantZstd = zstd.SpeedDefault
|
||||
case level <= 8:
|
||||
wantZstd = zstd.SpeedBetterCompression
|
||||
default:
|
||||
wantZstd = zstd.SpeedBestCompression
|
||||
}
|
||||
if got := zstdLevel(level); got != wantZstd {
|
||||
t.Fatalf("zstd level %d -> %v want %v", level, got, wantZstd)
|
||||
}
|
||||
if got, want := xzDictCap(level), xzDictMiB[level-1]<<20; got != want {
|
||||
t.Fatalf("xz level %d -> %d want %d", level, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range []string{"none", "gzip", "zstd", "xz"} {
|
||||
for _, level := range []int{-1, 0, 10} {
|
||||
if _, err := NewCompressor(name, level); err == nil {
|
||||
t.Fatalf("%s level %d: expected error", name, level)
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := NewCompressor("brotli", DefaultLevel); err == nil {
|
||||
t.Fatal("unsupported compressor must fail")
|
||||
}
|
||||
|
||||
for name, ext := range map[string]string{"none": "", "gzip": ".gz", "zstd": ".zst", "xz": ".xz"} {
|
||||
c, err := NewCompressor(name, DefaultLevel)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if c.Ext != ext {
|
||||
t.Fatalf("%s: ext %q want %q", name, c.Ext, ext)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestXZCheckpointKeepsAllStreams(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
defer man.Close()
|
||||
comp, err := NewCompressor("xz", 9)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opt := testOpts(dir, "none")
|
||||
opt.Compressor = comp
|
||||
w, _ := Open(opt, man)
|
||||
if err := w.WriteChunk(chunk(1, "a", "b")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Checkpoint(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.WriteChunk(chunk(3, "c", "d")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
entries := man.Entries()
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("entries %d", len(entries))
|
||||
}
|
||||
if entries[0].Rows != 4 {
|
||||
t.Fatalf("rows %d", entries[0].Rows)
|
||||
}
|
||||
if !strings.HasSuffix(entries[0].Path, ".dump.xz") {
|
||||
t.Fatalf("path %q", entries[0].Path)
|
||||
}
|
||||
|
||||
f, err := os.Open(filepath.Join(dir, entries[0].Path))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
r, err := comp.Reader(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
got, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != "1\ta\n2\tb\n3\tc\n4\td\n" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestAdoptXZOrphan(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
defer man.Close()
|
||||
comp, err := NewCompressor("xz", DefaultLevel)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
xw, err := comp.Wrap(&buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := xw.Write([]byte("9\tx\n10\ty\n")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := xw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
orphan := filepath.Join(dir, "000007.9-12.dump.xz")
|
||||
if err := os.WriteFile(orphan, buf.Bytes(), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
maxSeq, err := man.Adopt(dir, "dump", comp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if maxSeq[""] != 7 {
|
||||
t.Fatalf("maxSeq %+v", maxSeq)
|
||||
}
|
||||
entries := man.Entries()
|
||||
if len(entries) != 1 || entries[0].Seq != 7 || entries[0].Rows != 2 {
|
||||
t.Fatalf("entries %+v", entries)
|
||||
}
|
||||
if entries[0].Path != "000007.9-12.dump.xz" {
|
||||
t.Fatalf("path %q", entries[0].Path)
|
||||
}
|
||||
|
||||
// A resumed writer must continue after the adopted seq, not overwrite it.
|
||||
opt := testOpts(dir, "none")
|
||||
opt.Compressor = comp
|
||||
opt.StartSeq = maxSeq[""]
|
||||
w, _ := Open(opt, man)
|
||||
if err := w.WriteChunk(chunk(1, "a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "000008.1-1.dump.xz")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(orphan); err != nil {
|
||||
t.Fatalf("adopted shard was overwritten: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package sink
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"archer/internal/row"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
// Execer is satisfied by *sql.DB, *sql.Tx and *sql.Conn.
|
||||
type Execer interface {
|
||||
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
|
||||
}
|
||||
|
||||
const defaultMaxPacket = 4 << 20
|
||||
|
||||
// DB inserts chunk rows into the destination table in batches that stay under
|
||||
// max_allowed_packet.
|
||||
type DB struct {
|
||||
DBName string
|
||||
Table string
|
||||
Columns []string
|
||||
Mode string // "", "ignore", "replace"
|
||||
MaxPacket int64
|
||||
OutIdx []int
|
||||
}
|
||||
|
||||
// NewDB reads max_allowed_packet and returns a sink. cols are the output
|
||||
// column names; outIdx maps them to positions in a scanned row.
|
||||
func NewDB(ctx context.Context, db *sql.DB, dbName, table string, cols []string, mode string, outIdx []int) (*DB, error) {
|
||||
maxPacket := int64(defaultMaxPacket)
|
||||
var v sql.NullInt64
|
||||
if err := db.QueryRowContext(ctx, "SELECT @@max_allowed_packet").Scan(&v); err == nil && v.Valid && v.Int64 > 0 {
|
||||
maxPacket = v.Int64
|
||||
}
|
||||
return &DB{
|
||||
DBName: dbName,
|
||||
Table: table,
|
||||
Columns: cols,
|
||||
Mode: mode,
|
||||
MaxPacket: maxPacket,
|
||||
OutIdx: outIdx,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *DB) table() string { return sqlutil.Qualified(d.DBName, d.Table) }
|
||||
|
||||
func (d *DB) verb() string {
|
||||
switch d.Mode {
|
||||
case "ignore":
|
||||
return "INSERT IGNORE INTO"
|
||||
case "replace":
|
||||
return "REPLACE INTO"
|
||||
default:
|
||||
return "INSERT INTO"
|
||||
}
|
||||
}
|
||||
|
||||
// InsertChunk writes all chunk rows through exec.
|
||||
func (d *DB) InsertChunk(ctx context.Context, exec Execer, c *row.Chunk) (int64, error) {
|
||||
if len(c.Rows) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
limit := d.MaxPacket * 8 / 10
|
||||
var total int64
|
||||
start := 0
|
||||
for start < len(c.Rows) {
|
||||
end := start
|
||||
var size int64
|
||||
for end < len(c.Rows) {
|
||||
rowSize := int64(8 + len(d.Columns)*4)
|
||||
for _, i := range d.OutIdx {
|
||||
rowSize += int64(len(c.Rows[end].Values[i].Raw))
|
||||
}
|
||||
if end > start && size+rowSize > limit {
|
||||
break
|
||||
}
|
||||
size += rowSize
|
||||
end++
|
||||
}
|
||||
if end == start {
|
||||
end = start + 1
|
||||
}
|
||||
n, err := d.insertBatch(ctx, exec, c.Rows[start:end])
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
total += n
|
||||
start = end
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (d *DB) insertBatch(ctx context.Context, exec Execer, rows []row.Row) (int64, error) {
|
||||
var b strings.Builder
|
||||
b.WriteString(d.verb())
|
||||
b.WriteByte(' ')
|
||||
b.WriteString(d.table())
|
||||
b.WriteString(" (")
|
||||
b.WriteString(sqlutil.Columns(d.Columns))
|
||||
b.WriteString(") VALUES ")
|
||||
tuple := sqlutil.ParamTuple(len(d.Columns))
|
||||
args := make([]any, 0, len(rows)*len(d.Columns))
|
||||
for n, r := range rows {
|
||||
if n > 0 {
|
||||
b.WriteByte(',')
|
||||
}
|
||||
b.WriteString(tuple)
|
||||
for _, i := range d.OutIdx {
|
||||
args = append(args, r.Values[i].DriverValue())
|
||||
}
|
||||
}
|
||||
res, err := exec.ExecContext(ctx, b.String(), args...)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("insert into %s: %w", d.Table, err)
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return 0, nil
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// DryRunSQL returns a representative insert statement.
|
||||
func (d *DB) DryRunSQL() []string {
|
||||
tuple := sqlutil.ParamTuple(len(d.Columns))
|
||||
return []string{fmt.Sprintf("%s %s (%s) VALUES %s, %s, ...",
|
||||
d.verb(), d.table(), sqlutil.Columns(d.Columns), tuple, tuple)}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
// Package sink contains the file and database write targets.
|
||||
package sink
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"archer/internal/row"
|
||||
"archer/internal/shard"
|
||||
)
|
||||
|
||||
// File writes chunks into shard files.
|
||||
type File struct {
|
||||
W *shard.Writer
|
||||
}
|
||||
|
||||
func (f *File) WriteChunk(_ context.Context, c *row.Chunk) error { return f.W.WriteChunk(c) }
|
||||
func (f *File) Flush(context.Context) error { return f.W.Flush() }
|
||||
func (f *File) Close() error { return f.W.Close() }
|
||||
|
||||
// Checkpoint makes buffered shard data durable before a destructive commit.
|
||||
func (f *File) Checkpoint() error { return f.W.Checkpoint() }
|
||||
@@ -0,0 +1,71 @@
|
||||
// Package sqlutil holds small helpers for building SQL text safely.
|
||||
package sqlutil
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Ident quotes a MySQL identifier.
|
||||
func Ident(s string) string {
|
||||
return "`" + strings.ReplaceAll(s, "`", "``") + "`"
|
||||
}
|
||||
|
||||
// Qualified quotes a database-qualified table name.
|
||||
func Qualified(db, table string) string {
|
||||
if db == "" {
|
||||
return Ident(table)
|
||||
}
|
||||
return Ident(db) + "." + Ident(table)
|
||||
}
|
||||
|
||||
// Columns joins quoted column names with commas.
|
||||
func Columns(cols []string) string {
|
||||
parts := make([]string, len(cols))
|
||||
for i, c := range cols {
|
||||
parts[i] = Ident(c)
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
// Placeholders returns n comma separated question marks.
|
||||
func Placeholders(n int) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.Repeat("?, ", n-1) + "?"
|
||||
}
|
||||
|
||||
// RowTuple renders a parenthesized, quoted column list.
|
||||
func RowTuple(cols []string) string {
|
||||
return "(" + Columns(cols) + ")"
|
||||
}
|
||||
|
||||
// ParamTuple renders a parenthesized list of n placeholders.
|
||||
func ParamTuple(n int) string {
|
||||
return "(" + Placeholders(n) + ")"
|
||||
}
|
||||
|
||||
// GreaterThan renders a keyset comparison (cols) > (?, ?, ...). For a single
|
||||
// column it emits the simpler col > ?.
|
||||
func GreaterThan(cols []string) string {
|
||||
if len(cols) == 1 {
|
||||
return Ident(cols[0]) + " > ?"
|
||||
}
|
||||
return RowTuple(cols) + " > " + ParamTuple(len(cols))
|
||||
}
|
||||
|
||||
// InList renders (cols) IN ((?,?),(?,?),...). count is the number of tuples.
|
||||
func InList(cols []string, count int) string {
|
||||
if count == 0 {
|
||||
return "1=0"
|
||||
}
|
||||
tuple := ParamTuple(len(cols))
|
||||
tuples := make([]string, count)
|
||||
for i := range tuples {
|
||||
tuples[i] = tuple
|
||||
}
|
||||
if len(cols) == 1 {
|
||||
return Ident(cols[0]) + " IN (" + Placeholders(count) + ")"
|
||||
}
|
||||
return RowTuple(cols) + " IN (" + strings.Join(tuples, ", ") + ")"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package sqlutil
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestIdentQuoting(t *testing.T) {
|
||||
if got := Ident("a`b"); got != "`a``b`" {
|
||||
t.Fatalf("got %s", got)
|
||||
}
|
||||
if got := Qualified("db", "t"); got != "`db`.`t`" {
|
||||
t.Fatalf("got %s", got)
|
||||
}
|
||||
if got := Qualified("", "t"); got != "`t`" {
|
||||
t.Fatalf("got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGreaterThan(t *testing.T) {
|
||||
if got := GreaterThan([]string{"id"}); got != "`id` > ?" {
|
||||
t.Fatalf("got %s", got)
|
||||
}
|
||||
if got := GreaterThan([]string{"a", "b"}); got != "(`a`, `b`) > (?, ?)" {
|
||||
t.Fatalf("got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInList(t *testing.T) {
|
||||
if got := InList([]string{"id"}, 3); got != "`id` IN (?, ?, ?)" {
|
||||
t.Fatalf("got %s", got)
|
||||
}
|
||||
if got := InList([]string{"a", "b"}, 2); got != "(`a`, `b`) IN ((?, ?), (?, ?))" {
|
||||
t.Fatalf("got %s", got)
|
||||
}
|
||||
if got := InList([]string{"id"}, 0); got != "1=0" {
|
||||
t.Fatalf("got %s", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
// Package version reports build information injected at link time.
|
||||
package version
|
||||
|
||||
import "fmt"
|
||||
|
||||
var (
|
||||
Version = "dev"
|
||||
Commit = "none"
|
||||
Date = "unknown"
|
||||
)
|
||||
|
||||
func String() string {
|
||||
return fmt.Sprintf("archer %s (commit %s, built %s)", Version, Commit, Date)
|
||||
}
|
||||
Reference in new issue
Block a user