archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
709 lines
18 KiB
Go
709 lines
18 KiB
Go
// Package app wires the CLI, schema probe, scanners, shard writer and
|
|
// database targets into one run.
|
|
package app
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"database/sql"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/go-sql-driver/mysql"
|
|
|
|
"archer/internal/archererr"
|
|
"archer/internal/checkpoint"
|
|
"archer/internal/cli"
|
|
"archer/internal/lag"
|
|
"archer/internal/progress"
|
|
"archer/internal/purge"
|
|
"archer/internal/row"
|
|
"archer/internal/scan"
|
|
"archer/internal/shard"
|
|
"archer/internal/sink"
|
|
)
|
|
|
|
// Stats collects counters and phase timings for --statistics.
|
|
type Stats struct {
|
|
Started time.Time
|
|
RowsRead int64
|
|
RowsCommitted int64
|
|
RowsInserted int64
|
|
RowsDeleted int64
|
|
ScanDur time.Duration
|
|
FileDur time.Duration
|
|
DBDur time.Duration
|
|
CheckpointDur time.Duration
|
|
Shards int64
|
|
Bytes int64
|
|
}
|
|
|
|
// App holds a run's state.
|
|
type App struct {
|
|
cfg *cli.Config
|
|
srcDB *sql.DB
|
|
destDB *sql.DB
|
|
srcCfg *mysql.Config
|
|
destCfg *mysql.Config
|
|
|
|
loc *time.Location
|
|
probe *probeResult
|
|
cfgHash string
|
|
cfgFields map[string]string
|
|
|
|
scanner scan.Scanner
|
|
fileSink *sink.File
|
|
writer *shard.Writer
|
|
manifest *shard.Manifest
|
|
destSink *sink.DB
|
|
purger *purge.Purger
|
|
lagChecker *lag.Checker
|
|
|
|
sameInstance bool
|
|
resume *scan.Resume
|
|
alreadyComplete bool
|
|
|
|
st *checkpoint.State
|
|
stats Stats
|
|
|
|
reporter *progress.Reporter
|
|
estimate *progress.Estimate
|
|
baseRows int64
|
|
}
|
|
|
|
// Run executes one archiving run.
|
|
func Run(ctx context.Context, cfg *cli.Config) error {
|
|
if err := cfg.Validate(); err != nil {
|
|
return err
|
|
}
|
|
a := &App{cfg: cfg, stats: Stats{Started: time.Now()}}
|
|
defer a.close()
|
|
if !cfg.DryRun {
|
|
a.startReporter()
|
|
defer a.reporter.Stop()
|
|
}
|
|
|
|
if err := a.setupSource(ctx); err != nil {
|
|
return err
|
|
}
|
|
if cfg.DryRun {
|
|
return a.printDryRun(os.Stdout)
|
|
}
|
|
if err := a.prepare(ctx); err != nil {
|
|
return err
|
|
}
|
|
if a.alreadyComplete {
|
|
fmt.Fprintf(os.Stdout, "archer: output in %s is already complete; nothing to do\n", cfg.FileDir)
|
|
return nil
|
|
}
|
|
return a.execute(ctx)
|
|
}
|
|
|
|
func (a *App) startReporter() {
|
|
tty := progress.IsTerminal(os.Stderr)
|
|
enabled := a.cfg.Progress.Enabled(tty)
|
|
a.reporter = progress.New(progress.Options{
|
|
Out: os.Stderr,
|
|
Enabled: enabled,
|
|
Live: enabled && tty,
|
|
Interval: a.cfg.ProgressInterval,
|
|
RowEvery: a.cfg.Progress.Rows,
|
|
})
|
|
a.reporter.Start()
|
|
}
|
|
|
|
func (a *App) phase(name, reason string) {
|
|
if a.reporter != nil {
|
|
a.reporter.Phase(name, reason)
|
|
}
|
|
}
|
|
|
|
func (a *App) message(format string, args ...any) {
|
|
if a.reporter != nil {
|
|
a.reporter.Message(format, args...)
|
|
return
|
|
}
|
|
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
|
}
|
|
|
|
func (a *App) setupSource(ctx context.Context) error {
|
|
a.phase("probe", "")
|
|
loc, err := time.LoadLocation(a.cfg.ShardTimeZone)
|
|
if err != nil {
|
|
return archererr.Newf(archererr.KindConfig, "invalid --shard-time-zone: %v", err)
|
|
}
|
|
a.loc = loc
|
|
srcCfg, err := a.cfg.SourceConfig()
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindConfig, err)
|
|
}
|
|
if srcCfg.DBName == "" {
|
|
return archererr.Newf(archererr.KindConfig, "--source must include a database name")
|
|
}
|
|
a.srcCfg = srcCfg
|
|
a.srcDB, err = openDB(srcCfg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
a.probe, err = probeTable(ctx, a.srcDB, a.cfg, srcCfg.DBName, loc)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
a.cfgHash, a.cfgFields, err = a.cfg.Fingerprint()
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindConfig, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (a *App) prepare(ctx context.Context) error {
|
|
a.phase("resume", "")
|
|
if err := a.openDest(ctx); err != nil {
|
|
return err
|
|
}
|
|
if a.cfg.Purge {
|
|
autoInc := false
|
|
if len(a.probe.key.Columns) == 1 {
|
|
if c, ok := a.probe.table.Column(a.probe.key.Columns[0]); ok {
|
|
autoInc = c.IsAutoIncrement()
|
|
}
|
|
}
|
|
a.purger = purge.New(a.srcDB, a.srcCfg.DBName, a.cfg.Table, a.probe.key.Columns,
|
|
a.cfg.Where, a.cfg.BulkDelete, a.cfg.SafeAutoIncrement, autoInc)
|
|
}
|
|
if a.cfg.CheckReplicaLag != "" {
|
|
c, err := lag.Open(a.cfg.CheckReplicaLag, a.cfg.MaxLag, a.cfg.CheckInterval)
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindConnection, err)
|
|
}
|
|
c.OnWait = func(reason string) {
|
|
a.phase("lag-wait", reason)
|
|
}
|
|
a.lagChecker = c
|
|
}
|
|
if a.cfg.HasFile() {
|
|
if err := a.prepareFile(ctx); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if a.alreadyComplete {
|
|
return nil
|
|
}
|
|
a.phase("estimate", "")
|
|
a.estimate = a.queryEstimate(ctx)
|
|
if a.reporter != nil {
|
|
a.reporter.SetEstimate(a.estimate)
|
|
}
|
|
return a.openScanner(ctx)
|
|
}
|
|
|
|
func (a *App) openDest(ctx context.Context) error {
|
|
if a.cfg.DestDSN == "" {
|
|
return nil
|
|
}
|
|
destCfg, err := a.cfg.DestConfig()
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindConfig, err)
|
|
}
|
|
if destCfg.DBName == "" {
|
|
return archererr.Newf(archererr.KindConfig, "--dest must include a database name")
|
|
}
|
|
a.destCfg = destCfg
|
|
a.sameInstance = cli.SameInstance(a.srcCfg, destCfg)
|
|
a.destDB, err = openDB(destCfg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
names := make([]string, len(a.probe.outCols))
|
|
for i, c := range a.probe.outCols {
|
|
names[i] = c.Name
|
|
}
|
|
s, err := sink.NewDB(ctx, a.destDB, destCfg.DBName, a.cfg.DestTable, names, a.cfg.InsertMode, a.probe.outIdx)
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindValidation, err)
|
|
}
|
|
a.destSink = s
|
|
return nil
|
|
}
|
|
|
|
func (a *App) prepareFile(ctx context.Context) error {
|
|
if err := os.MkdirAll(a.cfg.FileDir, 0o755); err != nil {
|
|
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("create output directory: %w", err))
|
|
}
|
|
comp, err := shard.NewCompressor(a.cfg.Compress, a.cfg.CompressLevel)
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindConfig, err)
|
|
}
|
|
man, err := shard.OpenManifest(a.cfg.FileDir)
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindWrite, err)
|
|
}
|
|
a.manifest = man
|
|
maxSeq, err := man.Adopt(a.cfg.FileDir, a.cfg.Format, comp)
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindWrite, err)
|
|
}
|
|
|
|
st, err := checkpoint.Load(a.cfg.FileDir)
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindValidation, err)
|
|
}
|
|
startBucket, startSeq := "", int64(0)
|
|
if st != nil {
|
|
if st.ConfigHash != a.cfgHash {
|
|
if !a.cfg.ForceResume {
|
|
diffs := cli.DiffFields(st.Config, a.cfgFields)
|
|
if len(diffs) == 0 {
|
|
diffs = []string{"fingerprint changed but no field differences were recorded"}
|
|
}
|
|
return archererr.Newf(archererr.KindValidation,
|
|
"checkpoint in %s was created with a different configuration:\n %s\nre-run with --force-resume to ignore this",
|
|
a.cfg.FileDir, strings.Join(diffs, "\n "))
|
|
}
|
|
a.message("warning: --force-resume ignores the changed configuration fingerprint")
|
|
}
|
|
if st.Complete {
|
|
a.alreadyComplete = true
|
|
a.st = st
|
|
return nil
|
|
}
|
|
a.st = st
|
|
a.baseRows = st.RowsDone
|
|
a.resume = &scan.Resume{Phase: st.Cursor.Phase, Key: st.Cursor.Key, DayStart: st.Cursor.DayStart}
|
|
startBucket, startSeq = st.LastBucket, st.LastShardSeq
|
|
} else {
|
|
a.st = &checkpoint.State{RunID: newRunID(), ShardBy: a.cfg.ShardBy}
|
|
}
|
|
if a.reporter != nil {
|
|
a.reporter.SetBase(a.baseRows)
|
|
}
|
|
if maxSeq[startBucket] > startSeq {
|
|
startSeq = maxSeq[startBucket]
|
|
}
|
|
|
|
var keep string
|
|
var resumeMeta *shard.PartialMeta
|
|
if a.cfg.Purge {
|
|
if m, err := shard.LoadPartialMeta(a.cfg.FileDir, startBucket, startSeq+1); err != nil {
|
|
return archererr.Wrap(archererr.KindValidation, err)
|
|
} else if m != nil {
|
|
resumeMeta = m
|
|
keep = shard.PartialPath(a.cfg.FileDir, startBucket, startSeq+1)
|
|
}
|
|
}
|
|
if err := shard.CleanupPartials(a.cfg.FileDir, keep); err != nil {
|
|
return archererr.Wrap(archererr.KindWrite, err)
|
|
}
|
|
|
|
w, err := shard.Open(shard.Options{
|
|
Dir: a.cfg.FileDir,
|
|
Mode: a.cfg.ShardBy,
|
|
ShardRows: int64(a.cfg.ShardRows),
|
|
Format: a.cfg.Format,
|
|
Compressor: comp,
|
|
Columns: a.probe.selectMeta,
|
|
OutIdx: a.probe.outIdx,
|
|
CSVNull: a.cfg.CSVNull,
|
|
StartBucket: startBucket,
|
|
StartSeq: startSeq,
|
|
}, man)
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindWrite, err)
|
|
}
|
|
if resumeMeta != nil {
|
|
if err := w.Resume(resumeMeta); err != nil {
|
|
return archererr.Wrap(archererr.KindValidation, err)
|
|
}
|
|
}
|
|
a.writer = w
|
|
a.fileSink = &sink.File{W: w}
|
|
return nil
|
|
}
|
|
|
|
func (a *App) openScanner(ctx context.Context) error {
|
|
opts := scan.Options{
|
|
DB: a.srcDB,
|
|
DBName: a.srcCfg.DBName,
|
|
Table: a.cfg.Table,
|
|
Where: a.cfg.Where,
|
|
SelectCols: a.probe.selectCols,
|
|
KeyCols: a.probe.key.Columns,
|
|
KeyIdx: a.probe.keyIdx,
|
|
ChunkSize: a.cfg.ChunkSize,
|
|
Shard: a.probe.shard,
|
|
ShardIdx: a.probe.shardIdx,
|
|
ShardLoc: a.loc,
|
|
ZeroDate: a.cfg.ZeroDate,
|
|
Resume: a.resume,
|
|
}
|
|
switch a.probe.path {
|
|
case "A":
|
|
a.scanner = scan.NewDateA(opts)
|
|
case "B":
|
|
s, err := scan.NewDateB(ctx, opts)
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindConnection, err)
|
|
}
|
|
a.scanner = s
|
|
default:
|
|
a.scanner = scan.NewPK(opts)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (a *App) execute(ctx context.Context) error {
|
|
if a.cfg.HasFile() {
|
|
a.message("note: file output is at-least-once; a crash and rerun may duplicate rows in shard files")
|
|
}
|
|
if a.cfg.Purge {
|
|
a.message("warning: --purge deletes archived rows from the source table")
|
|
}
|
|
if a.cfg.BulkDelete {
|
|
a.message("warning: --bulk-delete removes all rows in the scanned key range that match --where, including concurrent inserts")
|
|
}
|
|
if a.cfg.BulkDelete && a.cfg.IsDateShard() {
|
|
a.message("warning: date mode --bulk-delete assumes the shard column never moves backwards relative to the scan key; a late or backfilled row can be deleted before it is archived")
|
|
}
|
|
|
|
for {
|
|
if err := ctx.Err(); err != nil {
|
|
return a.interrupt(err)
|
|
}
|
|
if a.lagChecker != nil {
|
|
if err := a.lagChecker.Wait(ctx); err != nil {
|
|
if ctx.Err() != nil {
|
|
return a.interrupt(err)
|
|
}
|
|
return archererr.Wrap(archererr.KindConnection, err)
|
|
}
|
|
}
|
|
a.phase("scan", "")
|
|
start := time.Now()
|
|
chunk, err := a.scanner.Next(ctx)
|
|
a.stats.ScanDur += time.Since(start)
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return a.interrupt(err)
|
|
}
|
|
return archererr.Wrap(archererr.KindOther, fmt.Errorf("scan: %w", err))
|
|
}
|
|
if err := a.processChunk(ctx, chunk); err != nil {
|
|
if ctx.Err() != nil {
|
|
return a.interrupt(err)
|
|
}
|
|
return err
|
|
}
|
|
a.stats.RowsRead += int64(chunk.Len())
|
|
a.publishProgress(chunk)
|
|
}
|
|
return a.finish()
|
|
}
|
|
|
|
func (a *App) processChunk(ctx context.Context, chunk *row.Chunk) error {
|
|
fileStart := time.Now()
|
|
if a.fileSink != nil {
|
|
a.phase("file", "")
|
|
if err := a.fileSink.WriteChunk(ctx, chunk); err != nil {
|
|
return archererr.Wrap(archererr.KindWrite, err)
|
|
}
|
|
if a.cfg.Purge {
|
|
if err := a.fileSink.Checkpoint(); err != nil {
|
|
return archererr.Wrap(archererr.KindWrite, err)
|
|
}
|
|
}
|
|
}
|
|
a.stats.FileDur += time.Since(fileStart)
|
|
|
|
dbStart := time.Now()
|
|
if a.destSink != nil || a.purger != nil {
|
|
a.phase("db", "")
|
|
}
|
|
if err := a.runDB(ctx, chunk); err != nil {
|
|
return err
|
|
}
|
|
a.stats.DBDur += time.Since(dbStart)
|
|
|
|
if a.fileSink != nil && !a.cfg.Purge {
|
|
if err := a.fileSink.Flush(ctx); err != nil {
|
|
return archererr.Wrap(archererr.KindWrite, err)
|
|
}
|
|
}
|
|
|
|
a.stats.RowsCommitted += int64(chunk.Len())
|
|
if a.cfg.HasFile() {
|
|
a.phase("checkpoint", "")
|
|
}
|
|
cpStart := time.Now()
|
|
if err := a.saveState(false); err != nil {
|
|
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("save checkpoint: %w", err))
|
|
}
|
|
a.stats.CheckpointDur += time.Since(cpStart)
|
|
return nil
|
|
}
|
|
|
|
func (a *App) runDB(ctx context.Context, chunk *row.Chunk) error {
|
|
attempts := a.cfg.Retries + 1
|
|
if a.destSink != nil && a.purger != nil && a.sameInstance {
|
|
err := retry(ctx, attempts, func() error {
|
|
tx, err := a.destDB.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n, err := a.destSink.InsertChunk(ctx, tx, chunk)
|
|
if err != nil {
|
|
_ = tx.Rollback()
|
|
return err
|
|
}
|
|
d, err := a.purger.DeleteChunk(ctx, tx, chunk)
|
|
if err != nil {
|
|
_ = tx.Rollback()
|
|
return err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return err
|
|
}
|
|
a.stats.RowsInserted += n
|
|
a.stats.RowsDeleted += d
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindWrite, err)
|
|
}
|
|
return nil
|
|
}
|
|
if a.destSink != nil {
|
|
err := retry(ctx, attempts, func() error {
|
|
tx, err := a.destDB.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n, err := a.destSink.InsertChunk(ctx, tx, chunk)
|
|
if err != nil {
|
|
_ = tx.Rollback()
|
|
return err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return err
|
|
}
|
|
a.stats.RowsInserted += n
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindWrite, err)
|
|
}
|
|
}
|
|
if a.purger != nil {
|
|
err := retry(ctx, attempts, func() error {
|
|
tx, err := a.srcDB.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
d, err := a.purger.DeleteChunk(ctx, tx, chunk)
|
|
if err != nil {
|
|
_ = tx.Rollback()
|
|
return err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return err
|
|
}
|
|
a.stats.RowsDeleted += d
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindWrite, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func retry(ctx context.Context, attempts int, fn func() error) error {
|
|
var err error
|
|
for i := 0; i < attempts; i++ {
|
|
if i > 0 {
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-time.After(time.Duration(i) * time.Second):
|
|
}
|
|
}
|
|
if err = fn(); err == nil {
|
|
return nil
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (a *App) saveState(complete bool) error {
|
|
if a.cfg.FileDir == "" || a.st == nil || a.alreadyComplete {
|
|
return nil
|
|
}
|
|
st := a.st
|
|
st.ConfigHash = a.cfgHash
|
|
st.Config = a.cfgFields
|
|
st.ShardBy = a.cfg.ShardBy
|
|
st.RowsDone = a.baseRows + a.stats.RowsCommitted
|
|
if a.scanner != nil {
|
|
cs := a.scanner.ResumeState()
|
|
st.Cursor = checkpoint.Cursor{Phase: cs.Phase, Key: cs.Key, DayStart: cs.DayStart}
|
|
}
|
|
if a.writer != nil {
|
|
st.LastBucket, st.LastShardSeq = a.writer.StatePosition()
|
|
}
|
|
st.Complete = complete
|
|
return checkpoint.Save(a.cfg.FileDir, st)
|
|
}
|
|
|
|
func (a *App) finish() error {
|
|
a.phase("finalize", "")
|
|
if a.writer != nil {
|
|
if err := a.writer.Close(); err != nil {
|
|
return archererr.Wrap(archererr.KindWrite, err)
|
|
}
|
|
var shards, bytes int64
|
|
_, shards, bytes = a.writer.Stats()
|
|
a.stats.Shards, a.stats.Bytes = shards, bytes
|
|
}
|
|
if err := a.saveState(true); err != nil {
|
|
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("save checkpoint: %w", err))
|
|
}
|
|
a.printStats()
|
|
if a.reporter != nil {
|
|
if a.writer != nil {
|
|
a.reporter.SetShards(a.stats.Shards)
|
|
a.reporter.SetBytes(a.stats.Bytes)
|
|
}
|
|
a.reporter.Done()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (a *App) interrupt(cause error) error {
|
|
if a.reporter != nil {
|
|
a.reporter.Interrupted()
|
|
}
|
|
if a.writer != nil {
|
|
if err := a.writer.SealCurrent(); err != nil {
|
|
a.message("warning: could not seal current shard: %v", err)
|
|
}
|
|
}
|
|
if err := a.saveState(false); err != nil {
|
|
a.message("warning: could not save checkpoint: %v", err)
|
|
}
|
|
return archererr.Wrap(archererr.KindInterrupted, cause)
|
|
}
|
|
|
|
func (a *App) publishProgress(chunk *row.Chunk) {
|
|
if a.reporter == nil {
|
|
return
|
|
}
|
|
if a.estimate != nil {
|
|
switch a.estimate.Kind {
|
|
case "day":
|
|
if len(chunk.Buckets) > 0 {
|
|
b := chunk.Buckets[len(chunk.Buckets)-1]
|
|
if b != "" && b != "_null" && !strings.HasPrefix(b, "0000-") {
|
|
if t, err := time.ParseInLocation("2006-01-02", b, a.loc); err == nil {
|
|
a.reporter.SetPosition(float64(t.Unix()))
|
|
}
|
|
}
|
|
}
|
|
case "key":
|
|
if len(chunk.LastKey) == 1 && !chunk.LastKey[0].Null {
|
|
if n, err := strconv.ParseInt(chunk.LastKey[0].Val, 10, 64); err == nil {
|
|
a.reporter.SetPosition(float64(n))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if a.writer != nil {
|
|
a.reporter.SetShard(a.writer.CurrentShard())
|
|
a.reporter.SetBytes(a.writer.WrittenBytes())
|
|
}
|
|
a.reporter.AddRows(int64(chunk.Len()))
|
|
}
|
|
|
|
func (a *App) printStats() {
|
|
if !a.cfg.Statistics {
|
|
return
|
|
}
|
|
fmt.Fprintln(os.Stdout, "this run:")
|
|
fmt.Fprintf(os.Stdout, " rows read: %d\n", a.stats.RowsRead)
|
|
fmt.Fprintf(os.Stdout, " rows committed: %d\n", a.stats.RowsCommitted)
|
|
fmt.Fprintf(os.Stdout, " rows inserted: %d\n", a.stats.RowsInserted)
|
|
fmt.Fprintf(os.Stdout, " rows deleted: %d\n", a.stats.RowsDeleted)
|
|
if a.writer != nil {
|
|
fmt.Fprintf(os.Stdout, " shards: %d\n", a.stats.Shards)
|
|
fmt.Fprintf(os.Stdout, " bytes written: %d\n", a.stats.Bytes)
|
|
}
|
|
fmt.Fprintf(os.Stdout, " elapsed: %s (scan %s, file %s, db %s, checkpoint %s)\n",
|
|
time.Since(a.stats.Started).Round(time.Millisecond),
|
|
a.stats.ScanDur.Round(time.Millisecond), a.stats.FileDur.Round(time.Millisecond),
|
|
a.stats.DBDur.Round(time.Millisecond), a.stats.CheckpointDur.Round(time.Millisecond))
|
|
|
|
fmt.Fprintln(os.Stdout, "cumulative (including prior runs):")
|
|
fmt.Fprintf(os.Stdout, " rows read: %d\n", a.baseRows+a.stats.RowsRead)
|
|
fmt.Fprintf(os.Stdout, " rows committed: %d\n", a.baseRows+a.stats.RowsCommitted)
|
|
if a.writer != nil && a.manifest != nil {
|
|
var rows, bytes int64
|
|
entries := a.manifest.Entries()
|
|
for _, e := range entries {
|
|
rows += e.Rows
|
|
bytes += e.Bytes
|
|
}
|
|
fmt.Fprintf(os.Stdout, " shards: %d\n", len(entries))
|
|
fmt.Fprintf(os.Stdout, " rows written: %d\n", rows)
|
|
fmt.Fprintf(os.Stdout, " bytes written: %d\n", bytes)
|
|
}
|
|
}
|
|
|
|
func (a *App) close() {
|
|
if a.scanner != nil {
|
|
_ = a.scanner.Close()
|
|
}
|
|
if a.writer != nil {
|
|
_ = a.writer.Abandon()
|
|
} else if a.manifest != nil {
|
|
_ = a.manifest.Close()
|
|
}
|
|
if a.lagChecker != nil {
|
|
_ = a.lagChecker.Close()
|
|
}
|
|
if a.destDB != nil {
|
|
_ = a.destDB.Close()
|
|
}
|
|
if a.srcDB != nil {
|
|
_ = a.srcDB.Close()
|
|
}
|
|
}
|
|
|
|
func openDB(cfg *mysql.Config) (*sql.DB, error) {
|
|
db, err := sql.Open("mysql", cfg.FormatDSN())
|
|
if err != nil {
|
|
return nil, archererr.Wrap(archererr.KindConnection, err)
|
|
}
|
|
db.SetMaxOpenConns(8)
|
|
db.SetMaxIdleConns(4)
|
|
db.SetConnMaxLifetime(time.Hour)
|
|
if err := db.Ping(); err != nil {
|
|
db.Close()
|
|
return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("connect %s: %w", cfg.Addr, err))
|
|
}
|
|
return db, nil
|
|
}
|
|
|
|
func newRunID() string {
|
|
b := make([]byte, 8)
|
|
if _, err := rand.Read(b); err != nil {
|
|
return time.Now().UTC().Format("20060102T150405Z")
|
|
}
|
|
return time.Now().UTC().Format("20060102T150405Z") + "-" + hex.EncodeToString(b)
|
|
}
|