Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,705 @@
|
||||
// Package app wires the CLI, schema probe, scanners, shard writer and
|
||||
// database targets into one run.
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/checkpoint"
|
||||
"archer/internal/cli"
|
||||
"archer/internal/lag"
|
||||
"archer/internal/progress"
|
||||
"archer/internal/purge"
|
||||
"archer/internal/row"
|
||||
"archer/internal/scan"
|
||||
"archer/internal/shard"
|
||||
"archer/internal/sink"
|
||||
)
|
||||
|
||||
// Stats collects counters and phase timings for --statistics.
|
||||
type Stats struct {
|
||||
Started time.Time
|
||||
RowsRead int64
|
||||
RowsCommitted int64
|
||||
RowsInserted int64
|
||||
RowsDeleted int64
|
||||
ScanDur time.Duration
|
||||
FileDur time.Duration
|
||||
DBDur time.Duration
|
||||
CheckpointDur time.Duration
|
||||
Shards int64
|
||||
Bytes int64
|
||||
}
|
||||
|
||||
// App holds a run's state.
|
||||
type App struct {
|
||||
cfg *cli.Config
|
||||
srcDB *sql.DB
|
||||
destDB *sql.DB
|
||||
srcCfg *mysql.Config
|
||||
destCfg *mysql.Config
|
||||
|
||||
loc *time.Location
|
||||
probe *probeResult
|
||||
cfgHash string
|
||||
cfgFields map[string]string
|
||||
|
||||
scanner scan.Scanner
|
||||
fileSink *sink.File
|
||||
writer *shard.Writer
|
||||
manifest *shard.Manifest
|
||||
destSink *sink.DB
|
||||
purger *purge.Purger
|
||||
lagChecker *lag.Checker
|
||||
|
||||
sameInstance bool
|
||||
resume *scan.Resume
|
||||
alreadyComplete bool
|
||||
|
||||
st *checkpoint.State
|
||||
stats Stats
|
||||
|
||||
reporter *progress.Reporter
|
||||
estimate *progress.Estimate
|
||||
baseRows int64
|
||||
}
|
||||
|
||||
// Run executes one archiving run.
|
||||
func Run(ctx context.Context, cfg *cli.Config) error {
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
a := &App{cfg: cfg, stats: Stats{Started: time.Now()}}
|
||||
defer a.close()
|
||||
if !cfg.DryRun {
|
||||
a.startReporter()
|
||||
defer a.reporter.Stop()
|
||||
}
|
||||
|
||||
if err := a.setupSource(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.DryRun {
|
||||
return a.printDryRun(os.Stdout)
|
||||
}
|
||||
if err := a.prepare(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if a.alreadyComplete {
|
||||
fmt.Fprintf(os.Stdout, "archer: output in %s is already complete; nothing to do\n", cfg.FileDir)
|
||||
return nil
|
||||
}
|
||||
return a.execute(ctx)
|
||||
}
|
||||
|
||||
func (a *App) startReporter() {
|
||||
tty := progress.IsTerminal(os.Stderr)
|
||||
enabled := a.cfg.Progress.Enabled(tty)
|
||||
a.reporter = progress.New(progress.Options{
|
||||
Out: os.Stderr,
|
||||
Enabled: enabled,
|
||||
Live: enabled && tty,
|
||||
Interval: a.cfg.ProgressInterval,
|
||||
RowEvery: a.cfg.Progress.Rows,
|
||||
})
|
||||
a.reporter.Start()
|
||||
}
|
||||
|
||||
func (a *App) phase(name, reason string) {
|
||||
if a.reporter != nil {
|
||||
a.reporter.Phase(name, reason)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) message(format string, args ...any) {
|
||||
if a.reporter != nil {
|
||||
a.reporter.Message(format, args...)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
||||
}
|
||||
|
||||
func (a *App) setupSource(ctx context.Context) error {
|
||||
a.phase("probe", "")
|
||||
loc, err := time.LoadLocation(a.cfg.ShardTimeZone)
|
||||
if err != nil {
|
||||
return archererr.Newf(archererr.KindConfig, "invalid --shard-time-zone: %v", err)
|
||||
}
|
||||
a.loc = loc
|
||||
srcCfg, err := a.cfg.SourceConfig()
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConfig, err)
|
||||
}
|
||||
if srcCfg.DBName == "" {
|
||||
return archererr.Newf(archererr.KindConfig, "--source must include a database name")
|
||||
}
|
||||
a.srcCfg = srcCfg
|
||||
a.srcDB, err = openDB(srcCfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.probe, err = probeTable(ctx, a.srcDB, a.cfg, srcCfg.DBName, loc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.cfgHash, a.cfgFields, err = a.cfg.Fingerprint()
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConfig, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) prepare(ctx context.Context) error {
|
||||
a.phase("resume", "")
|
||||
if err := a.openDest(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if a.cfg.Purge {
|
||||
autoInc := false
|
||||
if len(a.probe.key.Columns) == 1 {
|
||||
if c, ok := a.probe.table.Column(a.probe.key.Columns[0]); ok {
|
||||
autoInc = c.IsAutoIncrement()
|
||||
}
|
||||
}
|
||||
a.purger = purge.New(a.srcDB, a.srcCfg.DBName, a.cfg.Table, a.probe.key.Columns,
|
||||
a.cfg.Where, a.cfg.BulkDelete, a.cfg.SafeAutoIncrement, autoInc)
|
||||
}
|
||||
if a.cfg.CheckReplicaLag != "" {
|
||||
c, err := lag.Open(a.cfg.CheckReplicaLag, a.cfg.MaxLag, a.cfg.CheckInterval)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConnection, err)
|
||||
}
|
||||
c.OnWait = func(reason string) {
|
||||
a.phase("lag-wait", reason)
|
||||
}
|
||||
a.lagChecker = c
|
||||
}
|
||||
if a.cfg.HasFile() {
|
||||
if err := a.prepareFile(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if a.alreadyComplete {
|
||||
return nil
|
||||
}
|
||||
a.phase("estimate", "")
|
||||
a.estimate = a.queryEstimate(ctx)
|
||||
if a.reporter != nil {
|
||||
a.reporter.SetEstimate(a.estimate)
|
||||
}
|
||||
return a.openScanner(ctx)
|
||||
}
|
||||
|
||||
func (a *App) openDest(ctx context.Context) error {
|
||||
if a.cfg.DestDSN == "" {
|
||||
return nil
|
||||
}
|
||||
destCfg, err := a.cfg.DestConfig()
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConfig, err)
|
||||
}
|
||||
if destCfg.DBName == "" {
|
||||
return archererr.Newf(archererr.KindConfig, "--dest must include a database name")
|
||||
}
|
||||
a.destCfg = destCfg
|
||||
a.sameInstance = cli.SameInstance(a.srcCfg, destCfg)
|
||||
a.destDB, err = openDB(destCfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names := make([]string, len(a.probe.outCols))
|
||||
for i, c := range a.probe.outCols {
|
||||
names[i] = c.Name
|
||||
}
|
||||
s, err := sink.NewDB(ctx, a.destDB, destCfg.DBName, a.cfg.DestTable, names, a.cfg.InsertMode, a.probe.outIdx)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
a.destSink = s
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) prepareFile(ctx context.Context) error {
|
||||
if err := os.MkdirAll(a.cfg.FileDir, 0o755); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("create output directory: %w", err))
|
||||
}
|
||||
comp, err := shard.NewCompressor(a.cfg.Compress, a.cfg.CompressLevel)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConfig, err)
|
||||
}
|
||||
man, err := shard.OpenManifest(a.cfg.FileDir)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
a.manifest = man
|
||||
maxSeq, err := man.Adopt(a.cfg.FileDir, a.cfg.Format, comp)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
|
||||
st, err := checkpoint.Load(a.cfg.FileDir)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
startBucket, startSeq := "", int64(0)
|
||||
if st != nil {
|
||||
if st.ConfigHash != a.cfgHash {
|
||||
if !a.cfg.ForceResume {
|
||||
diffs := cli.DiffFields(st.Config, a.cfgFields)
|
||||
if len(diffs) == 0 {
|
||||
diffs = []string{"fingerprint changed but no field differences were recorded"}
|
||||
}
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"checkpoint in %s was created with a different configuration:\n %s\nre-run with --force-resume to ignore this",
|
||||
a.cfg.FileDir, strings.Join(diffs, "\n "))
|
||||
}
|
||||
a.message("warning: --force-resume ignores the changed configuration fingerprint")
|
||||
}
|
||||
if st.Complete {
|
||||
a.alreadyComplete = true
|
||||
a.st = st
|
||||
return nil
|
||||
}
|
||||
a.st = st
|
||||
a.baseRows = st.RowsDone
|
||||
a.resume = &scan.Resume{Phase: st.Cursor.Phase, Key: st.Cursor.Key, DayStart: st.Cursor.DayStart}
|
||||
startBucket, startSeq = st.LastBucket, st.LastShardSeq
|
||||
} else {
|
||||
a.st = &checkpoint.State{RunID: newRunID(), ShardBy: a.cfg.ShardBy}
|
||||
}
|
||||
if a.reporter != nil {
|
||||
a.reporter.SetBase(a.baseRows)
|
||||
}
|
||||
if maxSeq[startBucket] > startSeq {
|
||||
startSeq = maxSeq[startBucket]
|
||||
}
|
||||
|
||||
var keep string
|
||||
var resumeMeta *shard.PartialMeta
|
||||
if a.cfg.Purge {
|
||||
if m, err := shard.LoadPartialMeta(a.cfg.FileDir, startBucket, startSeq+1); err != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, err)
|
||||
} else if m != nil {
|
||||
resumeMeta = m
|
||||
keep = shard.PartialPath(a.cfg.FileDir, startBucket, startSeq+1)
|
||||
}
|
||||
}
|
||||
if err := shard.CleanupPartials(a.cfg.FileDir, keep); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
|
||||
w, err := shard.Open(shard.Options{
|
||||
Dir: a.cfg.FileDir,
|
||||
Mode: a.cfg.ShardBy,
|
||||
ShardRows: int64(a.cfg.ShardRows),
|
||||
Format: a.cfg.Format,
|
||||
Compressor: comp,
|
||||
Columns: a.probe.selectMeta,
|
||||
OutIdx: a.probe.outIdx,
|
||||
CSVNull: a.cfg.CSVNull,
|
||||
StartBucket: startBucket,
|
||||
StartSeq: startSeq,
|
||||
}, man)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
if resumeMeta != nil {
|
||||
if err := w.Resume(resumeMeta); err != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
}
|
||||
a.writer = w
|
||||
a.fileSink = &sink.File{W: w}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) openScanner(ctx context.Context) error {
|
||||
opts := scan.Options{
|
||||
DB: a.srcDB,
|
||||
DBName: a.srcCfg.DBName,
|
||||
Table: a.cfg.Table,
|
||||
Where: a.cfg.Where,
|
||||
SelectCols: a.probe.selectCols,
|
||||
KeyCols: a.probe.key.Columns,
|
||||
KeyIdx: a.probe.keyIdx,
|
||||
ChunkSize: a.cfg.ChunkSize,
|
||||
Shard: a.probe.shard,
|
||||
ShardIdx: a.probe.shardIdx,
|
||||
ShardLoc: a.loc,
|
||||
ZeroDate: a.cfg.ZeroDate,
|
||||
Resume: a.resume,
|
||||
}
|
||||
switch a.probe.path {
|
||||
case "A":
|
||||
a.scanner = scan.NewDateA(opts)
|
||||
case "B":
|
||||
s, err := scan.NewDateB(ctx, opts)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConnection, err)
|
||||
}
|
||||
a.scanner = s
|
||||
default:
|
||||
a.scanner = scan.NewPK(opts)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) execute(ctx context.Context) error {
|
||||
if a.cfg.HasFile() {
|
||||
a.message("note: file output is at-least-once; a crash and rerun may duplicate rows in shard files")
|
||||
}
|
||||
if a.cfg.Purge {
|
||||
a.message("warning: --purge deletes archived rows from the source table")
|
||||
}
|
||||
if a.cfg.BulkDelete {
|
||||
a.message("warning: --bulk-delete removes all rows in the scanned key range that match --where, including concurrent inserts")
|
||||
}
|
||||
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return a.interrupt(err)
|
||||
}
|
||||
if a.lagChecker != nil {
|
||||
if err := a.lagChecker.Wait(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return a.interrupt(err)
|
||||
}
|
||||
return archererr.Wrap(archererr.KindConnection, err)
|
||||
}
|
||||
}
|
||||
a.phase("scan", "")
|
||||
start := time.Now()
|
||||
chunk, err := a.scanner.Next(ctx)
|
||||
a.stats.ScanDur += time.Since(start)
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return a.interrupt(err)
|
||||
}
|
||||
return archererr.Wrap(archererr.KindOther, fmt.Errorf("scan: %w", err))
|
||||
}
|
||||
if err := a.processChunk(ctx, chunk); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return a.interrupt(err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
a.stats.RowsRead += int64(chunk.Len())
|
||||
a.publishProgress(chunk)
|
||||
}
|
||||
return a.finish()
|
||||
}
|
||||
|
||||
func (a *App) processChunk(ctx context.Context, chunk *row.Chunk) error {
|
||||
fileStart := time.Now()
|
||||
if a.fileSink != nil {
|
||||
a.phase("file", "")
|
||||
if err := a.fileSink.WriteChunk(ctx, chunk); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
if a.cfg.Purge {
|
||||
if err := a.fileSink.Checkpoint(); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
a.stats.FileDur += time.Since(fileStart)
|
||||
|
||||
dbStart := time.Now()
|
||||
if a.destSink != nil || a.purger != nil {
|
||||
a.phase("db", "")
|
||||
}
|
||||
if err := a.runDB(ctx, chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
a.stats.DBDur += time.Since(dbStart)
|
||||
|
||||
if a.fileSink != nil && !a.cfg.Purge {
|
||||
if err := a.fileSink.Flush(ctx); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
}
|
||||
|
||||
a.stats.RowsCommitted += int64(chunk.Len())
|
||||
if a.cfg.HasFile() {
|
||||
a.phase("checkpoint", "")
|
||||
}
|
||||
cpStart := time.Now()
|
||||
if err := a.saveState(false); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("save checkpoint: %w", err))
|
||||
}
|
||||
a.stats.CheckpointDur += time.Since(cpStart)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) runDB(ctx context.Context, chunk *row.Chunk) error {
|
||||
attempts := a.cfg.Retries + 1
|
||||
if a.destSink != nil && a.purger != nil && a.sameInstance {
|
||||
err := retry(ctx, attempts, func() error {
|
||||
tx, err := a.destDB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := a.destSink.InsertChunk(ctx, tx, chunk)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
d, err := a.purger.DeleteChunk(ctx, tx, chunk)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
a.stats.RowsInserted += n
|
||||
a.stats.RowsDeleted += d
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if a.destSink != nil {
|
||||
err := retry(ctx, attempts, func() error {
|
||||
tx, err := a.destDB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := a.destSink.InsertChunk(ctx, tx, chunk)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
a.stats.RowsInserted += n
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
}
|
||||
if a.purger != nil {
|
||||
err := retry(ctx, attempts, func() error {
|
||||
tx, err := a.srcDB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d, err := a.purger.DeleteChunk(ctx, tx, chunk)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
a.stats.RowsDeleted += d
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func retry(ctx context.Context, attempts int, fn func() error) error {
|
||||
var err error
|
||||
for i := 0; i < attempts; i++ {
|
||||
if i > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(time.Duration(i) * time.Second):
|
||||
}
|
||||
}
|
||||
if err = fn(); err == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *App) saveState(complete bool) error {
|
||||
if a.cfg.FileDir == "" || a.st == nil || a.alreadyComplete {
|
||||
return nil
|
||||
}
|
||||
st := a.st
|
||||
st.ConfigHash = a.cfgHash
|
||||
st.Config = a.cfgFields
|
||||
st.ShardBy = a.cfg.ShardBy
|
||||
st.RowsDone = a.baseRows + a.stats.RowsCommitted
|
||||
if a.scanner != nil {
|
||||
cs := a.scanner.ResumeState()
|
||||
st.Cursor = checkpoint.Cursor{Phase: cs.Phase, Key: cs.Key, DayStart: cs.DayStart}
|
||||
}
|
||||
if a.writer != nil {
|
||||
st.LastBucket, st.LastShardSeq = a.writer.StatePosition()
|
||||
}
|
||||
st.Complete = complete
|
||||
return checkpoint.Save(a.cfg.FileDir, st)
|
||||
}
|
||||
|
||||
func (a *App) finish() error {
|
||||
a.phase("finalize", "")
|
||||
if a.writer != nil {
|
||||
if err := a.writer.Close(); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, err)
|
||||
}
|
||||
var shards, bytes int64
|
||||
_, shards, bytes = a.writer.Stats()
|
||||
a.stats.Shards, a.stats.Bytes = shards, bytes
|
||||
}
|
||||
if err := a.saveState(true); err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("save checkpoint: %w", err))
|
||||
}
|
||||
a.printStats()
|
||||
if a.reporter != nil {
|
||||
if a.writer != nil {
|
||||
a.reporter.SetShards(a.stats.Shards)
|
||||
a.reporter.SetBytes(a.stats.Bytes)
|
||||
}
|
||||
a.reporter.Done()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) interrupt(cause error) error {
|
||||
if a.reporter != nil {
|
||||
a.reporter.Interrupted()
|
||||
}
|
||||
if a.writer != nil {
|
||||
if err := a.writer.SealCurrent(); err != nil {
|
||||
a.message("warning: could not seal current shard: %v", err)
|
||||
}
|
||||
}
|
||||
if err := a.saveState(false); err != nil {
|
||||
a.message("warning: could not save checkpoint: %v", err)
|
||||
}
|
||||
return archererr.Wrap(archererr.KindInterrupted, cause)
|
||||
}
|
||||
|
||||
func (a *App) publishProgress(chunk *row.Chunk) {
|
||||
if a.reporter == nil {
|
||||
return
|
||||
}
|
||||
if a.estimate != nil {
|
||||
switch a.estimate.Kind {
|
||||
case "day":
|
||||
if len(chunk.Buckets) > 0 {
|
||||
b := chunk.Buckets[len(chunk.Buckets)-1]
|
||||
if b != "" && b != "_null" && !strings.HasPrefix(b, "0000-") {
|
||||
if t, err := time.ParseInLocation("2006-01-02", b, a.loc); err == nil {
|
||||
a.reporter.SetPosition(float64(t.Unix()))
|
||||
}
|
||||
}
|
||||
}
|
||||
case "key":
|
||||
if len(chunk.LastKey) == 1 && !chunk.LastKey[0].Null {
|
||||
if n, err := strconv.ParseInt(chunk.LastKey[0].Val, 10, 64); err == nil {
|
||||
a.reporter.SetPosition(float64(n))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if a.writer != nil {
|
||||
a.reporter.SetShard(a.writer.CurrentShard())
|
||||
a.reporter.SetBytes(a.writer.WrittenBytes())
|
||||
}
|
||||
a.reporter.AddRows(int64(chunk.Len()))
|
||||
}
|
||||
|
||||
func (a *App) printStats() {
|
||||
if !a.cfg.Statistics {
|
||||
return
|
||||
}
|
||||
fmt.Fprintln(os.Stdout, "this run:")
|
||||
fmt.Fprintf(os.Stdout, " rows read: %d\n", a.stats.RowsRead)
|
||||
fmt.Fprintf(os.Stdout, " rows committed: %d\n", a.stats.RowsCommitted)
|
||||
fmt.Fprintf(os.Stdout, " rows inserted: %d\n", a.stats.RowsInserted)
|
||||
fmt.Fprintf(os.Stdout, " rows deleted: %d\n", a.stats.RowsDeleted)
|
||||
if a.writer != nil {
|
||||
fmt.Fprintf(os.Stdout, " shards: %d\n", a.stats.Shards)
|
||||
fmt.Fprintf(os.Stdout, " bytes written: %d\n", a.stats.Bytes)
|
||||
}
|
||||
fmt.Fprintf(os.Stdout, " elapsed: %s (scan %s, file %s, db %s, checkpoint %s)\n",
|
||||
time.Since(a.stats.Started).Round(time.Millisecond),
|
||||
a.stats.ScanDur.Round(time.Millisecond), a.stats.FileDur.Round(time.Millisecond),
|
||||
a.stats.DBDur.Round(time.Millisecond), a.stats.CheckpointDur.Round(time.Millisecond))
|
||||
|
||||
fmt.Fprintln(os.Stdout, "cumulative (including prior runs):")
|
||||
fmt.Fprintf(os.Stdout, " rows read: %d\n", a.baseRows+a.stats.RowsRead)
|
||||
fmt.Fprintf(os.Stdout, " rows committed: %d\n", a.baseRows+a.stats.RowsCommitted)
|
||||
if a.writer != nil && a.manifest != nil {
|
||||
var rows, bytes int64
|
||||
entries := a.manifest.Entries()
|
||||
for _, e := range entries {
|
||||
rows += e.Rows
|
||||
bytes += e.Bytes
|
||||
}
|
||||
fmt.Fprintf(os.Stdout, " shards: %d\n", len(entries))
|
||||
fmt.Fprintf(os.Stdout, " rows written: %d\n", rows)
|
||||
fmt.Fprintf(os.Stdout, " bytes written: %d\n", bytes)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) close() {
|
||||
if a.scanner != nil {
|
||||
_ = a.scanner.Close()
|
||||
}
|
||||
if a.writer != nil {
|
||||
_ = a.writer.Abandon()
|
||||
} else if a.manifest != nil {
|
||||
_ = a.manifest.Close()
|
||||
}
|
||||
if a.lagChecker != nil {
|
||||
_ = a.lagChecker.Close()
|
||||
}
|
||||
if a.destDB != nil {
|
||||
_ = a.destDB.Close()
|
||||
}
|
||||
if a.srcDB != nil {
|
||||
_ = a.srcDB.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func openDB(cfg *mysql.Config) (*sql.DB, error) {
|
||||
db, err := sql.Open("mysql", cfg.FormatDSN())
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindConnection, err)
|
||||
}
|
||||
db.SetMaxOpenConns(8)
|
||||
db.SetMaxIdleConns(4)
|
||||
db.SetConnMaxLifetime(time.Hour)
|
||||
if err := db.Ping(); err != nil {
|
||||
db.Close()
|
||||
return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("connect %s: %w", cfg.Addr, err))
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func newRunID() string {
|
||||
b := make([]byte, 8)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return time.Now().UTC().Format("20060102T150405Z")
|
||||
}
|
||||
return time.Now().UTC().Format("20060102T150405Z") + "-" + hex.EncodeToString(b)
|
||||
}
|
||||
Reference in new issue
Block a user