// Package app wires the CLI, schema probe, scanners, shard writer and // database targets into one run. package app import ( "context" "crypto/rand" "database/sql" "encoding/hex" "fmt" "io" "os" "strconv" "strings" "time" "github.com/go-sql-driver/mysql" "archer/internal/archererr" "archer/internal/checkpoint" "archer/internal/cli" "archer/internal/lag" "archer/internal/progress" "archer/internal/purge" "archer/internal/row" "archer/internal/scan" "archer/internal/shard" "archer/internal/sink" ) // Stats collects counters and phase timings for --statistics. type Stats struct { Started time.Time RowsRead int64 RowsCommitted int64 RowsInserted int64 RowsDeleted int64 ScanDur time.Duration FileDur time.Duration DBDur time.Duration CheckpointDur time.Duration Shards int64 Bytes int64 } // App holds a run's state. type App struct { cfg *cli.Config srcDB *sql.DB destDB *sql.DB srcCfg *mysql.Config destCfg *mysql.Config loc *time.Location probe *probeResult cfgHash string cfgFields map[string]string scanner scan.Scanner fileSink *sink.File writer *shard.Writer manifest *shard.Manifest destSink *sink.DB purger *purge.Purger lagChecker *lag.Checker sameInstance bool resume *scan.Resume alreadyComplete bool st *checkpoint.State stats Stats reporter *progress.Reporter estimate *progress.Estimate baseRows int64 } // Run executes one archiving run. func Run(ctx context.Context, cfg *cli.Config) error { if err := cfg.Validate(); err != nil { return err } a := &App{cfg: cfg, stats: Stats{Started: time.Now()}} defer a.close() if !cfg.DryRun { a.startReporter() defer a.reporter.Stop() } if err := a.setupSource(ctx); err != nil { return err } if cfg.DryRun { return a.printDryRun(os.Stdout) } if err := a.prepare(ctx); err != nil { return err } if a.alreadyComplete { fmt.Fprintf(os.Stdout, "archer: output in %s is already complete; nothing to do\n", cfg.FileDir) return nil } return a.execute(ctx) } func (a *App) startReporter() { tty := progress.IsTerminal(os.Stderr) enabled := a.cfg.Progress.Enabled(tty) a.reporter = progress.New(progress.Options{ Out: os.Stderr, Enabled: enabled, Live: enabled && tty, Interval: a.cfg.ProgressInterval, RowEvery: a.cfg.Progress.Rows, }) a.reporter.Start() } func (a *App) phase(name, reason string) { if a.reporter != nil { a.reporter.Phase(name, reason) } } func (a *App) message(format string, args ...any) { if a.reporter != nil { a.reporter.Message(format, args...) return } fmt.Fprintf(os.Stderr, format+"\n", args...) } func (a *App) setupSource(ctx context.Context) error { a.phase("probe", "") loc, err := time.LoadLocation(a.cfg.ShardTimeZone) if err != nil { return archererr.Newf(archererr.KindConfig, "invalid --shard-time-zone: %v", err) } a.loc = loc srcCfg, err := a.cfg.SourceConfig() if err != nil { return archererr.Wrap(archererr.KindConfig, err) } if srcCfg.DBName == "" { return archererr.Newf(archererr.KindConfig, "--source must include a database name") } a.srcCfg = srcCfg a.srcDB, err = openDB(srcCfg) if err != nil { return err } a.probe, err = probeTable(ctx, a.srcDB, a.cfg, srcCfg.DBName, loc) if err != nil { return err } a.cfgHash, a.cfgFields, err = a.cfg.Fingerprint() if err != nil { return archererr.Wrap(archererr.KindConfig, err) } return nil } func (a *App) prepare(ctx context.Context) error { a.phase("resume", "") if err := a.openDest(ctx); err != nil { return err } if a.cfg.Purge { autoInc := false if len(a.probe.key.Columns) == 1 { if c, ok := a.probe.table.Column(a.probe.key.Columns[0]); ok { autoInc = c.IsAutoIncrement() } } a.purger = purge.New(a.srcDB, a.srcCfg.DBName, a.cfg.Table, a.probe.key.Columns, a.cfg.Where, a.cfg.BulkDelete, a.cfg.SafeAutoIncrement, autoInc) } if a.cfg.CheckReplicaLag != "" { c, err := lag.Open(a.cfg.CheckReplicaLag, a.cfg.MaxLag, a.cfg.CheckInterval) if err != nil { return archererr.Wrap(archererr.KindConnection, err) } c.OnWait = func(reason string) { a.phase("lag-wait", reason) } a.lagChecker = c } if a.cfg.HasFile() { if err := a.prepareFile(ctx); err != nil { return err } } if a.alreadyComplete { return nil } a.phase("estimate", "") a.estimate = a.queryEstimate(ctx) if a.reporter != nil { a.reporter.SetEstimate(a.estimate) } return a.openScanner(ctx) } func (a *App) openDest(ctx context.Context) error { if a.cfg.DestDSN == "" { return nil } destCfg, err := a.cfg.DestConfig() if err != nil { return archererr.Wrap(archererr.KindConfig, err) } if destCfg.DBName == "" { return archererr.Newf(archererr.KindConfig, "--dest must include a database name") } a.destCfg = destCfg a.sameInstance = cli.SameInstance(a.srcCfg, destCfg) a.destDB, err = openDB(destCfg) if err != nil { return err } names := make([]string, len(a.probe.outCols)) for i, c := range a.probe.outCols { names[i] = c.Name } s, err := sink.NewDB(ctx, a.destDB, destCfg.DBName, a.cfg.DestTable, names, a.cfg.InsertMode, a.probe.outIdx) if err != nil { return archererr.Wrap(archererr.KindValidation, err) } a.destSink = s return nil } func (a *App) prepareFile(ctx context.Context) error { if err := os.MkdirAll(a.cfg.FileDir, 0o755); err != nil { return archererr.Wrap(archererr.KindWrite, fmt.Errorf("create output directory: %w", err)) } comp, err := shard.NewCompressor(a.cfg.Compress, a.cfg.CompressLevel) if err != nil { return archererr.Wrap(archererr.KindConfig, err) } man, err := shard.OpenManifest(a.cfg.FileDir) if err != nil { return archererr.Wrap(archererr.KindWrite, err) } a.manifest = man maxSeq, err := man.Adopt(a.cfg.FileDir, a.cfg.Format, comp) if err != nil { return archererr.Wrap(archererr.KindWrite, err) } st, err := checkpoint.Load(a.cfg.FileDir) if err != nil { return archererr.Wrap(archererr.KindValidation, err) } startBucket, startSeq := "", int64(0) if st != nil { if st.ConfigHash != a.cfgHash { if !a.cfg.ForceResume { diffs := cli.DiffFields(st.Config, a.cfgFields) if len(diffs) == 0 { diffs = []string{"fingerprint changed but no field differences were recorded"} } return archererr.Newf(archererr.KindValidation, "checkpoint in %s was created with a different configuration:\n %s\nre-run with --force-resume to ignore this", a.cfg.FileDir, strings.Join(diffs, "\n ")) } a.message("warning: --force-resume ignores the changed configuration fingerprint") } if st.Complete { a.alreadyComplete = true a.st = st return nil } a.st = st a.baseRows = st.RowsDone a.resume = &scan.Resume{Phase: st.Cursor.Phase, Key: st.Cursor.Key, DayStart: st.Cursor.DayStart} startBucket, startSeq = st.LastBucket, st.LastShardSeq } else { a.st = &checkpoint.State{RunID: newRunID(), ShardBy: a.cfg.ShardBy} } if a.reporter != nil { a.reporter.SetBase(a.baseRows) } if maxSeq[startBucket] > startSeq { startSeq = maxSeq[startBucket] } var keep string var resumeMeta *shard.PartialMeta if a.cfg.Purge { if m, err := shard.LoadPartialMeta(a.cfg.FileDir, startBucket, startSeq+1); err != nil { return archererr.Wrap(archererr.KindValidation, err) } else if m != nil { resumeMeta = m keep = shard.PartialPath(a.cfg.FileDir, startBucket, startSeq+1) } } if err := shard.CleanupPartials(a.cfg.FileDir, keep); err != nil { return archererr.Wrap(archererr.KindWrite, err) } w, err := shard.Open(shard.Options{ Dir: a.cfg.FileDir, Mode: a.cfg.ShardBy, ShardRows: int64(a.cfg.ShardRows), Format: a.cfg.Format, Compressor: comp, Columns: a.probe.selectMeta, OutIdx: a.probe.outIdx, CSVNull: a.cfg.CSVNull, StartBucket: startBucket, StartSeq: startSeq, }, man) if err != nil { return archererr.Wrap(archererr.KindWrite, err) } if resumeMeta != nil { if err := w.Resume(resumeMeta); err != nil { return archererr.Wrap(archererr.KindValidation, err) } } a.writer = w a.fileSink = &sink.File{W: w} return nil } func (a *App) openScanner(ctx context.Context) error { opts := scan.Options{ DB: a.srcDB, DBName: a.srcCfg.DBName, Table: a.cfg.Table, Where: a.cfg.Where, SelectCols: a.probe.selectCols, KeyCols: a.probe.key.Columns, KeyIdx: a.probe.keyIdx, ChunkSize: a.cfg.ChunkSize, Shard: a.probe.shard, ShardIdx: a.probe.shardIdx, ShardLoc: a.loc, ZeroDate: a.cfg.ZeroDate, Resume: a.resume, } switch a.probe.path { case "A": a.scanner = scan.NewDateA(opts) case "B": s, err := scan.NewDateB(ctx, opts) if err != nil { return archererr.Wrap(archererr.KindConnection, err) } a.scanner = s default: a.scanner = scan.NewPK(opts) } return nil } func (a *App) execute(ctx context.Context) error { if a.cfg.HasFile() { a.message("note: file output is at-least-once; a crash and rerun may duplicate rows in shard files") } if a.cfg.Purge { a.message("warning: --purge deletes archived rows from the source table") } if a.cfg.BulkDelete { a.message("warning: --bulk-delete removes all rows in the scanned key range that match --where, including concurrent inserts") } if a.cfg.BulkDelete && a.cfg.IsDateShard() { a.message("warning: date mode --bulk-delete assumes the shard column never moves backwards relative to the scan key; a late or backfilled row can be deleted before it is archived") } for { if err := ctx.Err(); err != nil { return a.interrupt(err) } if a.lagChecker != nil { if err := a.lagChecker.Wait(ctx); err != nil { if ctx.Err() != nil { return a.interrupt(err) } return archererr.Wrap(archererr.KindConnection, err) } } a.phase("scan", "") start := time.Now() chunk, err := a.scanner.Next(ctx) a.stats.ScanDur += time.Since(start) if err == io.EOF { break } if err != nil { if ctx.Err() != nil { return a.interrupt(err) } return archererr.Wrap(archererr.KindOther, fmt.Errorf("scan: %w", err)) } if err := a.processChunk(ctx, chunk); err != nil { if ctx.Err() != nil { return a.interrupt(err) } return err } a.stats.RowsRead += int64(chunk.Len()) a.publishProgress(chunk) } return a.finish() } func (a *App) processChunk(ctx context.Context, chunk *row.Chunk) error { fileStart := time.Now() if a.fileSink != nil { a.phase("file", "") if err := a.fileSink.WriteChunk(ctx, chunk); err != nil { return archererr.Wrap(archererr.KindWrite, err) } if a.cfg.Purge { if err := a.fileSink.Checkpoint(); err != nil { return archererr.Wrap(archererr.KindWrite, err) } } } a.stats.FileDur += time.Since(fileStart) dbStart := time.Now() if a.destSink != nil || a.purger != nil { a.phase("db", "") } if err := a.runDB(ctx, chunk); err != nil { return err } a.stats.DBDur += time.Since(dbStart) if a.fileSink != nil && !a.cfg.Purge { if err := a.fileSink.Flush(ctx); err != nil { return archererr.Wrap(archererr.KindWrite, err) } } a.stats.RowsCommitted += int64(chunk.Len()) if a.cfg.HasFile() { a.phase("checkpoint", "") } cpStart := time.Now() if err := a.saveState(false); err != nil { return archererr.Wrap(archererr.KindWrite, fmt.Errorf("save checkpoint: %w", err)) } a.stats.CheckpointDur += time.Since(cpStart) return nil } func (a *App) runDB(ctx context.Context, chunk *row.Chunk) error { attempts := a.cfg.Retries + 1 if a.destSink != nil && a.purger != nil && a.sameInstance { err := retry(ctx, attempts, func() error { tx, err := a.destDB.BeginTx(ctx, nil) if err != nil { return err } n, err := a.destSink.InsertChunk(ctx, tx, chunk) if err != nil { _ = tx.Rollback() return err } d, err := a.purger.DeleteChunk(ctx, tx, chunk) if err != nil { _ = tx.Rollback() return err } if err := tx.Commit(); err != nil { return err } a.stats.RowsInserted += n a.stats.RowsDeleted += d return nil }) if err != nil { return archererr.Wrap(archererr.KindWrite, err) } return nil } if a.destSink != nil { err := retry(ctx, attempts, func() error { tx, err := a.destDB.BeginTx(ctx, nil) if err != nil { return err } n, err := a.destSink.InsertChunk(ctx, tx, chunk) if err != nil { _ = tx.Rollback() return err } if err := tx.Commit(); err != nil { return err } a.stats.RowsInserted += n return nil }) if err != nil { return archererr.Wrap(archererr.KindWrite, err) } } if a.purger != nil { err := retry(ctx, attempts, func() error { tx, err := a.srcDB.BeginTx(ctx, nil) if err != nil { return err } d, err := a.purger.DeleteChunk(ctx, tx, chunk) if err != nil { _ = tx.Rollback() return err } if err := tx.Commit(); err != nil { return err } a.stats.RowsDeleted += d return nil }) if err != nil { return archererr.Wrap(archererr.KindWrite, err) } } return nil } func retry(ctx context.Context, attempts int, fn func() error) error { var err error for i := 0; i < attempts; i++ { if i > 0 { select { case <-ctx.Done(): return ctx.Err() case <-time.After(time.Duration(i) * time.Second): } } if err = fn(); err == nil { return nil } } return err } func (a *App) saveState(complete bool) error { if a.cfg.FileDir == "" || a.st == nil || a.alreadyComplete { return nil } st := a.st st.ConfigHash = a.cfgHash st.Config = a.cfgFields st.ShardBy = a.cfg.ShardBy st.RowsDone = a.baseRows + a.stats.RowsCommitted if a.scanner != nil { cs := a.scanner.ResumeState() st.Cursor = checkpoint.Cursor{Phase: cs.Phase, Key: cs.Key, DayStart: cs.DayStart} } if a.writer != nil { st.LastBucket, st.LastShardSeq = a.writer.StatePosition() } st.Complete = complete return checkpoint.Save(a.cfg.FileDir, st) } func (a *App) finish() error { a.phase("finalize", "") if a.writer != nil { if err := a.writer.Close(); err != nil { return archererr.Wrap(archererr.KindWrite, err) } var shards, bytes int64 _, shards, bytes = a.writer.Stats() a.stats.Shards, a.stats.Bytes = shards, bytes } if err := a.saveState(true); err != nil { return archererr.Wrap(archererr.KindWrite, fmt.Errorf("save checkpoint: %w", err)) } a.printStats() if a.reporter != nil { if a.writer != nil { a.reporter.SetShards(a.stats.Shards) a.reporter.SetBytes(a.stats.Bytes) } a.reporter.Done() } return nil } func (a *App) interrupt(cause error) error { if a.reporter != nil { a.reporter.Interrupted() } if a.writer != nil { if err := a.writer.SealCurrent(); err != nil { a.message("warning: could not seal current shard: %v", err) } } if err := a.saveState(false); err != nil { a.message("warning: could not save checkpoint: %v", err) } return archererr.Wrap(archererr.KindInterrupted, cause) } func (a *App) publishProgress(chunk *row.Chunk) { if a.reporter == nil { return } if a.estimate != nil { switch a.estimate.Kind { case "day": if len(chunk.Buckets) > 0 { b := chunk.Buckets[len(chunk.Buckets)-1] if b != "" && b != "_null" && !strings.HasPrefix(b, "0000-") { if t, err := time.ParseInLocation("2006-01-02", b, a.loc); err == nil { a.reporter.SetPosition(float64(t.Unix())) } } } case "key": if len(chunk.LastKey) == 1 && !chunk.LastKey[0].Null { if n, err := strconv.ParseInt(chunk.LastKey[0].Val, 10, 64); err == nil { a.reporter.SetPosition(float64(n)) } } } } if a.writer != nil { a.reporter.SetShard(a.writer.CurrentShard()) a.reporter.SetBytes(a.writer.WrittenBytes()) } a.reporter.AddRows(int64(chunk.Len())) } func (a *App) printStats() { if !a.cfg.Statistics { return } fmt.Fprintln(os.Stdout, "this run:") fmt.Fprintf(os.Stdout, " rows read: %d\n", a.stats.RowsRead) fmt.Fprintf(os.Stdout, " rows committed: %d\n", a.stats.RowsCommitted) fmt.Fprintf(os.Stdout, " rows inserted: %d\n", a.stats.RowsInserted) fmt.Fprintf(os.Stdout, " rows deleted: %d\n", a.stats.RowsDeleted) if a.writer != nil { fmt.Fprintf(os.Stdout, " shards: %d\n", a.stats.Shards) fmt.Fprintf(os.Stdout, " bytes written: %d\n", a.stats.Bytes) } fmt.Fprintf(os.Stdout, " elapsed: %s (scan %s, file %s, db %s, checkpoint %s)\n", time.Since(a.stats.Started).Round(time.Millisecond), a.stats.ScanDur.Round(time.Millisecond), a.stats.FileDur.Round(time.Millisecond), a.stats.DBDur.Round(time.Millisecond), a.stats.CheckpointDur.Round(time.Millisecond)) fmt.Fprintln(os.Stdout, "cumulative (including prior runs):") fmt.Fprintf(os.Stdout, " rows read: %d\n", a.baseRows+a.stats.RowsRead) fmt.Fprintf(os.Stdout, " rows committed: %d\n", a.baseRows+a.stats.RowsCommitted) if a.writer != nil && a.manifest != nil { var rows, bytes int64 entries := a.manifest.Entries() for _, e := range entries { rows += e.Rows bytes += e.Bytes } fmt.Fprintf(os.Stdout, " shards: %d\n", len(entries)) fmt.Fprintf(os.Stdout, " rows written: %d\n", rows) fmt.Fprintf(os.Stdout, " bytes written: %d\n", bytes) } } func (a *App) close() { if a.scanner != nil { _ = a.scanner.Close() } if a.writer != nil { _ = a.writer.Abandon() } else if a.manifest != nil { _ = a.manifest.Close() } if a.lagChecker != nil { _ = a.lagChecker.Close() } if a.destDB != nil { _ = a.destDB.Close() } if a.srcDB != nil { _ = a.srcDB.Close() } } func openDB(cfg *mysql.Config) (*sql.DB, error) { db, err := sql.Open("mysql", cfg.FormatDSN()) if err != nil { return nil, archererr.Wrap(archererr.KindConnection, err) } db.SetMaxOpenConns(8) db.SetMaxIdleConns(4) db.SetConnMaxLifetime(time.Hour) if err := db.Ping(); err != nil { db.Close() return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("connect %s: %w", cfg.Addr, err)) } return db, nil } func newRunID() string { b := make([]byte, 8) if _, err := rand.Read(b); err != nil { return time.Now().UTC().Format("20060102T150405Z") } return time.Now().UTC().Format("20060102T150405Z") + "-" + hex.EncodeToString(b) }