Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4a6b504f
commit
366d61ca25
49 files changed
+5099
-342
No files matched your search
@@ -0,0 +1,418 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/cli"
|
||||
"archer/internal/progress"
|
||||
"archer/internal/schema"
|
||||
"archer/internal/shard"
|
||||
)
|
||||
|
||||
// Stats collects the --statistics counters for one import run.
|
||||
type Stats struct {
|
||||
Started time.Time
|
||||
Elapsed time.Duration
|
||||
Selected int
|
||||
NotSelected int
|
||||
Succeeded int
|
||||
Failed int
|
||||
ExpectedRows int64
|
||||
ImportedRows int64
|
||||
SkippedRows int64
|
||||
Bytes int64
|
||||
IgnoredFiles int
|
||||
Partial bool
|
||||
}
|
||||
|
||||
// Run executes one import.
|
||||
func Run(ctx context.Context, cfg *cli.ImportConfig) error {
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
stats := &Stats{Started: time.Now()}
|
||||
|
||||
archive, err := LoadArchive(cfg.FileDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := archive.CheckComplete(cfg.AllowPartial); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := archive.CheckMappingChoice(cfg.Columns, cfg.AssumeAll); err != nil {
|
||||
return err
|
||||
}
|
||||
stats.Partial = cfg.AllowPartial && (archive.State == nil || !archive.State.Complete)
|
||||
|
||||
selected, warnings, err := shard.ApplySelection(archive.Entries, cfg.Selectors)
|
||||
if err != nil {
|
||||
return archererr.Newf(archererr.KindConfig, "%v", err)
|
||||
}
|
||||
if err := CheckShardFiles(archive.Root, selected); err != nil {
|
||||
return err
|
||||
}
|
||||
format, compress, comp, err := ResolveFormat(selected)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
archive.Format, archive.Compress, archive.Compressor = format, compress, comp
|
||||
stats.Selected = len(selected)
|
||||
stats.NotSelected = len(archive.Entries) - len(selected)
|
||||
for _, e := range selected {
|
||||
stats.ExpectedRows += e.Rows
|
||||
stats.Bytes += e.Bytes
|
||||
}
|
||||
|
||||
if cfg.DestTable == "" {
|
||||
cfg.DestTable = archive.SourceTable
|
||||
}
|
||||
if cfg.DestTable == "" {
|
||||
return archererr.Newf(archererr.KindConfig,
|
||||
"--dest-table is required: the archive does not record the source table name")
|
||||
}
|
||||
|
||||
if cfg.DryRun {
|
||||
fields, hasRow, ferr := FirstLineFieldCount(shardPath(archive.Root, selected[0]), comp)
|
||||
if ferr != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, ferr)
|
||||
}
|
||||
return PrintDryRun(os.Stdout, cfg, archive, selected, warnings, compress, fields, hasRow)
|
||||
}
|
||||
|
||||
for _, w := range warnings {
|
||||
fmt.Fprintln(os.Stderr, "warning:", w)
|
||||
}
|
||||
if len(warnings) > 0 {
|
||||
fmt.Fprintln(os.Stderr, "warning: shard selection is whole-shard; rows outside the requested key range may be imported")
|
||||
}
|
||||
|
||||
destCfg, err := cfg.DestConfig()
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConfig, err)
|
||||
}
|
||||
db, err := openDest(destCfg, cfg.Workers)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if err := checkTarget(ctx, db, cfg.Engine); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkTableExists(ctx, db, destCfg.DBName, cfg.DestTable); err != nil {
|
||||
return err
|
||||
}
|
||||
table, err := schema.Load(ctx, db, destCfg.DBName, cfg.DestTable)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
mapping, err := BuildMapping(table, archive.ArchivedColumns, cfg.Columns, cfg.AssumeAll)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if mapping.Positional {
|
||||
fmt.Fprintln(os.Stderr, "warning: --assume-all-columns accepts positional mapping; only the field count is checked, not column names or order")
|
||||
}
|
||||
if err := CheckFieldCount(shardPath(archive.Root, selected[0]), comp, len(mapping.FileColumns)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rep := startReporter(cfg, stats.ExpectedRows)
|
||||
if rep != nil {
|
||||
defer rep.Stop()
|
||||
}
|
||||
if err := runWorkers(ctx, db, destCfg.DBName, cfg, archive, mapping, selected, stats, rep); err != nil {
|
||||
if archererr.KindOf(err) == archererr.KindInterrupted && rep != nil {
|
||||
rep.Interrupted()
|
||||
}
|
||||
return err
|
||||
}
|
||||
stats.Elapsed = time.Since(stats.Started)
|
||||
stats.IgnoredFiles = UnregisteredFiles(archive.Root, archive.Entries)
|
||||
if rep != nil {
|
||||
rep.SetShards(int64(len(selected)))
|
||||
rep.SetBytes(stats.Bytes)
|
||||
rep.Done()
|
||||
}
|
||||
if cfg.Statistics {
|
||||
PrintStats(os.Stdout, archive, stats)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func openDest(cfg *mysql.Config, workers int) (*sql.DB, error) {
|
||||
db, err := sql.Open("mysql", cfg.FormatDSN())
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindConnection, err)
|
||||
}
|
||||
db.SetMaxOpenConns(workers + 2)
|
||||
db.SetMaxIdleConns(workers + 1)
|
||||
db.SetConnMaxLifetime(time.Hour)
|
||||
if err := db.Ping(); err != nil {
|
||||
db.Close()
|
||||
return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("connect %s: %w", cfg.Addr, err))
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func checkTarget(ctx context.Context, db *sql.DB, engine string) error {
|
||||
var localInfile int
|
||||
var version string
|
||||
if err := db.QueryRowContext(ctx, "SELECT @@local_infile, @@version").Scan(&localInfile, &version); err != nil {
|
||||
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("query local_infile: %w", err))
|
||||
}
|
||||
if engine == "load" && localInfile == 0 {
|
||||
return archererr.Newf(archererr.KindConfig,
|
||||
"target server (%s) has local_infile disabled; enable it on the server/proxy, or use --engine insert", version)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkTableExists(ctx context.Context, db *sql.DB, dbName, table string) error {
|
||||
var n int
|
||||
err := db.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*) FROM information_schema.tables
|
||||
WHERE table_schema = ? AND table_name = ?`, dbName, table).Scan(&n)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("check target table: %w", err))
|
||||
}
|
||||
if n == 0 {
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"target table %s.%s does not exist; create it before importing (archer does not create tables)", dbName, table)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func startReporter(cfg *cli.ImportConfig, total int64) *progress.Reporter {
|
||||
tty := progress.IsTerminal(os.Stderr)
|
||||
enabled := cfg.Progress.Enabled(tty)
|
||||
r := progress.New(progress.Options{
|
||||
Out: os.Stderr,
|
||||
Enabled: enabled,
|
||||
Live: enabled && tty,
|
||||
Interval: time.Second,
|
||||
RowEvery: cfg.Progress.Rows,
|
||||
})
|
||||
r.Start()
|
||||
r.SetEstimate(&progress.Estimate{Kind: "rows", Lo: 0, Hi: float64(total)})
|
||||
r.SetPosition(0)
|
||||
return r
|
||||
}
|
||||
|
||||
func runWorkers(ctx context.Context, db *sql.DB, dbName string, cfg *cli.ImportConfig, archive *Archive, mapping *Mapping, selected []shard.Entry, stats *Stats, rep *progress.Reporter) error {
|
||||
parent := ctx
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
firstErr error
|
||||
failedSeq int64
|
||||
processedRows int64
|
||||
rowsAtFailure int64
|
||||
importedRows int64
|
||||
skippedRows int64
|
||||
okShards int
|
||||
)
|
||||
recordFailure := func(e shard.Entry, err error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if firstErr != nil {
|
||||
return
|
||||
}
|
||||
if parent.Err() != nil {
|
||||
firstErr = archererr.Wrap(archererr.KindInterrupted, parent.Err())
|
||||
} else {
|
||||
firstErr = err
|
||||
}
|
||||
failedSeq = e.Seq
|
||||
rowsAtFailure = processedRows
|
||||
}
|
||||
|
||||
ranges := splitRanges(len(selected), cfg.Workers)
|
||||
for _, rg := range ranges {
|
||||
if rg[0] >= rg[1] {
|
||||
continue
|
||||
}
|
||||
part := selected[rg[0]:rg[1]]
|
||||
wg.Add(1)
|
||||
go func(part []shard.Entry) {
|
||||
defer wg.Done()
|
||||
if err := ctx.Err(); err != nil {
|
||||
recordFailure(part[0], err)
|
||||
return
|
||||
}
|
||||
w, err := newWorker(ctx, db, cfg, archive, mapping, dbName)
|
||||
if err != nil {
|
||||
recordFailure(part[0], err)
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
defer w.close()
|
||||
for _, e := range part {
|
||||
if err := ctx.Err(); err != nil {
|
||||
recordFailure(e, err)
|
||||
return
|
||||
}
|
||||
affected, skipped, err := w.importShard(ctx, e)
|
||||
if err != nil {
|
||||
recordFailure(e, err)
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
processedRows += e.Rows
|
||||
importedRows += affected
|
||||
skippedRows += skipped
|
||||
okShards++
|
||||
done := processedRows
|
||||
mu.Unlock()
|
||||
if rep != nil {
|
||||
rep.AddRows(e.Rows)
|
||||
rep.SetPosition(float64(done))
|
||||
}
|
||||
}
|
||||
}(part)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
mu.Lock()
|
||||
stats.Succeeded = okShards
|
||||
stats.ImportedRows = importedRows
|
||||
stats.SkippedRows = skippedRows
|
||||
err := firstErr
|
||||
seq := failedSeq
|
||||
atFailure := rowsAtFailure
|
||||
doneRows := processedRows
|
||||
mu.Unlock()
|
||||
if err != nil {
|
||||
if archererr.KindOf(err) == archererr.KindInterrupted {
|
||||
return fmt.Errorf("interrupted after %d rows: %w", doneRows, err)
|
||||
}
|
||||
stats.Failed = 1
|
||||
return fmt.Errorf("shard %d failed after %d rows: %w", seq, atFailure, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func splitRanges(n, workers int) [][2]int {
|
||||
if n <= 0 || workers <= 0 {
|
||||
return nil
|
||||
}
|
||||
if workers > n {
|
||||
workers = n
|
||||
}
|
||||
out := make([][2]int, 0, workers)
|
||||
base, rem := n/workers, n%workers
|
||||
start := 0
|
||||
for i := 0; i < workers; i++ {
|
||||
size := base
|
||||
if i < rem {
|
||||
size++
|
||||
}
|
||||
out = append(out, [2]int{start, start + size})
|
||||
start += size
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PrintDryRun prints the resolved configuration and selection without opening
|
||||
// a database connection.
|
||||
func PrintDryRun(w io.Writer, cfg *cli.ImportConfig, archive *Archive, selected []shard.Entry, warnings []string, compress string, fields int, hasRow bool) error {
|
||||
fmt.Fprintln(w, "== import configuration ==")
|
||||
fmt.Fprintf(w, "archive: %s\n", archive.Root)
|
||||
fmt.Fprintf(w, "dest: %s\n", redactDSN(cfg.DestDSN))
|
||||
fmt.Fprintf(w, "dest table: %s\n", cfg.DestTable)
|
||||
if len(archive.ArchivedColumns) > 0 {
|
||||
fmt.Fprintf(w, "archive columns: %s\n", join(archive.ArchivedColumns))
|
||||
} else if len(cfg.Columns) > 0 {
|
||||
fmt.Fprintf(w, "columns: %s\n", join(cfg.Columns))
|
||||
} else if cfg.AssumeAll {
|
||||
fmt.Fprintf(w, "columns: positional (--assume-all-columns)\n")
|
||||
} else {
|
||||
fmt.Fprintf(w, "columns: (unresolved; --columns or --assume-all-columns is required)\n")
|
||||
}
|
||||
fmt.Fprintf(w, "engine: %s\n", cfg.Engine)
|
||||
fmt.Fprintf(w, "mode: %s\n", cfg.Mode)
|
||||
fmt.Fprintf(w, "verify: %s\n", cfg.Verify)
|
||||
fmt.Fprintf(w, "workers: %d\n", cfg.Workers)
|
||||
fmt.Fprintf(w, "charset: %s\n", cfg.Charset)
|
||||
if cfg.TimeZone != "" {
|
||||
fmt.Fprintf(w, "time zone: %s\n", cfg.TimeZone)
|
||||
}
|
||||
fmt.Fprintf(w, "compression: %s\n", compress)
|
||||
switch {
|
||||
case archive.State == nil:
|
||||
fmt.Fprintln(w, "archive complete: unknown (state.json missing)")
|
||||
case archive.Complete:
|
||||
fmt.Fprintln(w, "archive complete: true")
|
||||
default:
|
||||
fmt.Fprintln(w, "archive complete: false (--allow-partial given)")
|
||||
}
|
||||
if hasRow {
|
||||
fmt.Fprintf(w, "first row fields: %d\n", fields)
|
||||
}
|
||||
|
||||
fmt.Fprintf(w, "\n== selected shards (%d) ==\n", len(selected))
|
||||
var rows, bytes int64
|
||||
for _, e := range selected {
|
||||
date := e.Date
|
||||
if date == "" {
|
||||
date = "-"
|
||||
}
|
||||
fmt.Fprintf(w, " seq=%-6d pk=%-24s date=%-12s rows=%-10d bytes=%-10d %s\n",
|
||||
e.Seq, e.PKLo+"-"+e.PKHi, date, e.Rows, e.Bytes, e.Path)
|
||||
rows += e.Rows
|
||||
bytes += e.Bytes
|
||||
}
|
||||
fmt.Fprintf(w, "total: %d shards, %d rows, %d bytes\n", len(selected), rows, bytes)
|
||||
fmt.Fprintln(w, "\nno database connection was made and no data was written")
|
||||
return nil
|
||||
}
|
||||
|
||||
// PrintStats prints the --statistics summary.
|
||||
func PrintStats(w io.Writer, archive *Archive, s *Stats) {
|
||||
fmt.Fprintln(w, "import summary:")
|
||||
fmt.Fprintf(w, " archive: %s\n", archive.Root)
|
||||
fmt.Fprintf(w, " partial archive: %t\n", s.Partial)
|
||||
fmt.Fprintf(w, " shards selected: %d\n", s.Selected)
|
||||
fmt.Fprintf(w, " shards succeeded: %d\n", s.Succeeded)
|
||||
fmt.Fprintf(w, " shards failed: %d\n", s.Failed)
|
||||
fmt.Fprintf(w, " shards not selected: %d\n", s.NotSelected)
|
||||
fmt.Fprintf(w, " rows expected: %d\n", s.ExpectedRows)
|
||||
fmt.Fprintf(w, " rows affected: %d\n", s.ImportedRows)
|
||||
fmt.Fprintf(w, " rows skipped: %d\n", s.SkippedRows)
|
||||
fmt.Fprintf(w, " bytes: %d\n", s.Bytes)
|
||||
fmt.Fprintf(w, " ignored files: %d (present on disk, not in manifest)\n", s.IgnoredFiles)
|
||||
fmt.Fprintf(w, " elapsed: %s\n", s.Elapsed.Round(time.Millisecond))
|
||||
}
|
||||
|
||||
func join(parts []string) string {
|
||||
out := ""
|
||||
for i, p := range parts {
|
||||
if i > 0 {
|
||||
out += ", "
|
||||
}
|
||||
out += p
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func redactDSN(dsn string) string {
|
||||
c, err := mysql.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
return "<invalid dsn>"
|
||||
}
|
||||
c.Passwd = ""
|
||||
return c.FormatDSN()
|
||||
}
|
||||
Reference in new issue
Block a user