archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
419 lines
12 KiB
Go
419 lines
12 KiB
Go
package importer
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/go-sql-driver/mysql"
|
|
|
|
"archer/internal/archererr"
|
|
"archer/internal/cli"
|
|
"archer/internal/progress"
|
|
"archer/internal/schema"
|
|
"archer/internal/shard"
|
|
)
|
|
|
|
// Stats collects the --statistics counters for one import run.
|
|
type Stats struct {
|
|
Started time.Time
|
|
Elapsed time.Duration
|
|
Selected int
|
|
NotSelected int
|
|
Succeeded int
|
|
Failed int
|
|
ExpectedRows int64
|
|
ImportedRows int64
|
|
SkippedRows int64
|
|
Bytes int64
|
|
IgnoredFiles int
|
|
Partial bool
|
|
}
|
|
|
|
// Run executes one import.
|
|
func Run(ctx context.Context, cfg *cli.ImportConfig) error {
|
|
if err := cfg.Validate(); err != nil {
|
|
return err
|
|
}
|
|
stats := &Stats{Started: time.Now()}
|
|
|
|
archive, err := LoadArchive(cfg.FileDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := archive.CheckComplete(cfg.AllowPartial); err != nil {
|
|
return err
|
|
}
|
|
if err := archive.CheckMappingChoice(cfg.Columns, cfg.AssumeAll); err != nil {
|
|
return err
|
|
}
|
|
stats.Partial = cfg.AllowPartial && (archive.State == nil || !archive.State.Complete)
|
|
|
|
selected, warnings, err := shard.ApplySelection(archive.Entries, cfg.Selectors)
|
|
if err != nil {
|
|
return archererr.Newf(archererr.KindConfig, "%v", err)
|
|
}
|
|
if err := CheckShardFiles(archive.Root, selected); err != nil {
|
|
return err
|
|
}
|
|
format, compress, comp, err := ResolveFormat(selected)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
archive.Format, archive.Compress, archive.Compressor = format, compress, comp
|
|
stats.Selected = len(selected)
|
|
stats.NotSelected = len(archive.Entries) - len(selected)
|
|
for _, e := range selected {
|
|
stats.ExpectedRows += e.Rows
|
|
stats.Bytes += e.Bytes
|
|
}
|
|
|
|
if cfg.DestTable == "" {
|
|
cfg.DestTable = archive.SourceTable
|
|
}
|
|
if cfg.DestTable == "" {
|
|
return archererr.Newf(archererr.KindConfig,
|
|
"--dest-table is required: the archive does not record the source table name")
|
|
}
|
|
|
|
if cfg.DryRun {
|
|
fields, hasRow, ferr := FirstLineFieldCount(shardPath(archive.Root, selected[0]), comp)
|
|
if ferr != nil {
|
|
return archererr.Wrap(archererr.KindValidation, ferr)
|
|
}
|
|
return PrintDryRun(os.Stdout, cfg, archive, selected, warnings, compress, fields, hasRow)
|
|
}
|
|
|
|
for _, w := range warnings {
|
|
fmt.Fprintln(os.Stderr, "warning:", w)
|
|
}
|
|
if len(warnings) > 0 {
|
|
fmt.Fprintln(os.Stderr, "warning: shard selection is whole-shard; rows outside the requested key range may be imported")
|
|
}
|
|
|
|
destCfg, err := cfg.DestConfig()
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindConfig, err)
|
|
}
|
|
db, err := openDest(destCfg, cfg.Workers)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer db.Close()
|
|
|
|
if err := checkTarget(ctx, db, cfg.Engine); err != nil {
|
|
return err
|
|
}
|
|
if err := checkTableExists(ctx, db, destCfg.DBName, cfg.DestTable); err != nil {
|
|
return err
|
|
}
|
|
table, err := schema.Load(ctx, db, destCfg.DBName, cfg.DestTable)
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindValidation, err)
|
|
}
|
|
mapping, err := BuildMapping(table, archive.ArchivedColumns, cfg.Columns, cfg.AssumeAll)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if mapping.Positional {
|
|
fmt.Fprintln(os.Stderr, "warning: --assume-all-columns accepts positional mapping; only the field count is checked, not column names or order")
|
|
}
|
|
if err := CheckFieldCount(shardPath(archive.Root, selected[0]), comp, len(mapping.FileColumns)); err != nil {
|
|
return err
|
|
}
|
|
|
|
rep := startReporter(cfg, stats.ExpectedRows)
|
|
if rep != nil {
|
|
defer rep.Stop()
|
|
}
|
|
if err := runWorkers(ctx, db, destCfg.DBName, cfg, archive, mapping, selected, stats, rep); err != nil {
|
|
if archererr.KindOf(err) == archererr.KindInterrupted && rep != nil {
|
|
rep.Interrupted()
|
|
}
|
|
return err
|
|
}
|
|
stats.Elapsed = time.Since(stats.Started)
|
|
stats.IgnoredFiles = UnregisteredFiles(archive.Root, archive.Entries)
|
|
if rep != nil {
|
|
rep.SetShards(int64(len(selected)))
|
|
rep.SetBytes(stats.Bytes)
|
|
rep.Done()
|
|
}
|
|
if cfg.Statistics {
|
|
PrintStats(os.Stdout, archive, stats)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func openDest(cfg *mysql.Config, workers int) (*sql.DB, error) {
|
|
db, err := sql.Open("mysql", cfg.FormatDSN())
|
|
if err != nil {
|
|
return nil, archererr.Wrap(archererr.KindConnection, err)
|
|
}
|
|
db.SetMaxOpenConns(workers + 2)
|
|
db.SetMaxIdleConns(workers + 1)
|
|
db.SetConnMaxLifetime(time.Hour)
|
|
if err := db.Ping(); err != nil {
|
|
db.Close()
|
|
return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("connect %s: %w", cfg.Addr, err))
|
|
}
|
|
return db, nil
|
|
}
|
|
|
|
func checkTarget(ctx context.Context, db *sql.DB, engine string) error {
|
|
var localInfile int
|
|
var version string
|
|
if err := db.QueryRowContext(ctx, "SELECT @@local_infile, @@version").Scan(&localInfile, &version); err != nil {
|
|
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("query local_infile: %w", err))
|
|
}
|
|
if engine == "load" && localInfile == 0 {
|
|
return archererr.Newf(archererr.KindConfig,
|
|
"target server (%s) has local_infile disabled; enable it on the server/proxy, or use --engine insert", version)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func checkTableExists(ctx context.Context, db *sql.DB, dbName, table string) error {
|
|
var n int
|
|
err := db.QueryRowContext(ctx, `
|
|
SELECT COUNT(*) FROM information_schema.tables
|
|
WHERE table_schema = ? AND table_name = ?`, dbName, table).Scan(&n)
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("check target table: %w", err))
|
|
}
|
|
if n == 0 {
|
|
return archererr.Newf(archererr.KindValidation,
|
|
"target table %s.%s does not exist; create it before importing (archer does not create tables)", dbName, table)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func startReporter(cfg *cli.ImportConfig, total int64) *progress.Reporter {
|
|
tty := progress.IsTerminal(os.Stderr)
|
|
enabled := cfg.Progress.Enabled(tty)
|
|
r := progress.New(progress.Options{
|
|
Out: os.Stderr,
|
|
Enabled: enabled,
|
|
Live: enabled && tty,
|
|
Interval: time.Second,
|
|
RowEvery: cfg.Progress.Rows,
|
|
})
|
|
r.Start()
|
|
r.SetEstimate(&progress.Estimate{Kind: "rows", Lo: 0, Hi: float64(total)})
|
|
r.SetPosition(0)
|
|
return r
|
|
}
|
|
|
|
func runWorkers(ctx context.Context, db *sql.DB, dbName string, cfg *cli.ImportConfig, archive *Archive, mapping *Mapping, selected []shard.Entry, stats *Stats, rep *progress.Reporter) error {
|
|
parent := ctx
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
|
|
var (
|
|
wg sync.WaitGroup
|
|
mu sync.Mutex
|
|
firstErr error
|
|
failedSeq int64
|
|
processedRows int64
|
|
rowsAtFailure int64
|
|
importedRows int64
|
|
skippedRows int64
|
|
okShards int
|
|
)
|
|
recordFailure := func(e shard.Entry, err error) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if firstErr != nil {
|
|
return
|
|
}
|
|
if parent.Err() != nil {
|
|
firstErr = archererr.Wrap(archererr.KindInterrupted, parent.Err())
|
|
} else {
|
|
firstErr = err
|
|
}
|
|
failedSeq = e.Seq
|
|
rowsAtFailure = processedRows
|
|
}
|
|
|
|
ranges := splitRanges(len(selected), cfg.Workers)
|
|
for _, rg := range ranges {
|
|
if rg[0] >= rg[1] {
|
|
continue
|
|
}
|
|
part := selected[rg[0]:rg[1]]
|
|
wg.Add(1)
|
|
go func(part []shard.Entry) {
|
|
defer wg.Done()
|
|
if err := ctx.Err(); err != nil {
|
|
recordFailure(part[0], err)
|
|
return
|
|
}
|
|
w, err := newWorker(ctx, db, cfg, archive, mapping, dbName)
|
|
if err != nil {
|
|
recordFailure(part[0], err)
|
|
cancel()
|
|
return
|
|
}
|
|
defer w.close()
|
|
for _, e := range part {
|
|
if err := ctx.Err(); err != nil {
|
|
recordFailure(e, err)
|
|
return
|
|
}
|
|
affected, skipped, err := w.importShard(ctx, e)
|
|
if err != nil {
|
|
recordFailure(e, err)
|
|
cancel()
|
|
return
|
|
}
|
|
mu.Lock()
|
|
processedRows += e.Rows
|
|
importedRows += affected
|
|
skippedRows += skipped
|
|
okShards++
|
|
done := processedRows
|
|
mu.Unlock()
|
|
if rep != nil {
|
|
rep.AddRows(e.Rows)
|
|
rep.SetPosition(float64(done))
|
|
}
|
|
}
|
|
}(part)
|
|
}
|
|
wg.Wait()
|
|
|
|
mu.Lock()
|
|
stats.Succeeded = okShards
|
|
stats.ImportedRows = importedRows
|
|
stats.SkippedRows = skippedRows
|
|
err := firstErr
|
|
seq := failedSeq
|
|
atFailure := rowsAtFailure
|
|
doneRows := processedRows
|
|
mu.Unlock()
|
|
if err != nil {
|
|
if archererr.KindOf(err) == archererr.KindInterrupted {
|
|
return fmt.Errorf("interrupted after %d rows: %w", doneRows, err)
|
|
}
|
|
stats.Failed = 1
|
|
return fmt.Errorf("shard %d failed after %d rows: %w", seq, atFailure, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func splitRanges(n, workers int) [][2]int {
|
|
if n <= 0 || workers <= 0 {
|
|
return nil
|
|
}
|
|
if workers > n {
|
|
workers = n
|
|
}
|
|
out := make([][2]int, 0, workers)
|
|
base, rem := n/workers, n%workers
|
|
start := 0
|
|
for i := 0; i < workers; i++ {
|
|
size := base
|
|
if i < rem {
|
|
size++
|
|
}
|
|
out = append(out, [2]int{start, start + size})
|
|
start += size
|
|
}
|
|
return out
|
|
}
|
|
|
|
// PrintDryRun prints the resolved configuration and selection without opening
|
|
// a database connection.
|
|
func PrintDryRun(w io.Writer, cfg *cli.ImportConfig, archive *Archive, selected []shard.Entry, warnings []string, compress string, fields int, hasRow bool) error {
|
|
fmt.Fprintln(w, "== import configuration ==")
|
|
fmt.Fprintf(w, "archive: %s\n", archive.Root)
|
|
fmt.Fprintf(w, "dest: %s\n", redactDSN(cfg.DestDSN))
|
|
fmt.Fprintf(w, "dest table: %s\n", cfg.DestTable)
|
|
if len(archive.ArchivedColumns) > 0 {
|
|
fmt.Fprintf(w, "archive columns: %s\n", join(archive.ArchivedColumns))
|
|
} else if len(cfg.Columns) > 0 {
|
|
fmt.Fprintf(w, "columns: %s\n", join(cfg.Columns))
|
|
} else if cfg.AssumeAll {
|
|
fmt.Fprintf(w, "columns: positional (--assume-all-columns)\n")
|
|
} else {
|
|
fmt.Fprintf(w, "columns: (unresolved; --columns or --assume-all-columns is required)\n")
|
|
}
|
|
fmt.Fprintf(w, "engine: %s\n", cfg.Engine)
|
|
fmt.Fprintf(w, "mode: %s\n", cfg.Mode)
|
|
fmt.Fprintf(w, "verify: %s\n", cfg.Verify)
|
|
fmt.Fprintf(w, "workers: %d\n", cfg.Workers)
|
|
fmt.Fprintf(w, "charset: %s\n", cfg.Charset)
|
|
if cfg.TimeZone != "" {
|
|
fmt.Fprintf(w, "time zone: %s\n", cfg.TimeZone)
|
|
}
|
|
fmt.Fprintf(w, "compression: %s\n", compress)
|
|
switch {
|
|
case archive.State == nil:
|
|
fmt.Fprintln(w, "archive complete: unknown (state.json missing)")
|
|
case archive.Complete:
|
|
fmt.Fprintln(w, "archive complete: true")
|
|
default:
|
|
fmt.Fprintln(w, "archive complete: false (--allow-partial given)")
|
|
}
|
|
if hasRow {
|
|
fmt.Fprintf(w, "first row fields: %d\n", fields)
|
|
}
|
|
|
|
fmt.Fprintf(w, "\n== selected shards (%d) ==\n", len(selected))
|
|
var rows, bytes int64
|
|
for _, e := range selected {
|
|
date := e.Date
|
|
if date == "" {
|
|
date = "-"
|
|
}
|
|
fmt.Fprintf(w, " seq=%-6d pk=%-24s date=%-12s rows=%-10d bytes=%-10d %s\n",
|
|
e.Seq, e.PKLo+"-"+e.PKHi, date, e.Rows, e.Bytes, e.Path)
|
|
rows += e.Rows
|
|
bytes += e.Bytes
|
|
}
|
|
fmt.Fprintf(w, "total: %d shards, %d rows, %d bytes\n", len(selected), rows, bytes)
|
|
fmt.Fprintln(w, "\nno database connection was made and no data was written")
|
|
return nil
|
|
}
|
|
|
|
// PrintStats prints the --statistics summary.
|
|
func PrintStats(w io.Writer, archive *Archive, s *Stats) {
|
|
fmt.Fprintln(w, "import summary:")
|
|
fmt.Fprintf(w, " archive: %s\n", archive.Root)
|
|
fmt.Fprintf(w, " partial archive: %t\n", s.Partial)
|
|
fmt.Fprintf(w, " shards selected: %d\n", s.Selected)
|
|
fmt.Fprintf(w, " shards succeeded: %d\n", s.Succeeded)
|
|
fmt.Fprintf(w, " shards failed: %d\n", s.Failed)
|
|
fmt.Fprintf(w, " shards not selected: %d\n", s.NotSelected)
|
|
fmt.Fprintf(w, " rows expected: %d\n", s.ExpectedRows)
|
|
fmt.Fprintf(w, " rows affected: %d\n", s.ImportedRows)
|
|
fmt.Fprintf(w, " rows skipped: %d\n", s.SkippedRows)
|
|
fmt.Fprintf(w, " bytes: %d\n", s.Bytes)
|
|
fmt.Fprintf(w, " ignored files: %d (present on disk, not in manifest)\n", s.IgnoredFiles)
|
|
fmt.Fprintf(w, " elapsed: %s\n", s.Elapsed.Round(time.Millisecond))
|
|
}
|
|
|
|
func join(parts []string) string {
|
|
out := ""
|
|
for i, p := range parts {
|
|
if i > 0 {
|
|
out += ", "
|
|
}
|
|
out += p
|
|
}
|
|
return out
|
|
}
|
|
|
|
func redactDSN(dsn string) string {
|
|
c, err := mysql.ParseDSN(dsn)
|
|
if err != nil {
|
|
return "<invalid dsn>"
|
|
}
|
|
c.Passwd = ""
|
|
return c.FormatDSN()
|
|
}
|