Files
voidandClaude Sonnet 4.6 366d61ca25 Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-11-08 21:14:23 +08:00

419 lines
12 KiB
Go

package importer
import (
"context"
"database/sql"
"fmt"
"io"
"os"
"sync"
"time"
"github.com/go-sql-driver/mysql"
"archer/internal/archererr"
"archer/internal/cli"
"archer/internal/progress"
"archer/internal/schema"
"archer/internal/shard"
)
// Stats collects the --statistics counters for one import run.
type Stats struct {
Started time.Time
Elapsed time.Duration
Selected int
NotSelected int
Succeeded int
Failed int
ExpectedRows int64
ImportedRows int64
SkippedRows int64
Bytes int64
IgnoredFiles int
Partial bool
}
// Run executes one import.
func Run(ctx context.Context, cfg *cli.ImportConfig) error {
if err := cfg.Validate(); err != nil {
return err
}
stats := &Stats{Started: time.Now()}
archive, err := LoadArchive(cfg.FileDir)
if err != nil {
return err
}
if err := archive.CheckComplete(cfg.AllowPartial); err != nil {
return err
}
if err := archive.CheckMappingChoice(cfg.Columns, cfg.AssumeAll); err != nil {
return err
}
stats.Partial = cfg.AllowPartial && (archive.State == nil || !archive.State.Complete)
selected, warnings, err := shard.ApplySelection(archive.Entries, cfg.Selectors)
if err != nil {
return archererr.Newf(archererr.KindConfig, "%v", err)
}
if err := CheckShardFiles(archive.Root, selected); err != nil {
return err
}
format, compress, comp, err := ResolveFormat(selected)
if err != nil {
return err
}
archive.Format, archive.Compress, archive.Compressor = format, compress, comp
stats.Selected = len(selected)
stats.NotSelected = len(archive.Entries) - len(selected)
for _, e := range selected {
stats.ExpectedRows += e.Rows
stats.Bytes += e.Bytes
}
if cfg.DestTable == "" {
cfg.DestTable = archive.SourceTable
}
if cfg.DestTable == "" {
return archererr.Newf(archererr.KindConfig,
"--dest-table is required: the archive does not record the source table name")
}
if cfg.DryRun {
fields, hasRow, ferr := FirstLineFieldCount(shardPath(archive.Root, selected[0]), comp)
if ferr != nil {
return archererr.Wrap(archererr.KindValidation, ferr)
}
return PrintDryRun(os.Stdout, cfg, archive, selected, warnings, compress, fields, hasRow)
}
for _, w := range warnings {
fmt.Fprintln(os.Stderr, "warning:", w)
}
if len(warnings) > 0 {
fmt.Fprintln(os.Stderr, "warning: shard selection is whole-shard; rows outside the requested key range may be imported")
}
destCfg, err := cfg.DestConfig()
if err != nil {
return archererr.Wrap(archererr.KindConfig, err)
}
db, err := openDest(destCfg, cfg.Workers)
if err != nil {
return err
}
defer db.Close()
if err := checkTarget(ctx, db, cfg.Engine); err != nil {
return err
}
if err := checkTableExists(ctx, db, destCfg.DBName, cfg.DestTable); err != nil {
return err
}
table, err := schema.Load(ctx, db, destCfg.DBName, cfg.DestTable)
if err != nil {
return archererr.Wrap(archererr.KindValidation, err)
}
mapping, err := BuildMapping(table, archive.ArchivedColumns, cfg.Columns, cfg.AssumeAll)
if err != nil {
return err
}
if mapping.Positional {
fmt.Fprintln(os.Stderr, "warning: --assume-all-columns accepts positional mapping; only the field count is checked, not column names or order")
}
if err := CheckFieldCount(shardPath(archive.Root, selected[0]), comp, len(mapping.FileColumns)); err != nil {
return err
}
rep := startReporter(cfg, stats.ExpectedRows)
if rep != nil {
defer rep.Stop()
}
if err := runWorkers(ctx, db, destCfg.DBName, cfg, archive, mapping, selected, stats, rep); err != nil {
if archererr.KindOf(err) == archererr.KindInterrupted && rep != nil {
rep.Interrupted()
}
return err
}
stats.Elapsed = time.Since(stats.Started)
stats.IgnoredFiles = UnregisteredFiles(archive.Root, archive.Entries)
if rep != nil {
rep.SetShards(int64(len(selected)))
rep.SetBytes(stats.Bytes)
rep.Done()
}
if cfg.Statistics {
PrintStats(os.Stdout, archive, stats)
}
return nil
}
func openDest(cfg *mysql.Config, workers int) (*sql.DB, error) {
db, err := sql.Open("mysql", cfg.FormatDSN())
if err != nil {
return nil, archererr.Wrap(archererr.KindConnection, err)
}
db.SetMaxOpenConns(workers + 2)
db.SetMaxIdleConns(workers + 1)
db.SetConnMaxLifetime(time.Hour)
if err := db.Ping(); err != nil {
db.Close()
return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("connect %s: %w", cfg.Addr, err))
}
return db, nil
}
func checkTarget(ctx context.Context, db *sql.DB, engine string) error {
var localInfile int
var version string
if err := db.QueryRowContext(ctx, "SELECT @@local_infile, @@version").Scan(&localInfile, &version); err != nil {
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("query local_infile: %w", err))
}
if engine == "load" && localInfile == 0 {
return archererr.Newf(archererr.KindConfig,
"target server (%s) has local_infile disabled; enable it on the server/proxy, or use --engine insert", version)
}
return nil
}
func checkTableExists(ctx context.Context, db *sql.DB, dbName, table string) error {
var n int
err := db.QueryRowContext(ctx, `
SELECT COUNT(*) FROM information_schema.tables
WHERE table_schema = ? AND table_name = ?`, dbName, table).Scan(&n)
if err != nil {
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("check target table: %w", err))
}
if n == 0 {
return archererr.Newf(archererr.KindValidation,
"target table %s.%s does not exist; create it before importing (archer does not create tables)", dbName, table)
}
return nil
}
func startReporter(cfg *cli.ImportConfig, total int64) *progress.Reporter {
tty := progress.IsTerminal(os.Stderr)
enabled := cfg.Progress.Enabled(tty)
r := progress.New(progress.Options{
Out: os.Stderr,
Enabled: enabled,
Live: enabled && tty,
Interval: time.Second,
RowEvery: cfg.Progress.Rows,
})
r.Start()
r.SetEstimate(&progress.Estimate{Kind: "rows", Lo: 0, Hi: float64(total)})
r.SetPosition(0)
return r
}
func runWorkers(ctx context.Context, db *sql.DB, dbName string, cfg *cli.ImportConfig, archive *Archive, mapping *Mapping, selected []shard.Entry, stats *Stats, rep *progress.Reporter) error {
parent := ctx
ctx, cancel := context.WithCancel(ctx)
defer cancel()
var (
wg sync.WaitGroup
mu sync.Mutex
firstErr error
failedSeq int64
processedRows int64
rowsAtFailure int64
importedRows int64
skippedRows int64
okShards int
)
recordFailure := func(e shard.Entry, err error) {
mu.Lock()
defer mu.Unlock()
if firstErr != nil {
return
}
if parent.Err() != nil {
firstErr = archererr.Wrap(archererr.KindInterrupted, parent.Err())
} else {
firstErr = err
}
failedSeq = e.Seq
rowsAtFailure = processedRows
}
ranges := splitRanges(len(selected), cfg.Workers)
for _, rg := range ranges {
if rg[0] >= rg[1] {
continue
}
part := selected[rg[0]:rg[1]]
wg.Add(1)
go func(part []shard.Entry) {
defer wg.Done()
if err := ctx.Err(); err != nil {
recordFailure(part[0], err)
return
}
w, err := newWorker(ctx, db, cfg, archive, mapping, dbName)
if err != nil {
recordFailure(part[0], err)
cancel()
return
}
defer w.close()
for _, e := range part {
if err := ctx.Err(); err != nil {
recordFailure(e, err)
return
}
affected, skipped, err := w.importShard(ctx, e)
if err != nil {
recordFailure(e, err)
cancel()
return
}
mu.Lock()
processedRows += e.Rows
importedRows += affected
skippedRows += skipped
okShards++
done := processedRows
mu.Unlock()
if rep != nil {
rep.AddRows(e.Rows)
rep.SetPosition(float64(done))
}
}
}(part)
}
wg.Wait()
mu.Lock()
stats.Succeeded = okShards
stats.ImportedRows = importedRows
stats.SkippedRows = skippedRows
err := firstErr
seq := failedSeq
atFailure := rowsAtFailure
doneRows := processedRows
mu.Unlock()
if err != nil {
if archererr.KindOf(err) == archererr.KindInterrupted {
return fmt.Errorf("interrupted after %d rows: %w", doneRows, err)
}
stats.Failed = 1
return fmt.Errorf("shard %d failed after %d rows: %w", seq, atFailure, err)
}
return nil
}
func splitRanges(n, workers int) [][2]int {
if n <= 0 || workers <= 0 {
return nil
}
if workers > n {
workers = n
}
out := make([][2]int, 0, workers)
base, rem := n/workers, n%workers
start := 0
for i := 0; i < workers; i++ {
size := base
if i < rem {
size++
}
out = append(out, [2]int{start, start + size})
start += size
}
return out
}
// PrintDryRun prints the resolved configuration and selection without opening
// a database connection.
func PrintDryRun(w io.Writer, cfg *cli.ImportConfig, archive *Archive, selected []shard.Entry, warnings []string, compress string, fields int, hasRow bool) error {
fmt.Fprintln(w, "== import configuration ==")
fmt.Fprintf(w, "archive: %s\n", archive.Root)
fmt.Fprintf(w, "dest: %s\n", redactDSN(cfg.DestDSN))
fmt.Fprintf(w, "dest table: %s\n", cfg.DestTable)
if len(archive.ArchivedColumns) > 0 {
fmt.Fprintf(w, "archive columns: %s\n", join(archive.ArchivedColumns))
} else if len(cfg.Columns) > 0 {
fmt.Fprintf(w, "columns: %s\n", join(cfg.Columns))
} else if cfg.AssumeAll {
fmt.Fprintf(w, "columns: positional (--assume-all-columns)\n")
} else {
fmt.Fprintf(w, "columns: (unresolved; --columns or --assume-all-columns is required)\n")
}
fmt.Fprintf(w, "engine: %s\n", cfg.Engine)
fmt.Fprintf(w, "mode: %s\n", cfg.Mode)
fmt.Fprintf(w, "verify: %s\n", cfg.Verify)
fmt.Fprintf(w, "workers: %d\n", cfg.Workers)
fmt.Fprintf(w, "charset: %s\n", cfg.Charset)
if cfg.TimeZone != "" {
fmt.Fprintf(w, "time zone: %s\n", cfg.TimeZone)
}
fmt.Fprintf(w, "compression: %s\n", compress)
switch {
case archive.State == nil:
fmt.Fprintln(w, "archive complete: unknown (state.json missing)")
case archive.Complete:
fmt.Fprintln(w, "archive complete: true")
default:
fmt.Fprintln(w, "archive complete: false (--allow-partial given)")
}
if hasRow {
fmt.Fprintf(w, "first row fields: %d\n", fields)
}
fmt.Fprintf(w, "\n== selected shards (%d) ==\n", len(selected))
var rows, bytes int64
for _, e := range selected {
date := e.Date
if date == "" {
date = "-"
}
fmt.Fprintf(w, " seq=%-6d pk=%-24s date=%-12s rows=%-10d bytes=%-10d %s\n",
e.Seq, e.PKLo+"-"+e.PKHi, date, e.Rows, e.Bytes, e.Path)
rows += e.Rows
bytes += e.Bytes
}
fmt.Fprintf(w, "total: %d shards, %d rows, %d bytes\n", len(selected), rows, bytes)
fmt.Fprintln(w, "\nno database connection was made and no data was written")
return nil
}
// PrintStats prints the --statistics summary.
func PrintStats(w io.Writer, archive *Archive, s *Stats) {
fmt.Fprintln(w, "import summary:")
fmt.Fprintf(w, " archive: %s\n", archive.Root)
fmt.Fprintf(w, " partial archive: %t\n", s.Partial)
fmt.Fprintf(w, " shards selected: %d\n", s.Selected)
fmt.Fprintf(w, " shards succeeded: %d\n", s.Succeeded)
fmt.Fprintf(w, " shards failed: %d\n", s.Failed)
fmt.Fprintf(w, " shards not selected: %d\n", s.NotSelected)
fmt.Fprintf(w, " rows expected: %d\n", s.ExpectedRows)
fmt.Fprintf(w, " rows affected: %d\n", s.ImportedRows)
fmt.Fprintf(w, " rows skipped: %d\n", s.SkippedRows)
fmt.Fprintf(w, " bytes: %d\n", s.Bytes)
fmt.Fprintf(w, " ignored files: %d (present on disk, not in manifest)\n", s.IgnoredFiles)
fmt.Fprintf(w, " elapsed: %s\n", s.Elapsed.Round(time.Millisecond))
}
func join(parts []string) string {
out := ""
for i, p := range parts {
if i > 0 {
out += ", "
}
out += p
}
return out
}
func redactDSN(dsn string) string {
c, err := mysql.ParseDSN(dsn)
if err != nil {
return "<invalid dsn>"
}
c.Passwd = ""
return c.FormatDSN()
}