Files
voidandClaude Sonnet 4.6 366d61ca25 Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-11-08 21:14:23 +08:00

215 lines
5.7 KiB
Go

package importer
import (
"context"
"database/sql"
"fmt"
"io"
"os"
"sync/atomic"
"github.com/go-sql-driver/mysql"
"archer/internal/archererr"
"archer/internal/cli"
"archer/internal/row"
"archer/internal/shard"
"archer/internal/sink"
)
// insertBatchRows bounds the decoder's memory independently of
// max_allowed_packet; sink.DB still splits each batch by packet size.
const insertBatchRows = 1000
type worker struct {
conn *sql.Conn
sink *sink.DB
cfg *cli.ImportConfig
archive *Archive
mapping *Mapping
dbName string
}
func newWorker(ctx context.Context, db *sql.DB, cfg *cli.ImportConfig, archive *Archive, mapping *Mapping, dbName string) (*worker, error) {
conn, err := db.Conn(ctx)
if err != nil {
return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("acquire connection: %w", err))
}
if err := SetSession(ctx, conn, cfg.Charset, cfg.TimeZone); err != nil {
conn.Close()
return nil, err
}
w := &worker{conn: conn, cfg: cfg, archive: archive, mapping: mapping, dbName: dbName}
if cfg.Engine == "insert" {
s, err := sink.NewDB(ctx, db, dbName, cfg.DestTable, mapping.InsertColumns, cfg.Mode, mapping.InsertFieldIdx)
if err != nil {
conn.Close()
return nil, archererr.Wrap(archererr.KindValidation, err)
}
w.sink = s
}
return w, nil
}
func (w *worker) close() {
if w.conn != nil {
_ = w.conn.Close()
}
}
// importShard loads one shard and returns affected rows plus, for ignore mode,
// the number of rows the server skipped.
func (w *worker) importShard(ctx context.Context, e shard.Entry) (affected, skipped int64, err error) {
path := shardPath(w.archive.Root, e)
if w.cfg.Verify == "pre" {
if err := VerifyShard(path, e); err != nil {
return 0, 0, err
}
}
if w.cfg.Engine == "insert" {
return w.insertShard(ctx, e, path)
}
return w.loadShard(ctx, e, path)
}
func (w *worker) loadShard(ctx context.Context, e shard.Entry, path string) (int64, int64, error) {
name := newReaderName(e.Seq)
mysql.RegisterReaderHandler(name, func() io.Reader {
return openShardReader(path, w.archive.Compressor)
})
defer mysql.DeregisterReaderHandler(name)
q := BuildLoadSQL(w.dbName, w.cfg.DestTable, w.cfg.Charset, w.mapping, w.cfg.Mode, name)
res, err := w.conn.ExecContext(ctx, q)
if err != nil {
return 0, 0, archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: LOAD DATA: %w", e.Seq, err))
}
affected, _ := res.RowsAffected()
if err := w.checkWarnings(ctx, e.Seq); err != nil {
return affected, 0, err
}
skipped, err := CheckRowRule(w.cfg.Mode, e.Rows, affected)
if err != nil {
return affected, 0, fmt.Errorf("shard %d: %w", e.Seq, err)
}
return affected, skipped, nil
}
func (w *worker) insertShard(ctx context.Context, e shard.Entry, path string) (int64, int64, error) {
f, err := os.Open(path)
if err != nil {
return 0, 0, archererr.Wrap(archererr.KindValidation, fmt.Errorf("shard %d: open: %w", e.Seq, err))
}
defer f.Close()
r, err := w.archive.Compressor.Reader(f)
if err != nil {
return 0, 0, archererr.Wrap(archererr.KindValidation, fmt.Errorf("shard %d: decompress: %w", e.Seq, err))
}
defer r.Close()
tx, err := w.conn.BeginTx(ctx, nil)
if err != nil {
return 0, 0, archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: begin transaction: %w", e.Seq, err))
}
committed := false
defer func() {
if !committed {
_ = tx.Rollback()
}
}()
dr := NewDumpReader(r)
chunk := &row.Chunk{}
var affected int64
flush := func() error {
if len(chunk.Rows) == 0 {
return nil
}
n, err := w.sink.InsertChunk(ctx, tx, chunk)
if err != nil {
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: %w", e.Seq, err))
}
affected += n
chunk = &row.Chunk{}
return w.checkWarnings(ctx, e.Seq)
}
for {
rw, err := dr.ReadRow()
if err == io.EOF {
break
}
if err != nil {
return affected, 0, archererr.Wrap(archererr.KindValidation, fmt.Errorf("shard %d: decode: %w", e.Seq, err))
}
if len(rw.Values) != len(w.mapping.FileColumns) {
return affected, 0, archererr.Newf(archererr.KindValidation,
"shard %d: row has %d fields but the mapping expects %d", e.Seq, len(rw.Values), len(w.mapping.FileColumns))
}
chunk.Rows = append(chunk.Rows, rw)
if len(chunk.Rows) >= insertBatchRows {
if err := flush(); err != nil {
return affected, 0, err
}
}
}
if err := flush(); err != nil {
return affected, 0, err
}
skipped, err := CheckRowRule(w.cfg.Mode, e.Rows, affected)
if err != nil {
return affected, 0, fmt.Errorf("shard %d: %w", e.Seq, err)
}
if err := tx.Commit(); err != nil {
return affected, 0, archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: commit: %w", e.Seq, err))
}
committed = true
return affected, skipped, nil
}
func (w *worker) checkWarnings(ctx context.Context, seq int64) error {
warnings, err := ShowWarnings(ctx, w.conn)
if err != nil {
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: SHOW WARNINGS: %w", seq, err))
}
return CheckWarnings(warnings, w.cfg.AllowWarnings, w.cfg.Mode, seq)
}
var readerCounter atomic.Uint64
func newReaderName(seq int64) string {
return fmt.Sprintf("archer-import-%d-%d-%d", os.Getpid(), seq, readerCounter.Add(1))
}
type errReader struct{ err error }
func (e errReader) Read([]byte) (int, error) { return 0, e.err }
type readCloserPair struct {
r io.ReadCloser
f *os.File
}
func (p *readCloserPair) Read(b []byte) (int, error) { return p.r.Read(b) }
func (p *readCloserPair) Close() error {
err := p.r.Close()
if ferr := p.f.Close(); err == nil {
err = ferr
}
return err
}
func openShardReader(path string, comp *shard.Compressor) io.Reader {
f, err := os.Open(path)
if err != nil {
return errReader{err}
}
r, err := comp.Reader(f)
if err != nil {
_ = f.Close()
return errReader{err}
}
return &readCloserPair{r: r, f: f}
}