Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4a6b504f
commit
366d61ca25
49 files changed
+5099
-342
No files matched your search
@@ -0,0 +1,214 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/cli"
|
||||
"archer/internal/row"
|
||||
"archer/internal/shard"
|
||||
"archer/internal/sink"
|
||||
)
|
||||
|
||||
// insertBatchRows bounds the decoder's memory independently of
|
||||
// max_allowed_packet; sink.DB still splits each batch by packet size.
|
||||
const insertBatchRows = 1000
|
||||
|
||||
type worker struct {
|
||||
conn *sql.Conn
|
||||
sink *sink.DB
|
||||
cfg *cli.ImportConfig
|
||||
archive *Archive
|
||||
mapping *Mapping
|
||||
dbName string
|
||||
}
|
||||
|
||||
func newWorker(ctx context.Context, db *sql.DB, cfg *cli.ImportConfig, archive *Archive, mapping *Mapping, dbName string) (*worker, error) {
|
||||
conn, err := db.Conn(ctx)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("acquire connection: %w", err))
|
||||
}
|
||||
if err := SetSession(ctx, conn, cfg.Charset, cfg.TimeZone); err != nil {
|
||||
conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
w := &worker{conn: conn, cfg: cfg, archive: archive, mapping: mapping, dbName: dbName}
|
||||
if cfg.Engine == "insert" {
|
||||
s, err := sink.NewDB(ctx, db, dbName, cfg.DestTable, mapping.InsertColumns, cfg.Mode, mapping.InsertFieldIdx)
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
w.sink = s
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
func (w *worker) close() {
|
||||
if w.conn != nil {
|
||||
_ = w.conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// importShard loads one shard and returns affected rows plus, for ignore mode,
|
||||
// the number of rows the server skipped.
|
||||
func (w *worker) importShard(ctx context.Context, e shard.Entry) (affected, skipped int64, err error) {
|
||||
path := shardPath(w.archive.Root, e)
|
||||
if w.cfg.Verify == "pre" {
|
||||
if err := VerifyShard(path, e); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
}
|
||||
if w.cfg.Engine == "insert" {
|
||||
return w.insertShard(ctx, e, path)
|
||||
}
|
||||
return w.loadShard(ctx, e, path)
|
||||
}
|
||||
|
||||
func (w *worker) loadShard(ctx context.Context, e shard.Entry, path string) (int64, int64, error) {
|
||||
name := newReaderName(e.Seq)
|
||||
mysql.RegisterReaderHandler(name, func() io.Reader {
|
||||
return openShardReader(path, w.archive.Compressor)
|
||||
})
|
||||
defer mysql.DeregisterReaderHandler(name)
|
||||
|
||||
q := BuildLoadSQL(w.dbName, w.cfg.DestTable, w.cfg.Charset, w.mapping, w.cfg.Mode, name)
|
||||
res, err := w.conn.ExecContext(ctx, q)
|
||||
if err != nil {
|
||||
return 0, 0, archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: LOAD DATA: %w", e.Seq, err))
|
||||
}
|
||||
affected, _ := res.RowsAffected()
|
||||
if err := w.checkWarnings(ctx, e.Seq); err != nil {
|
||||
return affected, 0, err
|
||||
}
|
||||
skipped, err := CheckRowRule(w.cfg.Mode, e.Rows, affected)
|
||||
if err != nil {
|
||||
return affected, 0, fmt.Errorf("shard %d: %w", e.Seq, err)
|
||||
}
|
||||
return affected, skipped, nil
|
||||
}
|
||||
|
||||
func (w *worker) insertShard(ctx context.Context, e shard.Entry, path string) (int64, int64, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, 0, archererr.Wrap(archererr.KindValidation, fmt.Errorf("shard %d: open: %w", e.Seq, err))
|
||||
}
|
||||
defer f.Close()
|
||||
r, err := w.archive.Compressor.Reader(f)
|
||||
if err != nil {
|
||||
return 0, 0, archererr.Wrap(archererr.KindValidation, fmt.Errorf("shard %d: decompress: %w", e.Seq, err))
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
tx, err := w.conn.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, 0, archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: begin transaction: %w", e.Seq, err))
|
||||
}
|
||||
committed := false
|
||||
defer func() {
|
||||
if !committed {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
|
||||
dr := NewDumpReader(r)
|
||||
chunk := &row.Chunk{}
|
||||
var affected int64
|
||||
flush := func() error {
|
||||
if len(chunk.Rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
n, err := w.sink.InsertChunk(ctx, tx, chunk)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: %w", e.Seq, err))
|
||||
}
|
||||
affected += n
|
||||
chunk = &row.Chunk{}
|
||||
return w.checkWarnings(ctx, e.Seq)
|
||||
}
|
||||
|
||||
for {
|
||||
rw, err := dr.ReadRow()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return affected, 0, archererr.Wrap(archererr.KindValidation, fmt.Errorf("shard %d: decode: %w", e.Seq, err))
|
||||
}
|
||||
if len(rw.Values) != len(w.mapping.FileColumns) {
|
||||
return affected, 0, archererr.Newf(archererr.KindValidation,
|
||||
"shard %d: row has %d fields but the mapping expects %d", e.Seq, len(rw.Values), len(w.mapping.FileColumns))
|
||||
}
|
||||
chunk.Rows = append(chunk.Rows, rw)
|
||||
if len(chunk.Rows) >= insertBatchRows {
|
||||
if err := flush(); err != nil {
|
||||
return affected, 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := flush(); err != nil {
|
||||
return affected, 0, err
|
||||
}
|
||||
skipped, err := CheckRowRule(w.cfg.Mode, e.Rows, affected)
|
||||
if err != nil {
|
||||
return affected, 0, fmt.Errorf("shard %d: %w", e.Seq, err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return affected, 0, archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: commit: %w", e.Seq, err))
|
||||
}
|
||||
committed = true
|
||||
return affected, skipped, nil
|
||||
}
|
||||
|
||||
func (w *worker) checkWarnings(ctx context.Context, seq int64) error {
|
||||
warnings, err := ShowWarnings(ctx, w.conn)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: SHOW WARNINGS: %w", seq, err))
|
||||
}
|
||||
return CheckWarnings(warnings, w.cfg.AllowWarnings, w.cfg.Mode, seq)
|
||||
}
|
||||
|
||||
var readerCounter atomic.Uint64
|
||||
|
||||
func newReaderName(seq int64) string {
|
||||
return fmt.Sprintf("archer-import-%d-%d-%d", os.Getpid(), seq, readerCounter.Add(1))
|
||||
}
|
||||
|
||||
type errReader struct{ err error }
|
||||
|
||||
func (e errReader) Read([]byte) (int, error) { return 0, e.err }
|
||||
|
||||
type readCloserPair struct {
|
||||
r io.ReadCloser
|
||||
f *os.File
|
||||
}
|
||||
|
||||
func (p *readCloserPair) Read(b []byte) (int, error) { return p.r.Read(b) }
|
||||
|
||||
func (p *readCloserPair) Close() error {
|
||||
err := p.r.Close()
|
||||
if ferr := p.f.Close(); err == nil {
|
||||
err = ferr
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func openShardReader(path string, comp *shard.Compressor) io.Reader {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return errReader{err}
|
||||
}
|
||||
r, err := comp.Reader(f)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return errReader{err}
|
||||
}
|
||||
return &readCloserPair{r: r, f: f}
|
||||
}
|
||||
Reference in new issue
Block a user