Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4a6b504f
commit
366d61ca25
49 files changed
+5099
-342
No files matched your search
@@ -366,6 +366,9 @@ func (a *App) execute(ctx context.Context) error {
|
||||
if a.cfg.BulkDelete {
|
||||
a.message("warning: --bulk-delete removes all rows in the scanned key range that match --where, including concurrent inserts")
|
||||
}
|
||||
if a.cfg.BulkDelete && a.cfg.IsDateShard() {
|
||||
a.message("warning: date mode --bulk-delete assumes the shard column never moves backwards relative to the scan key; a late or backfilled row can be deleted before it is archived")
|
||||
}
|
||||
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
|
||||
@@ -882,6 +882,57 @@ func TestIntegrationPurgeAndDest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationDateBulkDelete(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
|
||||
seed := func(table string) {
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS "+table)
|
||||
execSQL(t, db, `CREATE TABLE `+table+` (
|
||||
id BIGINT NOT NULL AUTO_INCREMENT,
|
||||
v VARCHAR(32),
|
||||
created_at DATETIME NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
KEY d (created_at, id))`)
|
||||
execSQL(t, db, `INSERT INTO `+table+` (id, v, created_at) VALUES
|
||||
(1,'a','2025-01-01 00:00:00'),
|
||||
(2,'b','2025-01-02 00:00:00'),
|
||||
(3,'c','2025-01-03 00:00:00'),
|
||||
(4,'d','2025-01-04 00:00:00')`)
|
||||
}
|
||||
|
||||
seed("it_date_bulk")
|
||||
out := t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_bulk", out,
|
||||
"--shard-by", "date", "--shard-column", "created_at",
|
||||
"--purge", "--bulk-delete", "--safe-auto-increment=false", "--chunk-size", "2")...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var count int
|
||||
db.QueryRow("SELECT COUNT(*) FROM it_date_bulk").Scan(&count)
|
||||
if count != 0 {
|
||||
t.Fatalf("safe off: %d rows left", count)
|
||||
}
|
||||
if got := len(readTree(t, out)); got != 4 {
|
||||
t.Fatalf("expected 4 shards, got %d", got)
|
||||
}
|
||||
|
||||
seed("it_date_bulk_safe")
|
||||
out = t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_bulk_safe", out,
|
||||
"--shard-by", "date", "--shard-column", "created_at",
|
||||
"--purge", "--bulk-delete", "--chunk-size", "2")...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.QueryRow("SELECT COUNT(*) FROM it_date_bulk_safe").Scan(&count)
|
||||
var maxID int64
|
||||
db.QueryRow("SELECT MAX(id) FROM it_date_bulk_safe").Scan(&maxID)
|
||||
if count != 1 || maxID != 4 {
|
||||
t.Fatalf("safe on: count=%d max=%d", count, maxID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationCheckpointConfigMismatch(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
|
||||
Reference in new issue
Block a user