Add archer import and allow date-mode bulk delete

archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-08 21:14:23 +08:00
1 parent bc4a6b504f
commit 366d61ca25
49 files changed
+5099 -342

No files matched your search

+21 -7
View File
@@ -63,8 +63,8 @@ func (p *Purger) DeleteChunk(ctx context.Context, exec Execer, c *row.Chunk) (in
maxKey = v
}
if p.Bulk && maxKey == nil {
return p.deleteRange(ctx, exec, c)
if p.Bulk {
return p.deleteRange(ctx, exec, c, maxKey)
}
return p.deleteInList(ctx, exec, keys, maxKey)
}
@@ -85,8 +85,13 @@ func (p *Purger) currentMax(ctx context.Context) (*row.KeyPart, error) {
return &row.KeyPart{Val: v.String}, nil
}
func (p *Purger) deleteRange(ctx context.Context, exec Execer, c *row.Chunk) (int64, error) {
lo, hi := c.FirstKey, c.LastKey
func (p *Purger) deleteRange(ctx context.Context, exec Execer, c *row.Chunk, maxKey *row.KeyPart) (int64, error) {
// Date scanners rewrite LastKey into a (shard column, key...) cursor for
// resumption, so the bare key range must come from Keys.
if len(c.Keys) == 0 {
return 0, nil
}
lo, hi := c.Keys[0], c.Keys[len(c.Keys)-1]
if lo.IsZero() || hi.IsZero() {
return 0, nil
}
@@ -95,9 +100,18 @@ func (p *Purger) deleteRange(ctx context.Context, exec Execer, c *row.Chunk) (in
args []any
)
if len(p.KeyCols) == 1 {
q = fmt.Sprintf("DELETE FROM %s WHERE %s BETWEEN ? AND ? AND (%s)",
p.table(), sqlutil.Ident(p.KeyCols[0]), p.Where)
args = []any{lo[0].DriverValue(), hi[0].DriverValue()}
col := sqlutil.Ident(p.KeyCols[0])
if maxKey != nil {
// Keep safe-auto-increment semantics: the current max row is
// excluded even when it falls inside the range.
q = fmt.Sprintf("DELETE FROM %s WHERE %s BETWEEN ? AND ? AND %s <> ? AND (%s)",
p.table(), col, col, p.Where)
args = []any{lo[0].DriverValue(), hi[0].DriverValue(), maxKey.DriverValue()}
} else {
q = fmt.Sprintf("DELETE FROM %s WHERE %s BETWEEN ? AND ? AND (%s)",
p.table(), col, p.Where)
args = []any{lo[0].DriverValue(), hi[0].DriverValue()}
}
} else {
q = fmt.Sprintf("DELETE FROM %s WHERE %s >= %s AND %s <= %s AND (%s)",
p.table(), sqlutil.RowTuple(p.KeyCols), sqlutil.ParamTuple(len(p.KeyCols)),
+93
View File
@@ -0,0 +1,93 @@
package purge
import (
"context"
"database/sql"
"reflect"
"testing"
"archer/internal/row"
)
type recordingExecer struct {
query string
args []any
}
func (e *recordingExecer) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
e.query = query
e.args = args
return fakeResult(2), nil
}
type fakeResult int64
func (r fakeResult) LastInsertId() (int64, error) { return 0, nil }
func (r fakeResult) RowsAffected() (int64, error) { return int64(r), nil }
func TestDeleteRangeUsesBareKeys(t *testing.T) {
p := New(nil, "db", "t", []string{"id"}, "v > 0", true, false, true)
c := &row.Chunk{
Keys: []row.Key{row.KeyOf("1001"), row.KeyOf("3000")},
// Date scanners set LastKey to a (shard column, key...) cursor.
LastKey: row.KeyOf("2025-01-01 00:00:00", "3000"),
}
exec := &recordingExecer{}
n, err := p.DeleteChunk(context.Background(), exec, c)
if err != nil {
t.Fatal(err)
}
if n != 2 {
t.Fatalf("affected = %d", n)
}
want := "DELETE FROM `db`.`t` WHERE `id` BETWEEN ? AND ? AND (v > 0)"
if exec.query != want {
t.Fatalf("query:\n got %s\nwant %s", exec.query, want)
}
if !reflect.DeepEqual(exec.args, []any{"1001", "3000"}) {
t.Fatalf("args = %#v", exec.args)
}
}
func TestDeleteRangeExcludesMaxAutoIncrementKey(t *testing.T) {
p := New(nil, "db", "t", []string{"id"}, "1=1", true, true, true)
c := &row.Chunk{Keys: []row.Key{row.KeyOf("1001"), row.KeyOf("3000")}}
exec := &recordingExecer{}
maxKey := &row.KeyPart{Val: "3000"}
if _, err := p.deleteRange(context.Background(), exec, c, maxKey); err != nil {
t.Fatal(err)
}
want := "DELETE FROM `db`.`t` WHERE `id` BETWEEN ? AND ? AND `id` <> ? AND (1=1)"
if exec.query != want {
t.Fatalf("query:\n got %s\nwant %s", exec.query, want)
}
if !reflect.DeepEqual(exec.args, []any{"1001", "3000", "3000"}) {
t.Fatalf("args = %#v", exec.args)
}
}
func TestDeleteRangeCompositeKeyUsesBareKeys(t *testing.T) {
p := New(nil, "db", "t", []string{"created_at", "id"}, "1=1", true, false, false)
c := &row.Chunk{
Keys: []row.Key{
row.KeyOf("2025-01-01 00:00:00", "1001"),
row.KeyOf("2025-01-02 00:00:00", "3000"),
},
LastKey: row.KeyOf("2025-01-02 00:00:00", "2025-01-02 00:00:00", "3000"),
}
exec := &recordingExecer{}
if _, err := p.DeleteChunk(context.Background(), exec, c); err != nil {
t.Fatal(err)
}
want := "DELETE FROM `db`.`t` WHERE (`created_at`, `id`) >= (?, ?) AND (`created_at`, `id`) <= (?, ?) AND (1=1)"
if exec.query != want {
t.Fatalf("query:\n got %s\nwant %s", exec.query, want)
}
wantArgs := []any{"2025-01-01 00:00:00", "1001", "2025-01-02 00:00:00", "3000"}
if !reflect.DeepEqual(exec.args, wantArgs) {
t.Fatalf("args = %#v", exec.args)
}
}