Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4a6b504f
commit
366d61ca25
49 files changed
+5099
-342
No files matched your search
@@ -0,0 +1,93 @@
|
||||
package purge
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
type recordingExecer struct {
|
||||
query string
|
||||
args []any
|
||||
}
|
||||
|
||||
func (e *recordingExecer) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
|
||||
e.query = query
|
||||
e.args = args
|
||||
return fakeResult(2), nil
|
||||
}
|
||||
|
||||
type fakeResult int64
|
||||
|
||||
func (r fakeResult) LastInsertId() (int64, error) { return 0, nil }
|
||||
func (r fakeResult) RowsAffected() (int64, error) { return int64(r), nil }
|
||||
|
||||
func TestDeleteRangeUsesBareKeys(t *testing.T) {
|
||||
p := New(nil, "db", "t", []string{"id"}, "v > 0", true, false, true)
|
||||
c := &row.Chunk{
|
||||
Keys: []row.Key{row.KeyOf("1001"), row.KeyOf("3000")},
|
||||
// Date scanners set LastKey to a (shard column, key...) cursor.
|
||||
LastKey: row.KeyOf("2025-01-01 00:00:00", "3000"),
|
||||
}
|
||||
exec := &recordingExecer{}
|
||||
|
||||
n, err := p.DeleteChunk(context.Background(), exec, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("affected = %d", n)
|
||||
}
|
||||
want := "DELETE FROM `db`.`t` WHERE `id` BETWEEN ? AND ? AND (v > 0)"
|
||||
if exec.query != want {
|
||||
t.Fatalf("query:\n got %s\nwant %s", exec.query, want)
|
||||
}
|
||||
if !reflect.DeepEqual(exec.args, []any{"1001", "3000"}) {
|
||||
t.Fatalf("args = %#v", exec.args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteRangeExcludesMaxAutoIncrementKey(t *testing.T) {
|
||||
p := New(nil, "db", "t", []string{"id"}, "1=1", true, true, true)
|
||||
c := &row.Chunk{Keys: []row.Key{row.KeyOf("1001"), row.KeyOf("3000")}}
|
||||
exec := &recordingExecer{}
|
||||
|
||||
maxKey := &row.KeyPart{Val: "3000"}
|
||||
if _, err := p.deleteRange(context.Background(), exec, c, maxKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "DELETE FROM `db`.`t` WHERE `id` BETWEEN ? AND ? AND `id` <> ? AND (1=1)"
|
||||
if exec.query != want {
|
||||
t.Fatalf("query:\n got %s\nwant %s", exec.query, want)
|
||||
}
|
||||
if !reflect.DeepEqual(exec.args, []any{"1001", "3000", "3000"}) {
|
||||
t.Fatalf("args = %#v", exec.args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteRangeCompositeKeyUsesBareKeys(t *testing.T) {
|
||||
p := New(nil, "db", "t", []string{"created_at", "id"}, "1=1", true, false, false)
|
||||
c := &row.Chunk{
|
||||
Keys: []row.Key{
|
||||
row.KeyOf("2025-01-01 00:00:00", "1001"),
|
||||
row.KeyOf("2025-01-02 00:00:00", "3000"),
|
||||
},
|
||||
LastKey: row.KeyOf("2025-01-02 00:00:00", "2025-01-02 00:00:00", "3000"),
|
||||
}
|
||||
exec := &recordingExecer{}
|
||||
|
||||
if _, err := p.DeleteChunk(context.Background(), exec, c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "DELETE FROM `db`.`t` WHERE (`created_at`, `id`) >= (?, ?) AND (`created_at`, `id`) <= (?, ?) AND (1=1)"
|
||||
if exec.query != want {
|
||||
t.Fatalf("query:\n got %s\nwant %s", exec.query, want)
|
||||
}
|
||||
wantArgs := []any{"2025-01-01 00:00:00", "1001", "2025-01-02 00:00:00", "3000"}
|
||||
if !reflect.DeepEqual(exec.args, wantArgs) {
|
||||
t.Fatalf("args = %#v", exec.args)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user