Files
voidandClaude Sonnet 4.6 366d61ca25 Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-11-08 21:14:23 +08:00

132 lines
3.8 KiB
Go

package shard
import (
"strings"
"testing"
)
func sampleEntries() []Entry {
return []Entry{
{Seq: 1, Path: "000001.1-100.dump", PKLo: "1", PKHi: "100", Rows: 100},
{Seq: 2, Path: "000002.101-200.dump", PKLo: "101", PKHi: "200", Rows: 100},
{Seq: 3, Path: "000003.201-300.dump", PKLo: "201", PKHi: "300", Rows: 100},
{Seq: 4, Path: "000004.301-400.dump", PKLo: "301", PKHi: "400", Rows: 100},
}
}
func mustSelect(t *testing.T, entries []Entry, specs ...string) ([]Entry, []string) {
t.Helper()
sels, err := ParseSelection(specs)
if err != nil {
t.Fatal(err)
}
out, warnings, err := ApplySelection(entries, sels)
if err != nil {
t.Fatal(err)
}
return out, warnings
}
func seqs(entries []Entry) []int64 {
out := make([]int64, len(entries))
for i, e := range entries {
out[i] = e.Seq
}
return out
}
func TestSelectAllByDefault(t *testing.T) {
got, _ := mustSelect(t, sampleEntries())
if len(got) != 4 {
t.Fatalf("got %d", len(got))
}
}
func TestSelectSeqRange(t *testing.T) {
got, _ := mustSelect(t, sampleEntries(), "seq:2-3")
if len(got) != 2 || got[0].Seq != 2 || got[1].Seq != 3 {
t.Fatalf("got %v", seqs(got))
}
}
func TestSelectPKIntersection(t *testing.T) {
got, warnings := mustSelect(t, sampleEntries(), "pk:150-250")
if len(got) != 2 || got[0].Seq != 2 || got[1].Seq != 3 {
t.Fatalf("got %v", seqs(got))
}
if len(warnings) != 2 {
t.Fatalf("expected boundary warnings, got %v", warnings)
}
if !strings.Contains(warnings[0], "150") {
t.Fatalf("warning should name the requested bound: %v", warnings)
}
}
func TestSelectPKContainedNoWarning(t *testing.T) {
got, warnings := mustSelect(t, sampleEntries(), "pk:101-200")
if len(got) != 1 || got[0].Seq != 2 {
t.Fatalf("got %v", seqs(got))
}
if len(warnings) != 0 {
t.Fatalf("unexpected warnings: %v", warnings)
}
}
func TestSelectUnionDedupes(t *testing.T) {
got, _ := mustSelect(t, sampleEntries(), "seq:1-2", "seq:2-3")
if len(got) != 3 || got[0].Seq != 1 || got[1].Seq != 2 || got[2].Seq != 3 {
t.Fatalf("got %v", seqs(got))
}
}
func TestSelectEmptyIsError(t *testing.T) {
sels, _ := ParseSelection([]string{"seq:9-10"})
if _, _, err := ApplySelection(sampleEntries(), sels); err == nil || !strings.Contains(err.Error(), "no shards matched") {
t.Fatalf("got %v", err)
}
}
func TestSelectDateOnNonDateArchive(t *testing.T) {
sels, _ := ParseSelection([]string{"date:2025-01-01..2025-01-31"})
if _, _, err := ApplySelection(sampleEntries(), sels); err == nil || !strings.Contains(err.Error(), "not exported with --shard-by date") {
t.Fatalf("got %v", err)
}
}
func TestSelectDateBucket(t *testing.T) {
entries := []Entry{
{Seq: 1, Path: "date=2025-01-01/000001.1-1.dump", Date: "2025-01-01", PKLo: "1", PKHi: "1"},
{Seq: 1, Path: "date=2025-01-02/000001.2-2.dump", Date: "2025-01-02", PKLo: "2", PKHi: "2"},
{Seq: 2, Path: "date=2025-01-02/000002.3-3.dump", Date: "2025-01-02", PKLo: "3", PKHi: "3"},
{Seq: 1, Path: "date=2025-01-03/000001.4-4.dump", Date: "2025-01-03", PKLo: "4", PKHi: "4"},
}
got, _ := mustSelect(t, entries, "date:2025-01-02..2025-01-02")
if len(got) != 2 {
t.Fatalf("got %d entries", len(got))
}
// Seq repeats across buckets; dedupe must not collapse them.
if got[0].Path != entries[1].Path || got[1].Path != entries[2].Path {
t.Fatalf("wrong entries: %+v", got)
}
}
func TestParseSelectorErrors(t *testing.T) {
for _, s := range []string{"", "seq", "seq:1", "seq:a-b", "seq:3-1", "pk:1", "date:2025-01-01", "date:2025-01-02..2025-01-01", "nope:1"} {
if _, err := ParseSelector(s); err == nil {
t.Fatalf("%q: expected an error", s)
}
}
}
func TestCompareKeyText(t *testing.T) {
if CompareKeyText("9", "10") >= 0 {
t.Fatal("numeric keys must sort numerically")
}
if CompareKeyText("a", "b") >= 0 {
t.Fatal("non-numeric keys must sort lexically")
}
if CompareKeyText("5", "5") != 0 {
t.Fatal("equal keys must compare equal")
}
}