archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
132 lines
3.8 KiB
Go
132 lines
3.8 KiB
Go
package shard
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func sampleEntries() []Entry {
|
|
return []Entry{
|
|
{Seq: 1, Path: "000001.1-100.dump", PKLo: "1", PKHi: "100", Rows: 100},
|
|
{Seq: 2, Path: "000002.101-200.dump", PKLo: "101", PKHi: "200", Rows: 100},
|
|
{Seq: 3, Path: "000003.201-300.dump", PKLo: "201", PKHi: "300", Rows: 100},
|
|
{Seq: 4, Path: "000004.301-400.dump", PKLo: "301", PKHi: "400", Rows: 100},
|
|
}
|
|
}
|
|
|
|
func mustSelect(t *testing.T, entries []Entry, specs ...string) ([]Entry, []string) {
|
|
t.Helper()
|
|
sels, err := ParseSelection(specs)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
out, warnings, err := ApplySelection(entries, sels)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return out, warnings
|
|
}
|
|
|
|
func seqs(entries []Entry) []int64 {
|
|
out := make([]int64, len(entries))
|
|
for i, e := range entries {
|
|
out[i] = e.Seq
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestSelectAllByDefault(t *testing.T) {
|
|
got, _ := mustSelect(t, sampleEntries())
|
|
if len(got) != 4 {
|
|
t.Fatalf("got %d", len(got))
|
|
}
|
|
}
|
|
|
|
func TestSelectSeqRange(t *testing.T) {
|
|
got, _ := mustSelect(t, sampleEntries(), "seq:2-3")
|
|
if len(got) != 2 || got[0].Seq != 2 || got[1].Seq != 3 {
|
|
t.Fatalf("got %v", seqs(got))
|
|
}
|
|
}
|
|
|
|
func TestSelectPKIntersection(t *testing.T) {
|
|
got, warnings := mustSelect(t, sampleEntries(), "pk:150-250")
|
|
if len(got) != 2 || got[0].Seq != 2 || got[1].Seq != 3 {
|
|
t.Fatalf("got %v", seqs(got))
|
|
}
|
|
if len(warnings) != 2 {
|
|
t.Fatalf("expected boundary warnings, got %v", warnings)
|
|
}
|
|
if !strings.Contains(warnings[0], "150") {
|
|
t.Fatalf("warning should name the requested bound: %v", warnings)
|
|
}
|
|
}
|
|
|
|
func TestSelectPKContainedNoWarning(t *testing.T) {
|
|
got, warnings := mustSelect(t, sampleEntries(), "pk:101-200")
|
|
if len(got) != 1 || got[0].Seq != 2 {
|
|
t.Fatalf("got %v", seqs(got))
|
|
}
|
|
if len(warnings) != 0 {
|
|
t.Fatalf("unexpected warnings: %v", warnings)
|
|
}
|
|
}
|
|
|
|
func TestSelectUnionDedupes(t *testing.T) {
|
|
got, _ := mustSelect(t, sampleEntries(), "seq:1-2", "seq:2-3")
|
|
if len(got) != 3 || got[0].Seq != 1 || got[1].Seq != 2 || got[2].Seq != 3 {
|
|
t.Fatalf("got %v", seqs(got))
|
|
}
|
|
}
|
|
|
|
func TestSelectEmptyIsError(t *testing.T) {
|
|
sels, _ := ParseSelection([]string{"seq:9-10"})
|
|
if _, _, err := ApplySelection(sampleEntries(), sels); err == nil || !strings.Contains(err.Error(), "no shards matched") {
|
|
t.Fatalf("got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestSelectDateOnNonDateArchive(t *testing.T) {
|
|
sels, _ := ParseSelection([]string{"date:2025-01-01..2025-01-31"})
|
|
if _, _, err := ApplySelection(sampleEntries(), sels); err == nil || !strings.Contains(err.Error(), "not exported with --shard-by date") {
|
|
t.Fatalf("got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestSelectDateBucket(t *testing.T) {
|
|
entries := []Entry{
|
|
{Seq: 1, Path: "date=2025-01-01/000001.1-1.dump", Date: "2025-01-01", PKLo: "1", PKHi: "1"},
|
|
{Seq: 1, Path: "date=2025-01-02/000001.2-2.dump", Date: "2025-01-02", PKLo: "2", PKHi: "2"},
|
|
{Seq: 2, Path: "date=2025-01-02/000002.3-3.dump", Date: "2025-01-02", PKLo: "3", PKHi: "3"},
|
|
{Seq: 1, Path: "date=2025-01-03/000001.4-4.dump", Date: "2025-01-03", PKLo: "4", PKHi: "4"},
|
|
}
|
|
got, _ := mustSelect(t, entries, "date:2025-01-02..2025-01-02")
|
|
if len(got) != 2 {
|
|
t.Fatalf("got %d entries", len(got))
|
|
}
|
|
// Seq repeats across buckets; dedupe must not collapse them.
|
|
if got[0].Path != entries[1].Path || got[1].Path != entries[2].Path {
|
|
t.Fatalf("wrong entries: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestParseSelectorErrors(t *testing.T) {
|
|
for _, s := range []string{"", "seq", "seq:1", "seq:a-b", "seq:3-1", "pk:1", "date:2025-01-01", "date:2025-01-02..2025-01-01", "nope:1"} {
|
|
if _, err := ParseSelector(s); err == nil {
|
|
t.Fatalf("%q: expected an error", s)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCompareKeyText(t *testing.T) {
|
|
if CompareKeyText("9", "10") >= 0 {
|
|
t.Fatal("numeric keys must sort numerically")
|
|
}
|
|
if CompareKeyText("a", "b") >= 0 {
|
|
t.Fatal("non-numeric keys must sort lexically")
|
|
}
|
|
if CompareKeyText("5", "5") != 0 {
|
|
t.Fatal("equal keys must compare equal")
|
|
}
|
|
}
|