Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4a6b504f
commit
366d61ca25
49 files changed
+5099
-342
No files matched your search
@@ -28,6 +28,43 @@ type Entry struct {
|
||||
ClosedAt string `json:"closed_at"`
|
||||
}
|
||||
|
||||
// LoadManifest reads manifest.jsonl strictly for replay. Unlike OpenManifest it
|
||||
// never creates or appends to the archive, and a malformed line is an error
|
||||
// rather than a line to skip: an importer must not mistake corruption for a
|
||||
// missing shard.
|
||||
func LoadManifest(root string) ([]Entry, error) {
|
||||
path := filepath.Join(root, manifestName)
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open manifest: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var entries []Entry
|
||||
sc := bufio.NewScanner(f)
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
|
||||
line := 0
|
||||
for sc.Scan() {
|
||||
line++
|
||||
text := strings.TrimSpace(sc.Text())
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
var e Entry
|
||||
if err := json.Unmarshal([]byte(text), &e); err != nil {
|
||||
return nil, fmt.Errorf("parse manifest.jsonl line %d: %w", line, err)
|
||||
}
|
||||
if e.Path == "" {
|
||||
return nil, fmt.Errorf("manifest.jsonl line %d: missing path", line)
|
||||
}
|
||||
entries = append(entries, e)
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return nil, fmt.Errorf("read manifest: %w", err)
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// Manifest appends sealed-shard records to manifest.jsonl and knows which
|
||||
// shards already exist.
|
||||
type Manifest struct {
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archer/internal/schema"
|
||||
)
|
||||
|
||||
// Selector is one parsed --select condition. The zero value, Kind "all", holds
|
||||
// only the raw spelling.
|
||||
type Selector struct {
|
||||
Kind string // all|seq|pk|date
|
||||
Raw string
|
||||
|
||||
SeqLo int64
|
||||
SeqHi int64
|
||||
|
||||
Lo string
|
||||
Hi string
|
||||
|
||||
DateLo string
|
||||
DateHi string
|
||||
}
|
||||
|
||||
const dateLayout = "2006-01-02"
|
||||
|
||||
// ParseSelector parses one --select value.
|
||||
func ParseSelector(s string) (Selector, error) {
|
||||
raw := strings.TrimSpace(s)
|
||||
if raw == "" {
|
||||
return Selector{}, fmt.Errorf("--select cannot be empty")
|
||||
}
|
||||
if raw == "all" {
|
||||
return Selector{Kind: "all", Raw: raw}, nil
|
||||
}
|
||||
kind, rest, ok := strings.Cut(raw, ":")
|
||||
if !ok {
|
||||
return Selector{}, fmt.Errorf("invalid --select %q: expected all, seq:A-B, pk:LO-HI or date:D1..D2", raw)
|
||||
}
|
||||
switch kind {
|
||||
case "seq":
|
||||
lo, hi, err := parseSeqRange(rest)
|
||||
if err != nil {
|
||||
return Selector{}, fmt.Errorf("invalid --select %q: %v", raw, err)
|
||||
}
|
||||
return Selector{Kind: "seq", Raw: raw, SeqLo: lo, SeqHi: hi}, nil
|
||||
case "pk":
|
||||
lo, hi, ok := strings.Cut(rest, "-")
|
||||
lo = strings.TrimSpace(lo)
|
||||
hi = strings.TrimSpace(hi)
|
||||
if !ok || lo == "" || hi == "" {
|
||||
return Selector{}, fmt.Errorf("invalid --select %q: expected pk:LO-HI", raw)
|
||||
}
|
||||
return Selector{Kind: "pk", Raw: raw, Lo: lo, Hi: hi}, nil
|
||||
case "date":
|
||||
lo, hi, ok := strings.Cut(rest, "..")
|
||||
lo, hi = strings.TrimSpace(lo), strings.TrimSpace(hi)
|
||||
if !ok || lo == "" || hi == "" {
|
||||
return Selector{}, fmt.Errorf("invalid --select %q: expected date:D1..D2", raw)
|
||||
}
|
||||
if _, err := time.Parse(dateLayout, lo); err != nil {
|
||||
return Selector{}, fmt.Errorf("invalid --select %q: %v", raw, err)
|
||||
}
|
||||
if _, err := time.Parse(dateLayout, hi); err != nil {
|
||||
return Selector{}, fmt.Errorf("invalid --select %q: %v", raw, err)
|
||||
}
|
||||
if lo > hi {
|
||||
return Selector{}, fmt.Errorf("invalid --select %q: date range is reversed", raw)
|
||||
}
|
||||
return Selector{Kind: "date", Raw: raw, DateLo: lo, DateHi: hi}, nil
|
||||
default:
|
||||
return Selector{}, fmt.Errorf("invalid --select %q: unknown selector %q", raw, kind)
|
||||
}
|
||||
}
|
||||
|
||||
func parseSeqRange(s string) (int64, int64, error) {
|
||||
lo, hi, ok := strings.Cut(s, "-")
|
||||
if !ok {
|
||||
return 0, 0, fmt.Errorf("expected A-B")
|
||||
}
|
||||
a, err := strconv.ParseInt(strings.TrimSpace(lo), 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("bad range start %q", lo)
|
||||
}
|
||||
b, err := strconv.ParseInt(strings.TrimSpace(hi), 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("bad range end %q", hi)
|
||||
}
|
||||
if a < 0 || b < 0 {
|
||||
return 0, 0, fmt.Errorf("shard sequence numbers cannot be negative")
|
||||
}
|
||||
if a > b {
|
||||
return 0, 0, fmt.Errorf("range is reversed")
|
||||
}
|
||||
return a, b, nil
|
||||
}
|
||||
|
||||
// ParseSelection parses every --select value. An empty list means "all".
|
||||
func ParseSelection(specs []string) ([]Selector, error) {
|
||||
if len(specs) == 0 {
|
||||
return []Selector{{Kind: "all", Raw: "all"}}, nil
|
||||
}
|
||||
out := make([]Selector, 0, len(specs))
|
||||
for _, s := range specs {
|
||||
sel, err := ParseSelector(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, sel)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ApplySelection returns the deduplicated, ordered entries matched by sels.
|
||||
// Order follows the archive: bucket order (NULL, zero dates, real dates) and
|
||||
// then ascending seq, which is also the order a serial import should use.
|
||||
// Warnings describe boundary shards whose whole contents will be imported even
|
||||
// though only part of their key range was requested.
|
||||
func ApplySelection(entries []Entry, sels []Selector) ([]Entry, []string, error) {
|
||||
if len(sels) == 0 {
|
||||
sels = []Selector{{Kind: "all", Raw: "all"}}
|
||||
}
|
||||
hasDate := false
|
||||
hasAll := false
|
||||
var pkSelectors []Selector
|
||||
for _, s := range sels {
|
||||
switch s.Kind {
|
||||
case "date":
|
||||
for _, e := range entries {
|
||||
if e.Date != "" {
|
||||
hasDate = true
|
||||
break
|
||||
}
|
||||
}
|
||||
case "all":
|
||||
hasAll = true
|
||||
case "pk":
|
||||
pkSelectors = append(pkSelectors, s)
|
||||
}
|
||||
}
|
||||
for _, s := range sels {
|
||||
if s.Kind == "date" && !hasDate {
|
||||
return nil, nil, fmt.Errorf("archive has no date buckets (it was not exported with --shard-by date); --select date:%s..%s cannot be used", s.DateLo, s.DateHi)
|
||||
}
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
var selected []Entry
|
||||
var warnings []string
|
||||
for _, e := range entries {
|
||||
matched, pkFunded := matchEntry(e, sels)
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
key := e.Date + "\x00" + strconv.FormatInt(e.Seq, 10)
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
selected = append(selected, e)
|
||||
if pkFunded && !hasAll {
|
||||
if w, ok := pkBoundaryWarning(e, pkSelectors); ok {
|
||||
warnings = append(warnings, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(selected) == 0 {
|
||||
return nil, nil, fmt.Errorf("no shards matched the given --select conditions")
|
||||
}
|
||||
sort.Slice(selected, func(i, j int) bool {
|
||||
if selected[i].Date != selected[j].Date {
|
||||
return schema.BucketLess(selected[i].Date, selected[j].Date)
|
||||
}
|
||||
if selected[i].Seq != selected[j].Seq {
|
||||
return selected[i].Seq < selected[j].Seq
|
||||
}
|
||||
return selected[i].Path < selected[j].Path
|
||||
})
|
||||
return selected, warnings, nil
|
||||
}
|
||||
|
||||
func matchEntry(e Entry, sels []Selector) (matched bool, pk bool) {
|
||||
for _, s := range sels {
|
||||
switch s.Kind {
|
||||
case "all":
|
||||
return true, false
|
||||
case "seq":
|
||||
if e.Seq >= s.SeqLo && e.Seq <= s.SeqHi {
|
||||
return true, false
|
||||
}
|
||||
case "date":
|
||||
if e.Date != "" && e.Date >= s.DateLo && e.Date <= s.DateHi {
|
||||
return true, false
|
||||
}
|
||||
case "pk":
|
||||
if pkOverlap(e, s) {
|
||||
return true, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, false
|
||||
}
|
||||
|
||||
func pkOverlap(e Entry, s Selector) bool {
|
||||
if e.PKLo == "" || e.PKHi == "" {
|
||||
return false
|
||||
}
|
||||
return CompareKeyText(e.PKHi, s.Lo) >= 0 && CompareKeyText(e.PKLo, s.Hi) <= 0
|
||||
}
|
||||
|
||||
func containedInAnyPK(e Entry, sels []Selector) bool {
|
||||
for _, s := range sels {
|
||||
if CompareKeyText(e.PKLo, s.Lo) >= 0 && CompareKeyText(e.PKHi, s.Hi) <= 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func pkBoundaryWarning(e Entry, sels []Selector) (string, bool) {
|
||||
if containedInAnyPK(e, sels) {
|
||||
return "", false
|
||||
}
|
||||
for _, s := range sels {
|
||||
if !pkOverlap(e, s) {
|
||||
continue
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"shard %d covers pk [%s, %s]; requested %s overlaps it, so rows outside the requested range will be imported too",
|
||||
e.Seq, emptyAs(e.PKLo, "?"), emptyAs(e.PKHi, "?"), s.Raw), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// CompareKeyText compares two key fragments, using numeric order when both are
|
||||
// integers and lexical order otherwise.
|
||||
func CompareKeyText(a, b string) int {
|
||||
ai, aErr := strconv.ParseInt(a, 10, 64)
|
||||
bi, bErr := strconv.ParseInt(b, 10, 64)
|
||||
if aErr == nil && bErr == nil {
|
||||
switch {
|
||||
case ai < bi:
|
||||
return -1
|
||||
case ai > bi:
|
||||
return 1
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
return strings.Compare(a, b)
|
||||
}
|
||||
|
||||
func emptyAs(s, fallback string) string {
|
||||
if s == "" {
|
||||
return fallback
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func sampleEntries() []Entry {
|
||||
return []Entry{
|
||||
{Seq: 1, Path: "000001.1-100.dump", PKLo: "1", PKHi: "100", Rows: 100},
|
||||
{Seq: 2, Path: "000002.101-200.dump", PKLo: "101", PKHi: "200", Rows: 100},
|
||||
{Seq: 3, Path: "000003.201-300.dump", PKLo: "201", PKHi: "300", Rows: 100},
|
||||
{Seq: 4, Path: "000004.301-400.dump", PKLo: "301", PKHi: "400", Rows: 100},
|
||||
}
|
||||
}
|
||||
|
||||
func mustSelect(t *testing.T, entries []Entry, specs ...string) ([]Entry, []string) {
|
||||
t.Helper()
|
||||
sels, err := ParseSelection(specs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, warnings, err := ApplySelection(entries, sels)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out, warnings
|
||||
}
|
||||
|
||||
func seqs(entries []Entry) []int64 {
|
||||
out := make([]int64, len(entries))
|
||||
for i, e := range entries {
|
||||
out[i] = e.Seq
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestSelectAllByDefault(t *testing.T) {
|
||||
got, _ := mustSelect(t, sampleEntries())
|
||||
if len(got) != 4 {
|
||||
t.Fatalf("got %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSeqRange(t *testing.T) {
|
||||
got, _ := mustSelect(t, sampleEntries(), "seq:2-3")
|
||||
if len(got) != 2 || got[0].Seq != 2 || got[1].Seq != 3 {
|
||||
t.Fatalf("got %v", seqs(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectPKIntersection(t *testing.T) {
|
||||
got, warnings := mustSelect(t, sampleEntries(), "pk:150-250")
|
||||
if len(got) != 2 || got[0].Seq != 2 || got[1].Seq != 3 {
|
||||
t.Fatalf("got %v", seqs(got))
|
||||
}
|
||||
if len(warnings) != 2 {
|
||||
t.Fatalf("expected boundary warnings, got %v", warnings)
|
||||
}
|
||||
if !strings.Contains(warnings[0], "150") {
|
||||
t.Fatalf("warning should name the requested bound: %v", warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectPKContainedNoWarning(t *testing.T) {
|
||||
got, warnings := mustSelect(t, sampleEntries(), "pk:101-200")
|
||||
if len(got) != 1 || got[0].Seq != 2 {
|
||||
t.Fatalf("got %v", seqs(got))
|
||||
}
|
||||
if len(warnings) != 0 {
|
||||
t.Fatalf("unexpected warnings: %v", warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectUnionDedupes(t *testing.T) {
|
||||
got, _ := mustSelect(t, sampleEntries(), "seq:1-2", "seq:2-3")
|
||||
if len(got) != 3 || got[0].Seq != 1 || got[1].Seq != 2 || got[2].Seq != 3 {
|
||||
t.Fatalf("got %v", seqs(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectEmptyIsError(t *testing.T) {
|
||||
sels, _ := ParseSelection([]string{"seq:9-10"})
|
||||
if _, _, err := ApplySelection(sampleEntries(), sels); err == nil || !strings.Contains(err.Error(), "no shards matched") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectDateOnNonDateArchive(t *testing.T) {
|
||||
sels, _ := ParseSelection([]string{"date:2025-01-01..2025-01-31"})
|
||||
if _, _, err := ApplySelection(sampleEntries(), sels); err == nil || !strings.Contains(err.Error(), "not exported with --shard-by date") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectDateBucket(t *testing.T) {
|
||||
entries := []Entry{
|
||||
{Seq: 1, Path: "date=2025-01-01/000001.1-1.dump", Date: "2025-01-01", PKLo: "1", PKHi: "1"},
|
||||
{Seq: 1, Path: "date=2025-01-02/000001.2-2.dump", Date: "2025-01-02", PKLo: "2", PKHi: "2"},
|
||||
{Seq: 2, Path: "date=2025-01-02/000002.3-3.dump", Date: "2025-01-02", PKLo: "3", PKHi: "3"},
|
||||
{Seq: 1, Path: "date=2025-01-03/000001.4-4.dump", Date: "2025-01-03", PKLo: "4", PKHi: "4"},
|
||||
}
|
||||
got, _ := mustSelect(t, entries, "date:2025-01-02..2025-01-02")
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d entries", len(got))
|
||||
}
|
||||
// Seq repeats across buckets; dedupe must not collapse them.
|
||||
if got[0].Path != entries[1].Path || got[1].Path != entries[2].Path {
|
||||
t.Fatalf("wrong entries: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSelectorErrors(t *testing.T) {
|
||||
for _, s := range []string{"", "seq", "seq:1", "seq:a-b", "seq:3-1", "pk:1", "date:2025-01-01", "date:2025-01-02..2025-01-01", "nope:1"} {
|
||||
if _, err := ParseSelector(s); err == nil {
|
||||
t.Fatalf("%q: expected an error", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareKeyText(t *testing.T) {
|
||||
if CompareKeyText("9", "10") >= 0 {
|
||||
t.Fatal("numeric keys must sort numerically")
|
||||
}
|
||||
if CompareKeyText("a", "b") >= 0 {
|
||||
t.Fatal("non-numeric keys must sort lexically")
|
||||
}
|
||||
if CompareKeyText("5", "5") != 0 {
|
||||
t.Fatal("equal keys must compare equal")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user