Add archer import and allow date-mode bulk delete

archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-08 21:14:23 +08:00
1 parent bc4a6b504f
commit 366d61ca25
49 files changed
+5099 -342

No files matched your search

+262
View File
@@ -0,0 +1,262 @@
package shard
import (
"fmt"
"sort"
"strconv"
"strings"
"time"
"archer/internal/schema"
)
// Selector is one parsed --select condition. The zero value, Kind "all", holds
// only the raw spelling.
type Selector struct {
Kind string // all|seq|pk|date
Raw string
SeqLo int64
SeqHi int64
Lo string
Hi string
DateLo string
DateHi string
}
const dateLayout = "2006-01-02"
// ParseSelector parses one --select value.
func ParseSelector(s string) (Selector, error) {
raw := strings.TrimSpace(s)
if raw == "" {
return Selector{}, fmt.Errorf("--select cannot be empty")
}
if raw == "all" {
return Selector{Kind: "all", Raw: raw}, nil
}
kind, rest, ok := strings.Cut(raw, ":")
if !ok {
return Selector{}, fmt.Errorf("invalid --select %q: expected all, seq:A-B, pk:LO-HI or date:D1..D2", raw)
}
switch kind {
case "seq":
lo, hi, err := parseSeqRange(rest)
if err != nil {
return Selector{}, fmt.Errorf("invalid --select %q: %v", raw, err)
}
return Selector{Kind: "seq", Raw: raw, SeqLo: lo, SeqHi: hi}, nil
case "pk":
lo, hi, ok := strings.Cut(rest, "-")
lo = strings.TrimSpace(lo)
hi = strings.TrimSpace(hi)
if !ok || lo == "" || hi == "" {
return Selector{}, fmt.Errorf("invalid --select %q: expected pk:LO-HI", raw)
}
return Selector{Kind: "pk", Raw: raw, Lo: lo, Hi: hi}, nil
case "date":
lo, hi, ok := strings.Cut(rest, "..")
lo, hi = strings.TrimSpace(lo), strings.TrimSpace(hi)
if !ok || lo == "" || hi == "" {
return Selector{}, fmt.Errorf("invalid --select %q: expected date:D1..D2", raw)
}
if _, err := time.Parse(dateLayout, lo); err != nil {
return Selector{}, fmt.Errorf("invalid --select %q: %v", raw, err)
}
if _, err := time.Parse(dateLayout, hi); err != nil {
return Selector{}, fmt.Errorf("invalid --select %q: %v", raw, err)
}
if lo > hi {
return Selector{}, fmt.Errorf("invalid --select %q: date range is reversed", raw)
}
return Selector{Kind: "date", Raw: raw, DateLo: lo, DateHi: hi}, nil
default:
return Selector{}, fmt.Errorf("invalid --select %q: unknown selector %q", raw, kind)
}
}
func parseSeqRange(s string) (int64, int64, error) {
lo, hi, ok := strings.Cut(s, "-")
if !ok {
return 0, 0, fmt.Errorf("expected A-B")
}
a, err := strconv.ParseInt(strings.TrimSpace(lo), 10, 64)
if err != nil {
return 0, 0, fmt.Errorf("bad range start %q", lo)
}
b, err := strconv.ParseInt(strings.TrimSpace(hi), 10, 64)
if err != nil {
return 0, 0, fmt.Errorf("bad range end %q", hi)
}
if a < 0 || b < 0 {
return 0, 0, fmt.Errorf("shard sequence numbers cannot be negative")
}
if a > b {
return 0, 0, fmt.Errorf("range is reversed")
}
return a, b, nil
}
// ParseSelection parses every --select value. An empty list means "all".
func ParseSelection(specs []string) ([]Selector, error) {
if len(specs) == 0 {
return []Selector{{Kind: "all", Raw: "all"}}, nil
}
out := make([]Selector, 0, len(specs))
for _, s := range specs {
sel, err := ParseSelector(s)
if err != nil {
return nil, err
}
out = append(out, sel)
}
return out, nil
}
// ApplySelection returns the deduplicated, ordered entries matched by sels.
// Order follows the archive: bucket order (NULL, zero dates, real dates) and
// then ascending seq, which is also the order a serial import should use.
// Warnings describe boundary shards whose whole contents will be imported even
// though only part of their key range was requested.
func ApplySelection(entries []Entry, sels []Selector) ([]Entry, []string, error) {
if len(sels) == 0 {
sels = []Selector{{Kind: "all", Raw: "all"}}
}
hasDate := false
hasAll := false
var pkSelectors []Selector
for _, s := range sels {
switch s.Kind {
case "date":
for _, e := range entries {
if e.Date != "" {
hasDate = true
break
}
}
case "all":
hasAll = true
case "pk":
pkSelectors = append(pkSelectors, s)
}
}
for _, s := range sels {
if s.Kind == "date" && !hasDate {
return nil, nil, fmt.Errorf("archive has no date buckets (it was not exported with --shard-by date); --select date:%s..%s cannot be used", s.DateLo, s.DateHi)
}
}
seen := map[string]bool{}
var selected []Entry
var warnings []string
for _, e := range entries {
matched, pkFunded := matchEntry(e, sels)
if !matched {
continue
}
key := e.Date + "\x00" + strconv.FormatInt(e.Seq, 10)
if seen[key] {
continue
}
seen[key] = true
selected = append(selected, e)
if pkFunded && !hasAll {
if w, ok := pkBoundaryWarning(e, pkSelectors); ok {
warnings = append(warnings, w)
}
}
}
if len(selected) == 0 {
return nil, nil, fmt.Errorf("no shards matched the given --select conditions")
}
sort.Slice(selected, func(i, j int) bool {
if selected[i].Date != selected[j].Date {
return schema.BucketLess(selected[i].Date, selected[j].Date)
}
if selected[i].Seq != selected[j].Seq {
return selected[i].Seq < selected[j].Seq
}
return selected[i].Path < selected[j].Path
})
return selected, warnings, nil
}
func matchEntry(e Entry, sels []Selector) (matched bool, pk bool) {
for _, s := range sels {
switch s.Kind {
case "all":
return true, false
case "seq":
if e.Seq >= s.SeqLo && e.Seq <= s.SeqHi {
return true, false
}
case "date":
if e.Date != "" && e.Date >= s.DateLo && e.Date <= s.DateHi {
return true, false
}
case "pk":
if pkOverlap(e, s) {
return true, true
}
}
}
return false, false
}
func pkOverlap(e Entry, s Selector) bool {
if e.PKLo == "" || e.PKHi == "" {
return false
}
return CompareKeyText(e.PKHi, s.Lo) >= 0 && CompareKeyText(e.PKLo, s.Hi) <= 0
}
func containedInAnyPK(e Entry, sels []Selector) bool {
for _, s := range sels {
if CompareKeyText(e.PKLo, s.Lo) >= 0 && CompareKeyText(e.PKHi, s.Hi) <= 0 {
return true
}
}
return false
}
func pkBoundaryWarning(e Entry, sels []Selector) (string, bool) {
if containedInAnyPK(e, sels) {
return "", false
}
for _, s := range sels {
if !pkOverlap(e, s) {
continue
}
return fmt.Sprintf(
"shard %d covers pk [%s, %s]; requested %s overlaps it, so rows outside the requested range will be imported too",
e.Seq, emptyAs(e.PKLo, "?"), emptyAs(e.PKHi, "?"), s.Raw), true
}
return "", false
}
// CompareKeyText compares two key fragments, using numeric order when both are
// integers and lexical order otherwise.
func CompareKeyText(a, b string) int {
ai, aErr := strconv.ParseInt(a, 10, 64)
bi, bErr := strconv.ParseInt(b, 10, 64)
if aErr == nil && bErr == nil {
switch {
case ai < bi:
return -1
case ai > bi:
return 1
default:
return 0
}
}
return strings.Compare(a, b)
}
func emptyAs(s, fallback string) string {
if s == "" {
return fallback
}
return s
}