Add archer import and allow date-mode bulk delete

archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-08 21:14:23 +08:00
1 parent bc4a6b504f
commit 366d61ca25
49 files changed
+5099 -342

No files matched your search

+96
View File
@@ -0,0 +1,96 @@
package importer
import (
"bytes"
"io"
"strings"
"testing"
"archer/internal/row"
)
func TestDumpReaderEmptyFieldAndNull(t *testing.T) {
input := "a\t\\N\t\tx\\Ny\n"
dr := NewDumpReader(strings.NewReader(input))
got, err := dr.ReadRow()
if err != nil {
t.Fatal(err)
}
if len(got.Values) != 4 {
t.Fatalf("got %d fields", len(got.Values))
}
if got.Values[0].IsNull() || got.Values[0].String() != "a" {
t.Fatalf("field 0 = %+v", got.Values[0])
}
if !got.Values[1].IsNull() {
t.Fatalf("field 1 should be NULL")
}
if got.Values[2].IsNull() || len(got.Values[2].Raw) != 0 {
t.Fatalf("field 2 should be an empty string, got %+v", got.Values[2])
}
if got.Values[3].String() != "xNy" {
t.Fatalf("\\N inside a field should yield N, got %q", got.Values[3].String())
}
if _, err := dr.ReadRow(); err != io.EOF {
t.Fatalf("expected EOF, got %v", err)
}
}
func TestDumpReaderEscapeTable(t *testing.T) {
input := "a\t\\N\t\t\\t\\n\\r\\0\\b\\Z\\\\\t中文😀\n"
dr := NewDumpReader(strings.NewReader(input))
got, err := dr.ReadRow()
if err != nil {
t.Fatal(err)
}
if len(got.Values) != 5 {
t.Fatalf("got %d fields: %+v", len(got.Values), got.Values)
}
want := []byte{'\t', '\n', '\r', 0, '\b', 0x1a, '\\'}
if !bytes.Equal(got.Values[3].Raw, want) {
t.Fatalf("escapes = %q, want %q", got.Values[3].Raw, want)
}
if got.Values[4].String() != "中文😀" {
t.Fatalf("unicode = %q", got.Values[4].String())
}
}
func TestDumpReaderRoundTripWithEncoder(t *testing.T) {
cols := []row.Column{{Name: "a"}, {Name: "b"}, {Name: "c"}}
enc, err := row.NewEncoder("dump", cols, []int{0, 1, 2}, row.Options{})
if err != nil {
t.Fatal(err)
}
rows := []row.Row{
{Values: []row.Value{{Raw: []byte("")}, {}, {Raw: []byte{0, 1, 2, 255}}}},
{Values: []row.Value{{Raw: []byte("tab\there")}, {Raw: []byte("line\nbreak")}, {Raw: []byte("back\\slash")}}},
{Values: []row.Value{{Raw: []byte("中文")}, {Raw: []byte("😀")}, {Raw: []byte("")}}},
}
var buf bytes.Buffer
for _, r := range rows {
b, err := enc.Encode(nil, r)
if err != nil {
t.Fatal(err)
}
buf.Write(b)
}
dr := NewDumpReader(&buf)
for i, want := range rows {
got, err := dr.ReadRow()
if err != nil {
t.Fatalf("row %d: %v", i, err)
}
if len(got.Values) != len(want.Values) {
t.Fatalf("row %d: %d fields", i, len(got.Values))
}
for j := range want.Values {
w, g := want.Values[j], got.Values[j]
if w.IsNull() != g.IsNull() || !bytes.Equal(w.Raw, g.Raw) {
t.Fatalf("row %d field %d: got %+v want %+v", i, j, g, w)
}
}
}
if _, err := dr.ReadRow(); err != io.EOF {
t.Fatalf("expected EOF, got %v", err)
}
}