Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4a6b504f
commit
366d61ca25
49 files changed
+5099
-342
No files matched your search
@@ -0,0 +1,96 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
func TestDumpReaderEmptyFieldAndNull(t *testing.T) {
|
||||
input := "a\t\\N\t\tx\\Ny\n"
|
||||
dr := NewDumpReader(strings.NewReader(input))
|
||||
got, err := dr.ReadRow()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Values) != 4 {
|
||||
t.Fatalf("got %d fields", len(got.Values))
|
||||
}
|
||||
if got.Values[0].IsNull() || got.Values[0].String() != "a" {
|
||||
t.Fatalf("field 0 = %+v", got.Values[0])
|
||||
}
|
||||
if !got.Values[1].IsNull() {
|
||||
t.Fatalf("field 1 should be NULL")
|
||||
}
|
||||
if got.Values[2].IsNull() || len(got.Values[2].Raw) != 0 {
|
||||
t.Fatalf("field 2 should be an empty string, got %+v", got.Values[2])
|
||||
}
|
||||
if got.Values[3].String() != "xNy" {
|
||||
t.Fatalf("\\N inside a field should yield N, got %q", got.Values[3].String())
|
||||
}
|
||||
if _, err := dr.ReadRow(); err != io.EOF {
|
||||
t.Fatalf("expected EOF, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDumpReaderEscapeTable(t *testing.T) {
|
||||
input := "a\t\\N\t\t\\t\\n\\r\\0\\b\\Z\\\\\t中文😀\n"
|
||||
dr := NewDumpReader(strings.NewReader(input))
|
||||
got, err := dr.ReadRow()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Values) != 5 {
|
||||
t.Fatalf("got %d fields: %+v", len(got.Values), got.Values)
|
||||
}
|
||||
want := []byte{'\t', '\n', '\r', 0, '\b', 0x1a, '\\'}
|
||||
if !bytes.Equal(got.Values[3].Raw, want) {
|
||||
t.Fatalf("escapes = %q, want %q", got.Values[3].Raw, want)
|
||||
}
|
||||
if got.Values[4].String() != "中文😀" {
|
||||
t.Fatalf("unicode = %q", got.Values[4].String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDumpReaderRoundTripWithEncoder(t *testing.T) {
|
||||
cols := []row.Column{{Name: "a"}, {Name: "b"}, {Name: "c"}}
|
||||
enc, err := row.NewEncoder("dump", cols, []int{0, 1, 2}, row.Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows := []row.Row{
|
||||
{Values: []row.Value{{Raw: []byte("")}, {}, {Raw: []byte{0, 1, 2, 255}}}},
|
||||
{Values: []row.Value{{Raw: []byte("tab\there")}, {Raw: []byte("line\nbreak")}, {Raw: []byte("back\\slash")}}},
|
||||
{Values: []row.Value{{Raw: []byte("中文")}, {Raw: []byte("😀")}, {Raw: []byte("")}}},
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
for _, r := range rows {
|
||||
b, err := enc.Encode(nil, r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buf.Write(b)
|
||||
}
|
||||
dr := NewDumpReader(&buf)
|
||||
for i, want := range rows {
|
||||
got, err := dr.ReadRow()
|
||||
if err != nil {
|
||||
t.Fatalf("row %d: %v", i, err)
|
||||
}
|
||||
if len(got.Values) != len(want.Values) {
|
||||
t.Fatalf("row %d: %d fields", i, len(got.Values))
|
||||
}
|
||||
for j := range want.Values {
|
||||
w, g := want.Values[j], got.Values[j]
|
||||
if w.IsNull() != g.IsNull() || !bytes.Equal(w.Raw, g.Raw) {
|
||||
t.Fatalf("row %d field %d: got %+v want %+v", i, j, g, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := dr.ReadRow(); err != io.EOF {
|
||||
t.Fatalf("expected EOF, got %v", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user