Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4a6b504f
commit
366d61ca25
49 files changed
+5099
-342
No files matched your search
@@ -0,0 +1,151 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"archer/internal/schema"
|
||||
"archer/internal/shard"
|
||||
)
|
||||
|
||||
func targetColumns() []schema.Column {
|
||||
return []schema.Column{
|
||||
{Name: "id", Ordinal: 1},
|
||||
{Name: "title", Ordinal: 2},
|
||||
{Name: "full_name", Ordinal: 3, Extra: "STORED GENERATED"},
|
||||
{Name: "created_at", Ordinal: 4},
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingByNamePreservesOrder(t *testing.T) {
|
||||
m, err := buildMapping("orders", targetColumns(), []string{"title", "ID"}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(m.FileColumns, ",") != "title,id" {
|
||||
t.Fatalf("file columns = %v", m.FileColumns)
|
||||
}
|
||||
if strings.Join(m.InsertColumns, ",") != "title,id" {
|
||||
t.Fatalf("insert columns = %v", m.InsertColumns)
|
||||
}
|
||||
if strings.Join(m.LoadList, ",") != "`title`,`id`" {
|
||||
t.Fatalf("load list = %v", m.LoadList)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingByNameMissingColumn(t *testing.T) {
|
||||
_, err := buildMapping("orders", targetColumns(), []string{"id", "missing"}, nil, false)
|
||||
if err == nil || !strings.Contains(err.Error(), "missing") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingByNameRejectsGenerated(t *testing.T) {
|
||||
_, err := buildMapping("orders", targetColumns(), []string{"id", "full_name"}, nil, false)
|
||||
if err == nil || !strings.Contains(err.Error(), "generated") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingByNameRejectsDuplicate(t *testing.T) {
|
||||
_, err := buildMapping("orders", targetColumns(), []string{"id", "Id"}, nil, false)
|
||||
if err == nil || !strings.Contains(err.Error(), "more than once") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingPositionalDropsGenerated(t *testing.T) {
|
||||
m, err := buildMapping("orders", targetColumns(), nil, nil, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !m.Positional {
|
||||
t.Fatal("expected positional mapping")
|
||||
}
|
||||
if len(m.FileColumns) != 4 {
|
||||
t.Fatalf("file fields = %v", m.FileColumns)
|
||||
}
|
||||
if m.FileColumns[2] != "" {
|
||||
t.Fatalf("generated field should be empty, got %q", m.FileColumns[2])
|
||||
}
|
||||
if strings.Join(m.InsertColumns, ",") != "id,title,created_at" {
|
||||
t.Fatalf("insert columns = %v", m.InsertColumns)
|
||||
}
|
||||
if len(m.InsertFieldIdx) != 3 || m.InsertFieldIdx[2] != 3 {
|
||||
t.Fatalf("insert indices = %v", m.InsertFieldIdx)
|
||||
}
|
||||
if m.LoadList[2] != "@_archer_skip_3" {
|
||||
t.Fatalf("skip variable = %q", m.LoadList[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingRequiresSource(t *testing.T) {
|
||||
_, err := buildMapping("orders", targetColumns(), nil, nil, false)
|
||||
if err == nil || !strings.Contains(err.Error(), "--assume-all-columns") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingExplicitColumns(t *testing.T) {
|
||||
m, err := buildMapping("orders", targetColumns(), nil, []string{"id", "title"}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(m.FileColumns) != 2 || m.FileColumns[1] != "title" {
|
||||
t.Fatalf("file columns = %v", m.FileColumns)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountFields(t *testing.T) {
|
||||
cases := []struct {
|
||||
line string
|
||||
want int
|
||||
}{
|
||||
{"", 1},
|
||||
{"a", 1},
|
||||
{"a\tb", 2},
|
||||
{"a\tb\tc\t", 4},
|
||||
{`a\tb`, 1}, // escaped tab is not a separator
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := CountFields([]byte(tc.line)); got != tc.want {
|
||||
t.Fatalf("%q: got %d want %d", tc.line, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirstLineFieldCount(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, "x.dump")
|
||||
if err := os.WriteFile(path, []byte("a\tb\tc\n1\t2\t3\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
comp, _ := shard.NewCompressor("none", shard.DefaultLevel)
|
||||
n, ok, err := FirstLineFieldCount(path, comp)
|
||||
if err != nil || !ok || n != 3 {
|
||||
t.Fatalf("n=%d ok=%t err=%v", n, ok, err)
|
||||
}
|
||||
if err := CheckFieldCount(path, comp, 3); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := CheckFieldCount(path, comp, 4); err == nil || !strings.Contains(err.Error(), "3 fields") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirstLineFieldCountEmptyFile(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "empty.dump")
|
||||
if err := os.WriteFile(path, nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
comp, _ := shard.NewCompressor("none", shard.DefaultLevel)
|
||||
_, ok, err := FirstLineFieldCount(path, comp)
|
||||
if err != nil || ok {
|
||||
t.Fatalf("ok=%t err=%v", ok, err)
|
||||
}
|
||||
if err := CheckFieldCount(path, comp, 9); err != nil {
|
||||
t.Fatalf("empty shard must not fail the field check: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user