Files
voidandClaude Sonnet 4.6 366d61ca25 Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-11-08 21:14:23 +08:00

212 lines
6.2 KiB
Go

package importer
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"archer/internal/checkpoint"
"archer/internal/shard"
)
func writeManifest(t *testing.T, root string, entries []shard.Entry) {
t.Helper()
var b strings.Builder
for _, e := range entries {
j, err := json.Marshal(e)
if err != nil {
t.Fatal(err)
}
b.Write(j)
b.WriteByte('\n')
}
if err := os.WriteFile(filepath.Join(root, "manifest.jsonl"), []byte(b.String()), 0o644); err != nil {
t.Fatal(err)
}
}
func writeShard(t *testing.T, root, rel, body string) {
t.Helper()
path := filepath.Join(root, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
func saveState(t *testing.T, root string, complete bool, config map[string]string) {
t.Helper()
if err := checkpoint.Save(root, &checkpoint.State{Complete: complete, Config: config}); err != nil {
t.Fatal(err)
}
}
func TestLoadArchiveReadsStateAndRecords(t *testing.T) {
root := t.TempDir()
entries := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump", PKLo: "1", PKHi: "1", Rows: 1, Bytes: 3, SHA256: "aa"}}
writeManifest(t, root, entries)
writeShard(t, root, entries[0].Path, "1\tx\n")
saveState(t, root, true, map[string]string{"table": "orders", "columns": "id,title"})
a, err := LoadArchive(root)
if err != nil {
t.Fatal(err)
}
if !a.Complete || a.SourceTable != "orders" {
t.Fatalf("archive = %+v", a)
}
if strings.Join(a.ArchivedColumns, ",") != "id,title" {
t.Fatalf("columns = %v", a.ArchivedColumns)
}
}
func TestLoadArchiveRejectsCorruptManifest(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "manifest.jsonl"), []byte("{not json\n"), 0o644); err != nil {
t.Fatal(err)
}
_, err := LoadArchive(root)
if err == nil || !strings.Contains(err.Error(), "line 1") {
t.Fatalf("got %v", err)
}
}
func TestLoadArchiveMissingManifest(t *testing.T) {
_, err := LoadArchive(t.TempDir())
if err == nil || !strings.Contains(err.Error(), "manifest") {
t.Fatalf("got %v", err)
}
}
func TestLoadArchiveNotADirectory(t *testing.T) {
path := filepath.Join(t.TempDir(), "file")
if err := os.WriteFile(path, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := LoadArchive(path); err == nil {
t.Fatal("expected an error")
}
}
func TestCompletionGate(t *testing.T) {
root := t.TempDir()
writeManifest(t, root, []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}})
a, err := LoadArchive(root)
if err != nil {
t.Fatal(err)
}
if err := a.CheckComplete(false); err == nil || !strings.Contains(err.Error(), "state.json") {
t.Fatalf("missing state: %v", err)
}
if err := a.CheckComplete(true); err != nil {
t.Fatalf("--allow-partial: %v", err)
}
saveState(t, root, false, nil)
a, _ = LoadArchive(root)
if err := a.CheckComplete(false); err == nil || !strings.Contains(err.Error(), "incomplete") {
t.Fatalf("incomplete: %v", err)
}
if err := a.CheckComplete(true); err != nil {
t.Fatalf("--allow-partial: %v", err)
}
saveState(t, root, true, nil)
a, _ = LoadArchive(root)
if err := a.CheckComplete(false); err != nil {
t.Fatal(err)
}
}
func TestMappingChoiceConflict(t *testing.T) {
root := t.TempDir()
writeManifest(t, root, []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}})
saveState(t, root, true, map[string]string{"columns": "id,title"})
a, _ := LoadArchive(root)
if err := a.CheckMappingChoice([]string{"id"}, false); err == nil || !strings.Contains(err.Error(), "--columns") {
t.Fatalf("got %v", err)
}
if err := a.CheckMappingChoice(nil, true); err == nil || !strings.Contains(err.Error(), "--assume-all-columns") {
t.Fatalf("got %v", err)
}
if err := a.CheckMappingChoice(nil, false); err != nil {
t.Fatal(err)
}
}
func TestCheckShardFiles(t *testing.T) {
root := t.TempDir()
writeShard(t, root, "a/000001.1-1.dump", "1\n")
ok := []shard.Entry{{Seq: 1, Path: "a/000001.1-1.dump"}}
if err := CheckShardFiles(root, ok); err != nil {
t.Fatal(err)
}
missing := []shard.Entry{{Seq: 7, Path: "a/000007.1-1.dump"}}
if err := CheckShardFiles(root, missing); err == nil || !strings.Contains(err.Error(), "seq 7") {
t.Fatalf("got %v", err)
}
}
func TestResolveFormat(t *testing.T) {
dump := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}, {Seq: 2, Path: "000002.2-2.dump.gz"}}
if _, _, _, err := ResolveFormat(dump); err == nil || !strings.Contains(err.Error(), "mix compression") {
t.Fatalf("got %v", err)
}
gz := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump.gz"}, {Seq: 2, Path: "000002.2-2.dump.gz"}}
format, compress, comp, err := ResolveFormat(gz)
if err != nil {
t.Fatal(err)
}
if format != "dump" || compress != "gzip" || comp.Name != "gzip" {
t.Fatalf("format=%s compress=%s comp=%+v", format, compress, comp)
}
for _, bad := range []string{"000001.1-1.csv", "000001.1-1.jsonl", "000001.1-1.csv.gz"} {
_, _, _, err := ResolveFormat([]shard.Entry{{Seq: 1, Path: bad}})
if err == nil || !strings.Contains(err.Error(), "only dump") {
t.Fatalf("%s: got %v", bad, err)
}
}
mixed := []shard.Entry{{Seq: 1, Path: "a/000001.1-1.dump"}, {Seq: 2, Path: "b/000002.2-2.csv"}}
if _, _, _, err := ResolveFormat(mixed); err == nil || !strings.Contains(err.Error(), "mix formats") {
t.Fatalf("got %v", err)
}
}
func TestShardFormat(t *testing.T) {
for _, tc := range []struct{ path, format, compress string }{
{"000001.1-2.dump", "dump", "none"},
{"000001.1-2.dump.gz", "dump", "gzip"},
{"000001.1-2.dump.zst", "dump", "zstd"},
{"000001.1-2.dump.xz", "dump", "xz"},
{"date=2025-01-01/000001.1-2.jsonl", "jsonl", "none"},
} {
f, c, err := ShardFormat(tc.path)
if err != nil || f != tc.format || c != tc.compress {
t.Fatalf("%s: f=%s c=%s err=%v", tc.path, f, c, err)
}
}
if _, _, err := ShardFormat("000001.1-2"); err == nil {
t.Fatal("expected an error for a missing extension")
}
}
func TestUnregisteredFiles(t *testing.T) {
root := t.TempDir()
entries := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}}
writeManifest(t, root, entries)
writeShard(t, root, "000001.1-1.dump", "1\n")
writeShard(t, root, "000002.2-2.dump", "2\n")
writeShard(t, root, "notes.txt", "ignored")
if got := UnregisteredFiles(root, entries); got != 1 {
t.Fatalf("got %d", got)
}
}