Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4a6b504f
commit
366d61ca25
49 files changed
+5099
-342
No files matched your search
@@ -0,0 +1,211 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"archer/internal/checkpoint"
|
||||
"archer/internal/shard"
|
||||
)
|
||||
|
||||
func writeManifest(t *testing.T, root string, entries []shard.Entry) {
|
||||
t.Helper()
|
||||
var b strings.Builder
|
||||
for _, e := range entries {
|
||||
j, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.Write(j)
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "manifest.jsonl"), []byte(b.String()), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeShard(t *testing.T, root, rel, body string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(root, filepath.FromSlash(rel))
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func saveState(t *testing.T, root string, complete bool, config map[string]string) {
|
||||
t.Helper()
|
||||
if err := checkpoint.Save(root, &checkpoint.State{Complete: complete, Config: config}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadArchiveReadsStateAndRecords(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
entries := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump", PKLo: "1", PKHi: "1", Rows: 1, Bytes: 3, SHA256: "aa"}}
|
||||
writeManifest(t, root, entries)
|
||||
writeShard(t, root, entries[0].Path, "1\tx\n")
|
||||
saveState(t, root, true, map[string]string{"table": "orders", "columns": "id,title"})
|
||||
|
||||
a, err := LoadArchive(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !a.Complete || a.SourceTable != "orders" {
|
||||
t.Fatalf("archive = %+v", a)
|
||||
}
|
||||
if strings.Join(a.ArchivedColumns, ",") != "id,title" {
|
||||
t.Fatalf("columns = %v", a.ArchivedColumns)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadArchiveRejectsCorruptManifest(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "manifest.jsonl"), []byte("{not json\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := LoadArchive(root)
|
||||
if err == nil || !strings.Contains(err.Error(), "line 1") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadArchiveMissingManifest(t *testing.T) {
|
||||
_, err := LoadArchive(t.TempDir())
|
||||
if err == nil || !strings.Contains(err.Error(), "manifest") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadArchiveNotADirectory(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "file")
|
||||
if err := os.WriteFile(path, []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := LoadArchive(path); err == nil {
|
||||
t.Fatal("expected an error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompletionGate(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeManifest(t, root, []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}})
|
||||
|
||||
a, err := LoadArchive(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := a.CheckComplete(false); err == nil || !strings.Contains(err.Error(), "state.json") {
|
||||
t.Fatalf("missing state: %v", err)
|
||||
}
|
||||
if err := a.CheckComplete(true); err != nil {
|
||||
t.Fatalf("--allow-partial: %v", err)
|
||||
}
|
||||
|
||||
saveState(t, root, false, nil)
|
||||
a, _ = LoadArchive(root)
|
||||
if err := a.CheckComplete(false); err == nil || !strings.Contains(err.Error(), "incomplete") {
|
||||
t.Fatalf("incomplete: %v", err)
|
||||
}
|
||||
if err := a.CheckComplete(true); err != nil {
|
||||
t.Fatalf("--allow-partial: %v", err)
|
||||
}
|
||||
|
||||
saveState(t, root, true, nil)
|
||||
a, _ = LoadArchive(root)
|
||||
if err := a.CheckComplete(false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingChoiceConflict(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeManifest(t, root, []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}})
|
||||
saveState(t, root, true, map[string]string{"columns": "id,title"})
|
||||
a, _ := LoadArchive(root)
|
||||
if err := a.CheckMappingChoice([]string{"id"}, false); err == nil || !strings.Contains(err.Error(), "--columns") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
if err := a.CheckMappingChoice(nil, true); err == nil || !strings.Contains(err.Error(), "--assume-all-columns") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
if err := a.CheckMappingChoice(nil, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckShardFiles(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeShard(t, root, "a/000001.1-1.dump", "1\n")
|
||||
ok := []shard.Entry{{Seq: 1, Path: "a/000001.1-1.dump"}}
|
||||
if err := CheckShardFiles(root, ok); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
missing := []shard.Entry{{Seq: 7, Path: "a/000007.1-1.dump"}}
|
||||
if err := CheckShardFiles(root, missing); err == nil || !strings.Contains(err.Error(), "seq 7") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveFormat(t *testing.T) {
|
||||
dump := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}, {Seq: 2, Path: "000002.2-2.dump.gz"}}
|
||||
if _, _, _, err := ResolveFormat(dump); err == nil || !strings.Contains(err.Error(), "mix compression") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
|
||||
gz := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump.gz"}, {Seq: 2, Path: "000002.2-2.dump.gz"}}
|
||||
format, compress, comp, err := ResolveFormat(gz)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if format != "dump" || compress != "gzip" || comp.Name != "gzip" {
|
||||
t.Fatalf("format=%s compress=%s comp=%+v", format, compress, comp)
|
||||
}
|
||||
|
||||
for _, bad := range []string{"000001.1-1.csv", "000001.1-1.jsonl", "000001.1-1.csv.gz"} {
|
||||
_, _, _, err := ResolveFormat([]shard.Entry{{Seq: 1, Path: bad}})
|
||||
if err == nil || !strings.Contains(err.Error(), "only dump") {
|
||||
t.Fatalf("%s: got %v", bad, err)
|
||||
}
|
||||
}
|
||||
|
||||
mixed := []shard.Entry{{Seq: 1, Path: "a/000001.1-1.dump"}, {Seq: 2, Path: "b/000002.2-2.csv"}}
|
||||
if _, _, _, err := ResolveFormat(mixed); err == nil || !strings.Contains(err.Error(), "mix formats") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShardFormat(t *testing.T) {
|
||||
for _, tc := range []struct{ path, format, compress string }{
|
||||
{"000001.1-2.dump", "dump", "none"},
|
||||
{"000001.1-2.dump.gz", "dump", "gzip"},
|
||||
{"000001.1-2.dump.zst", "dump", "zstd"},
|
||||
{"000001.1-2.dump.xz", "dump", "xz"},
|
||||
{"date=2025-01-01/000001.1-2.jsonl", "jsonl", "none"},
|
||||
} {
|
||||
f, c, err := ShardFormat(tc.path)
|
||||
if err != nil || f != tc.format || c != tc.compress {
|
||||
t.Fatalf("%s: f=%s c=%s err=%v", tc.path, f, c, err)
|
||||
}
|
||||
}
|
||||
if _, _, err := ShardFormat("000001.1-2"); err == nil {
|
||||
t.Fatal("expected an error for a missing extension")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnregisteredFiles(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
entries := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}}
|
||||
writeManifest(t, root, entries)
|
||||
writeShard(t, root, "000001.1-1.dump", "1\n")
|
||||
writeShard(t, root, "000002.2-2.dump", "2\n")
|
||||
writeShard(t, root, "notes.txt", "ignored")
|
||||
if got := UnregisteredFiles(root, entries); got != 1 {
|
||||
t.Fatalf("got %d", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user