package importer import ( "encoding/json" "os" "path/filepath" "strings" "testing" "archer/internal/checkpoint" "archer/internal/shard" ) func writeManifest(t *testing.T, root string, entries []shard.Entry) { t.Helper() var b strings.Builder for _, e := range entries { j, err := json.Marshal(e) if err != nil { t.Fatal(err) } b.Write(j) b.WriteByte('\n') } if err := os.WriteFile(filepath.Join(root, "manifest.jsonl"), []byte(b.String()), 0o644); err != nil { t.Fatal(err) } } func writeShard(t *testing.T, root, rel, body string) { t.Helper() path := filepath.Join(root, filepath.FromSlash(rel)) if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(path, []byte(body), 0o644); err != nil { t.Fatal(err) } } func saveState(t *testing.T, root string, complete bool, config map[string]string) { t.Helper() if err := checkpoint.Save(root, &checkpoint.State{Complete: complete, Config: config}); err != nil { t.Fatal(err) } } func TestLoadArchiveReadsStateAndRecords(t *testing.T) { root := t.TempDir() entries := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump", PKLo: "1", PKHi: "1", Rows: 1, Bytes: 3, SHA256: "aa"}} writeManifest(t, root, entries) writeShard(t, root, entries[0].Path, "1\tx\n") saveState(t, root, true, map[string]string{"table": "orders", "columns": "id,title"}) a, err := LoadArchive(root) if err != nil { t.Fatal(err) } if !a.Complete || a.SourceTable != "orders" { t.Fatalf("archive = %+v", a) } if strings.Join(a.ArchivedColumns, ",") != "id,title" { t.Fatalf("columns = %v", a.ArchivedColumns) } } func TestLoadArchiveRejectsCorruptManifest(t *testing.T) { root := t.TempDir() if err := os.WriteFile(filepath.Join(root, "manifest.jsonl"), []byte("{not json\n"), 0o644); err != nil { t.Fatal(err) } _, err := LoadArchive(root) if err == nil || !strings.Contains(err.Error(), "line 1") { t.Fatalf("got %v", err) } } func TestLoadArchiveMissingManifest(t *testing.T) { _, err := LoadArchive(t.TempDir()) if err == nil || !strings.Contains(err.Error(), "manifest") { t.Fatalf("got %v", err) } } func TestLoadArchiveNotADirectory(t *testing.T) { path := filepath.Join(t.TempDir(), "file") if err := os.WriteFile(path, []byte("x"), 0o644); err != nil { t.Fatal(err) } if _, err := LoadArchive(path); err == nil { t.Fatal("expected an error") } } func TestCompletionGate(t *testing.T) { root := t.TempDir() writeManifest(t, root, []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}}) a, err := LoadArchive(root) if err != nil { t.Fatal(err) } if err := a.CheckComplete(false); err == nil || !strings.Contains(err.Error(), "state.json") { t.Fatalf("missing state: %v", err) } if err := a.CheckComplete(true); err != nil { t.Fatalf("--allow-partial: %v", err) } saveState(t, root, false, nil) a, _ = LoadArchive(root) if err := a.CheckComplete(false); err == nil || !strings.Contains(err.Error(), "incomplete") { t.Fatalf("incomplete: %v", err) } if err := a.CheckComplete(true); err != nil { t.Fatalf("--allow-partial: %v", err) } saveState(t, root, true, nil) a, _ = LoadArchive(root) if err := a.CheckComplete(false); err != nil { t.Fatal(err) } } func TestMappingChoiceConflict(t *testing.T) { root := t.TempDir() writeManifest(t, root, []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}}) saveState(t, root, true, map[string]string{"columns": "id,title"}) a, _ := LoadArchive(root) if err := a.CheckMappingChoice([]string{"id"}, false); err == nil || !strings.Contains(err.Error(), "--columns") { t.Fatalf("got %v", err) } if err := a.CheckMappingChoice(nil, true); err == nil || !strings.Contains(err.Error(), "--assume-all-columns") { t.Fatalf("got %v", err) } if err := a.CheckMappingChoice(nil, false); err != nil { t.Fatal(err) } } func TestCheckShardFiles(t *testing.T) { root := t.TempDir() writeShard(t, root, "a/000001.1-1.dump", "1\n") ok := []shard.Entry{{Seq: 1, Path: "a/000001.1-1.dump"}} if err := CheckShardFiles(root, ok); err != nil { t.Fatal(err) } missing := []shard.Entry{{Seq: 7, Path: "a/000007.1-1.dump"}} if err := CheckShardFiles(root, missing); err == nil || !strings.Contains(err.Error(), "seq 7") { t.Fatalf("got %v", err) } } func TestResolveFormat(t *testing.T) { dump := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}, {Seq: 2, Path: "000002.2-2.dump.gz"}} if _, _, _, err := ResolveFormat(dump); err == nil || !strings.Contains(err.Error(), "mix compression") { t.Fatalf("got %v", err) } gz := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump.gz"}, {Seq: 2, Path: "000002.2-2.dump.gz"}} format, compress, comp, err := ResolveFormat(gz) if err != nil { t.Fatal(err) } if format != "dump" || compress != "gzip" || comp.Name != "gzip" { t.Fatalf("format=%s compress=%s comp=%+v", format, compress, comp) } for _, bad := range []string{"000001.1-1.csv", "000001.1-1.jsonl", "000001.1-1.csv.gz"} { _, _, _, err := ResolveFormat([]shard.Entry{{Seq: 1, Path: bad}}) if err == nil || !strings.Contains(err.Error(), "only dump") { t.Fatalf("%s: got %v", bad, err) } } mixed := []shard.Entry{{Seq: 1, Path: "a/000001.1-1.dump"}, {Seq: 2, Path: "b/000002.2-2.csv"}} if _, _, _, err := ResolveFormat(mixed); err == nil || !strings.Contains(err.Error(), "mix formats") { t.Fatalf("got %v", err) } } func TestShardFormat(t *testing.T) { for _, tc := range []struct{ path, format, compress string }{ {"000001.1-2.dump", "dump", "none"}, {"000001.1-2.dump.gz", "dump", "gzip"}, {"000001.1-2.dump.zst", "dump", "zstd"}, {"000001.1-2.dump.xz", "dump", "xz"}, {"date=2025-01-01/000001.1-2.jsonl", "jsonl", "none"}, } { f, c, err := ShardFormat(tc.path) if err != nil || f != tc.format || c != tc.compress { t.Fatalf("%s: f=%s c=%s err=%v", tc.path, f, c, err) } } if _, _, err := ShardFormat("000001.1-2"); err == nil { t.Fatal("expected an error for a missing extension") } } func TestUnregisteredFiles(t *testing.T) { root := t.TempDir() entries := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}} writeManifest(t, root, entries) writeShard(t, root, "000001.1-1.dump", "1\n") writeShard(t, root, "000002.2-2.dump", "2\n") writeShard(t, root, "notes.txt", "ignored") if got := UnregisteredFiles(root, entries); got != 1 { t.Fatalf("got %d", got) } }