Add archer import and allow date-mode bulk delete

archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-08 21:14:23 +08:00
1 parent bc4a6b504f
commit 366d61ca25
49 files changed
+5099 -342

No files matched your search

+104
View File
@@ -0,0 +1,104 @@
package importer
import (
"bytes"
"context"
"strings"
"testing"
"archer/internal/archererr"
"archer/internal/cli"
"archer/internal/shard"
)
func TestSplitRanges(t *testing.T) {
got := splitRanges(10, 3)
want := [][2]int{{0, 4}, {4, 7}, {7, 10}}
if len(got) != len(want) {
t.Fatalf("got %v", got)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("got %v want %v", got, want)
}
}
if got := splitRanges(2, 5); len(got) != 2 || got[1] != [2]int{1, 2} {
t.Fatalf("workers beyond shards: %v", got)
}
if got := splitRanges(0, 3); got != nil {
t.Fatalf("empty selection: %v", got)
}
}
func TestRunConfigErrorExitCode(t *testing.T) {
err := Run(context.Background(), cli.ImportDefaults())
if err == nil {
t.Fatal("expected an error")
}
if got := archererr.ExitCode(err); got != 2 {
t.Fatalf("exit code = %d, want 2 (%v)", got, err)
}
}
func TestRunEmptySelectionIsConfigError(t *testing.T) {
root := t.TempDir()
writeManifest(t, root, []shard.Entry{{Seq: 1, Path: "000001.1-1.dump", Rows: 1}})
writeShard(t, root, "000001.1-1.dump", "1\n")
saveState(t, root, true, map[string]string{"table": "src"})
cfg := cli.ImportDefaults()
cfg.FileDir = root
cfg.DestDSN = "u:p@tcp(127.0.0.1:3306)/db"
cfg.Selects = []string{"seq:9-9"}
err := Run(context.Background(), cfg)
if err == nil || !strings.Contains(err.Error(), "no shards matched") {
t.Fatalf("got %v", err)
}
if got := archererr.ExitCode(err); got != 2 {
t.Fatalf("exit code = %d, want 2", got)
}
}
func TestRunDryRunDoesNotConnect(t *testing.T) {
root := t.TempDir()
writeManifest(t, root, []shard.Entry{{Seq: 1, Path: "000001.1-1.dump", PKLo: "1", PKHi: "1", Rows: 1, Bytes: 3}})
writeShard(t, root, "000001.1-1.dump", "1\tx\n")
saveState(t, root, true, map[string]string{"table": "src"})
cfg := cli.ImportDefaults()
cfg.FileDir = root
cfg.DestDSN = "u:p@tcp(127.0.0.1:1)/db" // would fail if a connection were attempted
cfg.DryRun = true
// PrintDryRun goes to stdout; Run reaching it without a connection error is
// what matters here.
if err := Run(context.Background(), cfg); err != nil {
t.Fatal(err)
}
}
func TestPrintDryRun(t *testing.T) {
root := t.TempDir()
entries := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump", PKLo: "1", PKHi: "1", Rows: 2, Bytes: 10}}
writeManifest(t, root, entries)
saveState(t, root, true, map[string]string{"table": "src", "columns": "id,title"})
archive, err := LoadArchive(root)
if err != nil {
t.Fatal(err)
}
cfg := cli.ImportDefaults()
cfg.FileDir = root
cfg.DestDSN = "u:p@tcp(h:3306)/db"
cfg.DestTable = "src"
var buf bytes.Buffer
if err := PrintDryRun(&buf, cfg, archive, entries, nil, "none", 2, true); err != nil {
t.Fatal(err)
}
out := buf.String()
for _, want := range []string{"archive:", "dest table:", "selected shards (1)", "seq=1", "total: 1 shards, 2 rows, 10 bytes", "no database connection"} {
if !strings.Contains(out, want) {
t.Fatalf("missing %q in:\n%s", want, out)
}
}
}