Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4a6b504f
commit
366d61ca25
49 files changed
+5099
-342
No files matched your search
@@ -0,0 +1,116 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"archer/internal/shard"
|
||||
)
|
||||
|
||||
func TestBuildLoadSQLShape(t *testing.T) {
|
||||
m := &Mapping{LoadList: []string{"`id`", "`title`"}}
|
||||
q := BuildLoadSQL("db", "orders", "utf8mb4", m, "ignore", "r1")
|
||||
for _, want := range []string{
|
||||
"LOAD DATA LOCAL INFILE 'Reader::r1'",
|
||||
"IGNORE INTO TABLE `db`.`orders`",
|
||||
"CHARACTER SET utf8mb4",
|
||||
`FIELDS TERMINATED BY '\t' ESCAPED BY '\\' LINES TERMINATED BY '\n'`,
|
||||
"(`id`, `title`)",
|
||||
} {
|
||||
if !strings.Contains(q, want) {
|
||||
t.Fatalf("missing %q in %s", want, q)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildLoadSQLModes(t *testing.T) {
|
||||
m := &Mapping{}
|
||||
for _, tc := range []struct{ mode, want string }{
|
||||
{"insert", "INFILE 'Reader::r' INTO TABLE"},
|
||||
{"ignore", "INFILE 'Reader::r' IGNORE INTO TABLE"},
|
||||
{"replace", "INFILE 'Reader::r' REPLACE INTO TABLE"},
|
||||
} {
|
||||
q := BuildLoadSQL("db", "t", "binary", m, tc.mode, "r")
|
||||
if !strings.Contains(q, tc.want) {
|
||||
t.Fatalf("%s: %s", tc.mode, q)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildLoadSQLNoColumnList(t *testing.T) {
|
||||
q := BuildLoadSQL("db", "t", "utf8mb4", &Mapping{}, "insert", "r")
|
||||
if strings.Contains(q, "(") {
|
||||
t.Fatalf("unexpected column list: %s", q)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEscapeSQLString(t *testing.T) {
|
||||
if got := escapeSQLString(`a'b\c`); got != `a''b\\c` {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckRowRule(t *testing.T) {
|
||||
if _, err := CheckRowRule("insert", 10, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := CheckRowRule("insert", 10, 9); err == nil || !strings.Contains(err.Error(), "expected 10") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
skipped, err := CheckRowRule("ignore", 10, 7)
|
||||
if err != nil || skipped != 3 {
|
||||
t.Fatalf("skipped=%d err=%v", skipped, err)
|
||||
}
|
||||
if _, err := CheckRowRule("ignore", 10, 11); err == nil {
|
||||
t.Fatal("ignore must fail when affected exceeds rows")
|
||||
}
|
||||
if _, err := CheckRowRule("replace", 10, 15); err != nil {
|
||||
t.Fatalf("replace only reports: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyShard(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, "000001.1-1.dump")
|
||||
body := []byte("1\ta\n")
|
||||
if err := os.WriteFile(path, body, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
good := shard.Entry{Seq: 1, Path: "000001.1-1.dump", SHA256: hex.EncodeToString(sum[:])}
|
||||
if err := VerifyShard(path, good); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bad := good
|
||||
bad.SHA256 = strings.Repeat("0", 64)
|
||||
if err := VerifyShard(path, bad); err == nil || !strings.Contains(err.Error(), "failed verification") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
if err := VerifyShard(path, shard.Entry{Seq: 1, Path: path}); err != nil {
|
||||
t.Fatalf("missing hash should be skipped: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckWarnings(t *testing.T) {
|
||||
if err := CheckWarnings(nil, false, "insert", 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := []Warning{{Level: "Warning", Code: 1265, Message: "data truncated"}}
|
||||
if err := CheckWarnings(w, true, "insert", 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := CheckWarnings(w, false, "insert", 3); err == nil || !strings.Contains(err.Error(), "shard 3") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
dup := []Warning{{Level: "Warning", Code: 1062, Message: "Duplicate entry '1'"}}
|
||||
if err := CheckWarnings(dup, false, "ignore", 1); err != nil {
|
||||
t.Fatalf("duplicate-key warnings are expected skips under ignore: %v", err)
|
||||
}
|
||||
if err := CheckWarnings(dup, false, "insert", 1); err == nil {
|
||||
t.Fatal("duplicate warnings must fail outside ignore mode")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user