archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
117 lines
3.4 KiB
Go
117 lines
3.4 KiB
Go
package importer
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"archer/internal/shard"
|
|
)
|
|
|
|
func TestBuildLoadSQLShape(t *testing.T) {
|
|
m := &Mapping{LoadList: []string{"`id`", "`title`"}}
|
|
q := BuildLoadSQL("db", "orders", "utf8mb4", m, "ignore", "r1")
|
|
for _, want := range []string{
|
|
"LOAD DATA LOCAL INFILE 'Reader::r1'",
|
|
"IGNORE INTO TABLE `db`.`orders`",
|
|
"CHARACTER SET utf8mb4",
|
|
`FIELDS TERMINATED BY '\t' ESCAPED BY '\\' LINES TERMINATED BY '\n'`,
|
|
"(`id`, `title`)",
|
|
} {
|
|
if !strings.Contains(q, want) {
|
|
t.Fatalf("missing %q in %s", want, q)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBuildLoadSQLModes(t *testing.T) {
|
|
m := &Mapping{}
|
|
for _, tc := range []struct{ mode, want string }{
|
|
{"insert", "INFILE 'Reader::r' INTO TABLE"},
|
|
{"ignore", "INFILE 'Reader::r' IGNORE INTO TABLE"},
|
|
{"replace", "INFILE 'Reader::r' REPLACE INTO TABLE"},
|
|
} {
|
|
q := BuildLoadSQL("db", "t", "binary", m, tc.mode, "r")
|
|
if !strings.Contains(q, tc.want) {
|
|
t.Fatalf("%s: %s", tc.mode, q)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBuildLoadSQLNoColumnList(t *testing.T) {
|
|
q := BuildLoadSQL("db", "t", "utf8mb4", &Mapping{}, "insert", "r")
|
|
if strings.Contains(q, "(") {
|
|
t.Fatalf("unexpected column list: %s", q)
|
|
}
|
|
}
|
|
|
|
func TestEscapeSQLString(t *testing.T) {
|
|
if got := escapeSQLString(`a'b\c`); got != `a''b\\c` {
|
|
t.Fatalf("got %q", got)
|
|
}
|
|
}
|
|
|
|
func TestCheckRowRule(t *testing.T) {
|
|
if _, err := CheckRowRule("insert", 10, 10); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := CheckRowRule("insert", 10, 9); err == nil || !strings.Contains(err.Error(), "expected 10") {
|
|
t.Fatalf("got %v", err)
|
|
}
|
|
skipped, err := CheckRowRule("ignore", 10, 7)
|
|
if err != nil || skipped != 3 {
|
|
t.Fatalf("skipped=%d err=%v", skipped, err)
|
|
}
|
|
if _, err := CheckRowRule("ignore", 10, 11); err == nil {
|
|
t.Fatal("ignore must fail when affected exceeds rows")
|
|
}
|
|
if _, err := CheckRowRule("replace", 10, 15); err != nil {
|
|
t.Fatalf("replace only reports: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestVerifyShard(t *testing.T) {
|
|
root := t.TempDir()
|
|
path := filepath.Join(root, "000001.1-1.dump")
|
|
body := []byte("1\ta\n")
|
|
if err := os.WriteFile(path, body, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sum := sha256.Sum256(body)
|
|
good := shard.Entry{Seq: 1, Path: "000001.1-1.dump", SHA256: hex.EncodeToString(sum[:])}
|
|
if err := VerifyShard(path, good); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
bad := good
|
|
bad.SHA256 = strings.Repeat("0", 64)
|
|
if err := VerifyShard(path, bad); err == nil || !strings.Contains(err.Error(), "failed verification") {
|
|
t.Fatalf("got %v", err)
|
|
}
|
|
if err := VerifyShard(path, shard.Entry{Seq: 1, Path: path}); err != nil {
|
|
t.Fatalf("missing hash should be skipped: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCheckWarnings(t *testing.T) {
|
|
if err := CheckWarnings(nil, false, "insert", 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w := []Warning{{Level: "Warning", Code: 1265, Message: "data truncated"}}
|
|
if err := CheckWarnings(w, true, "insert", 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := CheckWarnings(w, false, "insert", 3); err == nil || !strings.Contains(err.Error(), "shard 3") {
|
|
t.Fatalf("got %v", err)
|
|
}
|
|
dup := []Warning{{Level: "Warning", Code: 1062, Message: "Duplicate entry '1'"}}
|
|
if err := CheckWarnings(dup, false, "ignore", 1); err != nil {
|
|
t.Fatalf("duplicate-key warnings are expected skips under ignore: %v", err)
|
|
}
|
|
if err := CheckWarnings(dup, false, "insert", 1); err == nil {
|
|
t.Fatal("duplicate warnings must fail outside ignore mode")
|
|
}
|
|
}
|