archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
430 lines
14 KiB
Go
430 lines
14 KiB
Go
package importer
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/go-sql-driver/mysql"
|
|
|
|
"archer/internal/app"
|
|
"archer/internal/cli"
|
|
)
|
|
|
|
func TestMain(m *testing.M) {
|
|
if dsn := os.Getenv("ARCHER_TEST_DSN"); dsn != "" {
|
|
if cfg, err := mysql.ParseDSN(dsn); err == nil && cfg.DBName != "" {
|
|
dbName := cfg.DBName
|
|
cfg.DBName = ""
|
|
if db, err := sql.Open("mysql", cfg.FormatDSN()); err == nil {
|
|
_, _ = db.Exec("CREATE DATABASE IF NOT EXISTS `" + dbName + "`")
|
|
_, _ = db.Exec("SET GLOBAL local_infile=ON")
|
|
_, _ = db.Exec("SET GLOBAL sql_mode=''")
|
|
db.Close()
|
|
}
|
|
}
|
|
}
|
|
os.Exit(m.Run())
|
|
}
|
|
|
|
func testDSN(t *testing.T) string {
|
|
t.Helper()
|
|
d := os.Getenv("ARCHER_TEST_DSN")
|
|
if d == "" {
|
|
t.Skip("ARCHER_TEST_DSN is not set; skipping MySQL integration test")
|
|
}
|
|
return d
|
|
}
|
|
|
|
func charsetDSN(t *testing.T, dsn string) string {
|
|
t.Helper()
|
|
cfg, err := mysql.ParseDSN(dsn)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cfg.Params == nil {
|
|
cfg.Params = map[string]string{}
|
|
}
|
|
cfg.Params["charset"] = "utf8mb4"
|
|
return cfg.FormatDSN()
|
|
}
|
|
|
|
func openDB(t *testing.T, dsn string) *sql.DB {
|
|
t.Helper()
|
|
db, err := sql.Open("mysql", dsn)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.Ping(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return db
|
|
}
|
|
|
|
func execSQL(t *testing.T, db *sql.DB, q string, args ...any) {
|
|
t.Helper()
|
|
if _, err := db.Exec(q, args...); err != nil {
|
|
t.Fatalf("exec %q: %v", q, err)
|
|
}
|
|
}
|
|
|
|
func exportTable(t *testing.T, dsn, table, dir string, extra ...string) {
|
|
t.Helper()
|
|
args := []string{"--source", dsn, "--table", table, "--where", "1=1", "--file", dir, "--progress", "off"}
|
|
args = append(args, extra...)
|
|
cfg, err := cli.Parse(args, io.Discard)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := app.Run(context.Background(), cfg); err != nil {
|
|
t.Fatalf("export %s: %v", table, err)
|
|
}
|
|
}
|
|
|
|
func importArchive(t *testing.T, dsn, dir string, extra ...string) error {
|
|
t.Helper()
|
|
args := []string{"--file", dir, "--dest", dsn, "--progress", "off"}
|
|
args = append(args, extra...)
|
|
cfg, err := cli.ParseImport(args, io.Discard)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return Run(context.Background(), cfg)
|
|
}
|
|
|
|
func countRows(t *testing.T, db *sql.DB, table string) int {
|
|
t.Helper()
|
|
var n int
|
|
if err := db.QueryRow("SELECT COUNT(*) FROM `" + table + "`").Scan(&n); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return n
|
|
}
|
|
|
|
// fingerprint renders an ordered result set byte-for-byte, using hex so NULL,
|
|
// tabs, newlines and binary values are all distinguishable.
|
|
func fingerprint(t *testing.T, db *sql.DB, q string) string {
|
|
t.Helper()
|
|
rows, err := db.Query(q)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer rows.Close()
|
|
cols, err := rows.Columns()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var b strings.Builder
|
|
for rows.Next() {
|
|
raw := make([]sql.RawBytes, len(cols))
|
|
ptrs := make([]any, len(cols))
|
|
for i := range raw {
|
|
ptrs[i] = &raw[i]
|
|
}
|
|
if err := rows.Scan(ptrs...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i, v := range raw {
|
|
if i > 0 {
|
|
b.WriteByte('|')
|
|
}
|
|
if v == nil {
|
|
b.WriteString("<NULL>")
|
|
} else {
|
|
fmt.Fprintf(&b, "%x", []byte(v))
|
|
}
|
|
}
|
|
b.WriteByte('\n')
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func seedSimple(t *testing.T, db *sql.DB, table string) {
|
|
t.Helper()
|
|
execSQL(t, db, "DROP TABLE IF EXISTS `"+table+"`")
|
|
execSQL(t, db, "CREATE TABLE `"+table+"` (id BIGINT NOT NULL, v VARCHAR(64) NULL, PRIMARY KEY (id)) DEFAULT CHARSET=utf8mb4")
|
|
execSQL(t, db, "INSERT INTO `"+table+"` (id, v) VALUES (1,'a'),(2,'b'),(3,'c'),(4,'tab\\there'),(5,NULL)")
|
|
}
|
|
|
|
func freshTable(t *testing.T, db *sql.DB, table string) {
|
|
t.Helper()
|
|
execSQL(t, db, "DROP TABLE IF EXISTS `"+table+"`")
|
|
execSQL(t, db, "CREATE TABLE `"+table+"` (id BIGINT NOT NULL, v VARCHAR(64) NULL, PRIMARY KEY (id)) DEFAULT CHARSET=utf8mb4")
|
|
}
|
|
|
|
func TestIntegrationExportImport(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openDB(t, dsn)
|
|
defer db.Close()
|
|
seedSimple(t, db, "it_imp_src")
|
|
dir := t.TempDir()
|
|
exportTable(t, dsn, "it_imp_src", dir)
|
|
|
|
freshTable(t, db, "it_imp_dst")
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_dst", "--assume-all-columns"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n := countRows(t, db, "it_imp_dst"); n != 5 {
|
|
t.Fatalf("imported %d rows", n)
|
|
}
|
|
if got, want := fingerprint(t, db, "SELECT id, v FROM it_imp_dst ORDER BY id"),
|
|
fingerprint(t, db, "SELECT id, v FROM it_imp_src ORDER BY id"); got != want {
|
|
t.Fatalf("rows differ:\n%s\nwant:\n%s", got, want)
|
|
}
|
|
}
|
|
|
|
func TestIntegrationCompression(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openDB(t, dsn)
|
|
defer db.Close()
|
|
seedSimple(t, db, "it_imp_comp_src")
|
|
for _, comp := range []string{"none", "gzip", "zstd", "xz"} {
|
|
dir := t.TempDir()
|
|
exportTable(t, dsn, "it_imp_comp_src", dir, "--compress", comp)
|
|
if files, _ := filepath.Glob(filepath.Join(dir, "*.dump*")); len(files) != 1 {
|
|
t.Fatalf("%s: shard files %v", comp, files)
|
|
}
|
|
dst := "it_imp_comp_" + comp
|
|
freshTable(t, db, dst)
|
|
if err := importArchive(t, dsn, dir, "--dest-table", dst, "--assume-all-columns"); err != nil {
|
|
t.Fatalf("%s: %v", comp, err)
|
|
}
|
|
if n := countRows(t, db, dst); n != 5 {
|
|
t.Fatalf("%s: %d rows", comp, n)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestIntegrationModes(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openDB(t, dsn)
|
|
defer db.Close()
|
|
seedSimple(t, db, "it_imp_modes_src")
|
|
dir := t.TempDir()
|
|
exportTable(t, dsn, "it_imp_modes_src", dir)
|
|
|
|
freshTable(t, db, "it_imp_modes_dst")
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns"); err != nil {
|
|
t.Fatalf("ignore re-import must be idempotent: %v", err)
|
|
}
|
|
if n := countRows(t, db, "it_imp_modes_dst"); n != 5 {
|
|
t.Fatalf("ignore re-import changed row count: %d", n)
|
|
}
|
|
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns", "--mode", "insert"); err == nil {
|
|
t.Fatal("insert mode must fail on duplicate keys")
|
|
}
|
|
|
|
execSQL(t, db, "UPDATE it_imp_modes_dst SET v='changed' WHERE id=2")
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns", "--mode", "replace"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var v string
|
|
if err := db.QueryRow("SELECT v FROM it_imp_modes_dst WHERE id=2").Scan(&v); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if v != "b" {
|
|
t.Fatalf("replace did not restore value: %q", v)
|
|
}
|
|
}
|
|
|
|
func TestIntegrationVerifyAndPartial(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openDB(t, dsn)
|
|
defer db.Close()
|
|
seedSimple(t, db, "it_imp_verify_src")
|
|
dir := t.TempDir()
|
|
exportTable(t, dsn, "it_imp_verify_src", dir)
|
|
|
|
files, err := filepath.Glob(filepath.Join(dir, "*.dump"))
|
|
if err != nil || len(files) != 1 {
|
|
t.Fatalf("files %v err %v", files, err)
|
|
}
|
|
f, err := os.OpenFile(files[0], os.O_APPEND|os.O_WRONLY, 0o644)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.WriteString("tampered\n"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.Close()
|
|
freshTable(t, db, "it_imp_verify_dst")
|
|
err = importArchive(t, dsn, dir, "--dest-table", "it_imp_verify_dst", "--assume-all-columns")
|
|
if err == nil || !strings.Contains(err.Error(), "failed verification") {
|
|
t.Fatalf("got %v", err)
|
|
}
|
|
|
|
// Re-export a pristine archive for the completion-gate half.
|
|
dir = t.TempDir()
|
|
exportTable(t, dsn, "it_imp_verify_src", dir)
|
|
statePath := filepath.Join(dir, "state.json")
|
|
body, err := os.ReadFile(statePath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
body = []byte(strings.Replace(string(body), `"complete": true`, `"complete": false`, 1))
|
|
if err := os.WriteFile(statePath, body, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
freshTable(t, db, "it_imp_partial_dst")
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_partial_dst", "--assume-all-columns"); err == nil || !strings.Contains(err.Error(), "incomplete") {
|
|
t.Fatalf("incomplete archive must be refused, got %v", err)
|
|
}
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_partial_dst", "--assume-all-columns", "--allow-partial"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n := countRows(t, db, "it_imp_partial_dst"); n != 5 {
|
|
t.Fatalf("partial import: %d rows", n)
|
|
}
|
|
}
|
|
|
|
func TestIntegrationWorkersAndEnginesAgree(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openDB(t, dsn)
|
|
defer db.Close()
|
|
seedSimple(t, db, "it_imp_par_src")
|
|
for i := 6; i <= 20; i++ {
|
|
execSQL(t, db, "INSERT INTO it_imp_par_src (id, v) VALUES (?, ?)", i, fmt.Sprintf("v%d", i))
|
|
}
|
|
dir := t.TempDir()
|
|
exportTable(t, dsn, "it_imp_par_src", dir, "--shard-by", "count", "--shard-rows", "4", "--chunk-size", "3")
|
|
|
|
for _, dst := range []string{"it_imp_par_1", "it_imp_par_4", "it_imp_par_ins"} {
|
|
freshTable(t, db, dst)
|
|
}
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_par_1", "--workers", "1", "--assume-all-columns"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_par_4", "--workers", "4", "--assume-all-columns"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_par_ins", "--engine", "insert", "--assume-all-columns"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := fingerprint(t, db, "SELECT id, v FROM it_imp_par_1 ORDER BY id")
|
|
for _, dst := range []string{"it_imp_par_4", "it_imp_par_ins"} {
|
|
if got := fingerprint(t, db, "SELECT id, v FROM `"+dst+"` ORDER BY id"); got != want {
|
|
t.Fatalf("%s differs from serial load", dst)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestIntegrationFidelity(t *testing.T) {
|
|
dsn := charsetDSN(t, testDSN(t))
|
|
db := openDB(t, dsn)
|
|
defer db.Close()
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_fid_src")
|
|
execSQL(t, db, `CREATE TABLE it_imp_fid_src (
|
|
id BIGINT NOT NULL PRIMARY KEY,
|
|
s VARCHAR(128) NULL,
|
|
b BLOB NULL,
|
|
n INT NULL) DEFAULT CHARSET=utf8mb4`)
|
|
execSQL(t, db, "INSERT INTO it_imp_fid_src (id, s, b, n) VALUES (?,?,?,?)", 1, nil, nil, nil)
|
|
execSQL(t, db, "INSERT INTO it_imp_fid_src (id, s, b, n) VALUES (?,?,?,?)", 2, "tab\there\nnewline\\back", []byte{0x00, 0x01, 0x02, 0x03, 0xff}, 0)
|
|
execSQL(t, db, "INSERT INTO it_imp_fid_src (id, s, b, n) VALUES (?,?,?,?)", 3, "中文😀", []byte("hi"), 42)
|
|
|
|
dir := t.TempDir()
|
|
exportTable(t, dsn, "it_imp_fid_src", dir)
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_fid_dst")
|
|
execSQL(t, db, `CREATE TABLE it_imp_fid_dst (
|
|
id BIGINT NOT NULL PRIMARY KEY,
|
|
s VARCHAR(128) NULL,
|
|
b BLOB NULL,
|
|
n INT NULL) DEFAULT CHARSET=utf8mb4`)
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_fid_dst", "--assume-all-columns", "--charset", "binary"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
q := "SELECT id, s, HEX(b), n FROM %s ORDER BY id"
|
|
got := fingerprint(t, db, fmt.Sprintf(q, "it_imp_fid_dst"))
|
|
want := fingerprint(t, db, fmt.Sprintf(q, "it_imp_fid_src"))
|
|
if got != want {
|
|
t.Fatalf("fidelity mismatch:\n got: %s\nwant: %s", got, want)
|
|
}
|
|
}
|
|
|
|
func TestIntegrationColumnMappings(t *testing.T) {
|
|
dsn := charsetDSN(t, testDSN(t))
|
|
db := openDB(t, dsn)
|
|
defer db.Close()
|
|
|
|
// (a) archive-recorded column list
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_cols_src")
|
|
execSQL(t, db, "CREATE TABLE it_imp_cols_src (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))")
|
|
execSQL(t, db, "INSERT INTO it_imp_cols_src (id, a, b) VALUES (1,'a1','b1'),(2,'a2','b2')")
|
|
dir := t.TempDir()
|
|
exportTable(t, dsn, "it_imp_cols_src", dir, "--columns", "id,a")
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_cols_dst")
|
|
execSQL(t, db, "CREATE TABLE it_imp_cols_dst (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))")
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_cols_dst"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := fingerprint(t, db, "SELECT id, a, b FROM it_imp_cols_dst ORDER BY id"); got != "31|6131|<NULL>\n32|6132|<NULL>\n" {
|
|
t.Fatalf("archive-recorded mapping: %q", got)
|
|
}
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_cols_dst", "--columns", "id,a"); err == nil {
|
|
t.Fatal("archive column list plus --columns must be rejected")
|
|
}
|
|
|
|
// (b) explicit --columns on a default (all-columns) archive
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_all_src")
|
|
execSQL(t, db, "CREATE TABLE it_imp_all_src (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))")
|
|
execSQL(t, db, "INSERT INTO it_imp_all_src (id, a, b) VALUES (1,'a1','b1')")
|
|
dir = t.TempDir()
|
|
exportTable(t, dsn, "it_imp_all_src", dir)
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_all_dst")
|
|
execSQL(t, db, "CREATE TABLE it_imp_all_dst (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))")
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst"); err == nil {
|
|
t.Fatal("default all-columns archive without a mapping must be refused")
|
|
}
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst", "--columns", "id,a,b"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst", "--columns", "id,nope"); err == nil || !strings.Contains(err.Error(), "does not exist") {
|
|
t.Fatalf("missing column: %v", err)
|
|
}
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst", "--columns", "id"); err == nil || !strings.Contains(err.Error(), "fields") {
|
|
t.Fatalf("field count mismatch: %v", err)
|
|
}
|
|
|
|
// (c) positional mapping with a generated column
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_gen_src")
|
|
execSQL(t, db, `CREATE TABLE it_imp_gen_src (
|
|
id BIGINT NOT NULL PRIMARY KEY,
|
|
a INT NOT NULL,
|
|
b INT NOT NULL,
|
|
total INT AS (a + b) STORED)`)
|
|
execSQL(t, db, "INSERT INTO it_imp_gen_src (id, a, b) VALUES (1,2,3),(2,10,20)")
|
|
dir = t.TempDir()
|
|
exportTable(t, dsn, "it_imp_gen_src", dir)
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_gen_dst")
|
|
execSQL(t, db, `CREATE TABLE it_imp_gen_dst (
|
|
id BIGINT NOT NULL PRIMARY KEY,
|
|
a INT NOT NULL,
|
|
b INT NOT NULL,
|
|
total INT AS (a + b) STORED)`)
|
|
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_gen_dst", "--assume-all-columns"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, want := range []struct{ id, a, b, total int }{{1, 2, 3, 5}, {2, 10, 20, 30}} {
|
|
var gotID, gotA, gotB, gotTotal int
|
|
if err := db.QueryRow("SELECT id, a, b, total FROM it_imp_gen_dst WHERE id=?", want.id).
|
|
Scan(&gotID, &gotA, &gotB, &gotTotal); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if gotID != want.id || gotA != want.a || gotB != want.b || gotTotal != want.total {
|
|
t.Fatalf("row %d = (%d,%d,%d,%d), want %+v", want.id, gotID, gotA, gotB, gotTotal, want)
|
|
}
|
|
}
|
|
}
|