package importer import ( "context" "database/sql" "fmt" "io" "os" "path/filepath" "strings" "testing" "github.com/go-sql-driver/mysql" "archer/internal/app" "archer/internal/cli" ) func TestMain(m *testing.M) { if dsn := os.Getenv("ARCHER_TEST_DSN"); dsn != "" { if cfg, err := mysql.ParseDSN(dsn); err == nil && cfg.DBName != "" { dbName := cfg.DBName cfg.DBName = "" if db, err := sql.Open("mysql", cfg.FormatDSN()); err == nil { _, _ = db.Exec("CREATE DATABASE IF NOT EXISTS `" + dbName + "`") _, _ = db.Exec("SET GLOBAL local_infile=ON") _, _ = db.Exec("SET GLOBAL sql_mode=''") db.Close() } } } os.Exit(m.Run()) } func testDSN(t *testing.T) string { t.Helper() d := os.Getenv("ARCHER_TEST_DSN") if d == "" { t.Skip("ARCHER_TEST_DSN is not set; skipping MySQL integration test") } return d } func charsetDSN(t *testing.T, dsn string) string { t.Helper() cfg, err := mysql.ParseDSN(dsn) if err != nil { t.Fatal(err) } if cfg.Params == nil { cfg.Params = map[string]string{} } cfg.Params["charset"] = "utf8mb4" return cfg.FormatDSN() } func openDB(t *testing.T, dsn string) *sql.DB { t.Helper() db, err := sql.Open("mysql", dsn) if err != nil { t.Fatal(err) } if err := db.Ping(); err != nil { t.Fatal(err) } return db } func execSQL(t *testing.T, db *sql.DB, q string, args ...any) { t.Helper() if _, err := db.Exec(q, args...); err != nil { t.Fatalf("exec %q: %v", q, err) } } func exportTable(t *testing.T, dsn, table, dir string, extra ...string) { t.Helper() args := []string{"--source", dsn, "--table", table, "--where", "1=1", "--file", dir, "--progress", "off"} args = append(args, extra...) cfg, err := cli.Parse(args, io.Discard) if err != nil { t.Fatal(err) } if err := app.Run(context.Background(), cfg); err != nil { t.Fatalf("export %s: %v", table, err) } } func importArchive(t *testing.T, dsn, dir string, extra ...string) error { t.Helper() args := []string{"--file", dir, "--dest", dsn, "--progress", "off"} args = append(args, extra...) cfg, err := cli.ParseImport(args, io.Discard) if err != nil { return err } return Run(context.Background(), cfg) } func countRows(t *testing.T, db *sql.DB, table string) int { t.Helper() var n int if err := db.QueryRow("SELECT COUNT(*) FROM `" + table + "`").Scan(&n); err != nil { t.Fatal(err) } return n } // fingerprint renders an ordered result set byte-for-byte, using hex so NULL, // tabs, newlines and binary values are all distinguishable. func fingerprint(t *testing.T, db *sql.DB, q string) string { t.Helper() rows, err := db.Query(q) if err != nil { t.Fatal(err) } defer rows.Close() cols, err := rows.Columns() if err != nil { t.Fatal(err) } var b strings.Builder for rows.Next() { raw := make([]sql.RawBytes, len(cols)) ptrs := make([]any, len(cols)) for i := range raw { ptrs[i] = &raw[i] } if err := rows.Scan(ptrs...); err != nil { t.Fatal(err) } for i, v := range raw { if i > 0 { b.WriteByte('|') } if v == nil { b.WriteString("") } else { fmt.Fprintf(&b, "%x", []byte(v)) } } b.WriteByte('\n') } if err := rows.Err(); err != nil { t.Fatal(err) } return b.String() } func seedSimple(t *testing.T, db *sql.DB, table string) { t.Helper() execSQL(t, db, "DROP TABLE IF EXISTS `"+table+"`") execSQL(t, db, "CREATE TABLE `"+table+"` (id BIGINT NOT NULL, v VARCHAR(64) NULL, PRIMARY KEY (id)) DEFAULT CHARSET=utf8mb4") execSQL(t, db, "INSERT INTO `"+table+"` (id, v) VALUES (1,'a'),(2,'b'),(3,'c'),(4,'tab\\there'),(5,NULL)") } func freshTable(t *testing.T, db *sql.DB, table string) { t.Helper() execSQL(t, db, "DROP TABLE IF EXISTS `"+table+"`") execSQL(t, db, "CREATE TABLE `"+table+"` (id BIGINT NOT NULL, v VARCHAR(64) NULL, PRIMARY KEY (id)) DEFAULT CHARSET=utf8mb4") } func TestIntegrationExportImport(t *testing.T) { dsn := testDSN(t) db := openDB(t, dsn) defer db.Close() seedSimple(t, db, "it_imp_src") dir := t.TempDir() exportTable(t, dsn, "it_imp_src", dir) freshTable(t, db, "it_imp_dst") if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_dst", "--assume-all-columns"); err != nil { t.Fatal(err) } if n := countRows(t, db, "it_imp_dst"); n != 5 { t.Fatalf("imported %d rows", n) } if got, want := fingerprint(t, db, "SELECT id, v FROM it_imp_dst ORDER BY id"), fingerprint(t, db, "SELECT id, v FROM it_imp_src ORDER BY id"); got != want { t.Fatalf("rows differ:\n%s\nwant:\n%s", got, want) } } func TestIntegrationCompression(t *testing.T) { dsn := testDSN(t) db := openDB(t, dsn) defer db.Close() seedSimple(t, db, "it_imp_comp_src") for _, comp := range []string{"none", "gzip", "zstd", "xz"} { dir := t.TempDir() exportTable(t, dsn, "it_imp_comp_src", dir, "--compress", comp) if files, _ := filepath.Glob(filepath.Join(dir, "*.dump*")); len(files) != 1 { t.Fatalf("%s: shard files %v", comp, files) } dst := "it_imp_comp_" + comp freshTable(t, db, dst) if err := importArchive(t, dsn, dir, "--dest-table", dst, "--assume-all-columns"); err != nil { t.Fatalf("%s: %v", comp, err) } if n := countRows(t, db, dst); n != 5 { t.Fatalf("%s: %d rows", comp, n) } } } func TestIntegrationModes(t *testing.T) { dsn := testDSN(t) db := openDB(t, dsn) defer db.Close() seedSimple(t, db, "it_imp_modes_src") dir := t.TempDir() exportTable(t, dsn, "it_imp_modes_src", dir) freshTable(t, db, "it_imp_modes_dst") if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns"); err != nil { t.Fatal(err) } if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns"); err != nil { t.Fatalf("ignore re-import must be idempotent: %v", err) } if n := countRows(t, db, "it_imp_modes_dst"); n != 5 { t.Fatalf("ignore re-import changed row count: %d", n) } if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns", "--mode", "insert"); err == nil { t.Fatal("insert mode must fail on duplicate keys") } execSQL(t, db, "UPDATE it_imp_modes_dst SET v='changed' WHERE id=2") if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns", "--mode", "replace"); err != nil { t.Fatal(err) } var v string if err := db.QueryRow("SELECT v FROM it_imp_modes_dst WHERE id=2").Scan(&v); err != nil { t.Fatal(err) } if v != "b" { t.Fatalf("replace did not restore value: %q", v) } } func TestIntegrationVerifyAndPartial(t *testing.T) { dsn := testDSN(t) db := openDB(t, dsn) defer db.Close() seedSimple(t, db, "it_imp_verify_src") dir := t.TempDir() exportTable(t, dsn, "it_imp_verify_src", dir) files, err := filepath.Glob(filepath.Join(dir, "*.dump")) if err != nil || len(files) != 1 { t.Fatalf("files %v err %v", files, err) } f, err := os.OpenFile(files[0], os.O_APPEND|os.O_WRONLY, 0o644) if err != nil { t.Fatal(err) } if _, err := f.WriteString("tampered\n"); err != nil { t.Fatal(err) } f.Close() freshTable(t, db, "it_imp_verify_dst") err = importArchive(t, dsn, dir, "--dest-table", "it_imp_verify_dst", "--assume-all-columns") if err == nil || !strings.Contains(err.Error(), "failed verification") { t.Fatalf("got %v", err) } // Re-export a pristine archive for the completion-gate half. dir = t.TempDir() exportTable(t, dsn, "it_imp_verify_src", dir) statePath := filepath.Join(dir, "state.json") body, err := os.ReadFile(statePath) if err != nil { t.Fatal(err) } body = []byte(strings.Replace(string(body), `"complete": true`, `"complete": false`, 1)) if err := os.WriteFile(statePath, body, 0o644); err != nil { t.Fatal(err) } freshTable(t, db, "it_imp_partial_dst") if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_partial_dst", "--assume-all-columns"); err == nil || !strings.Contains(err.Error(), "incomplete") { t.Fatalf("incomplete archive must be refused, got %v", err) } if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_partial_dst", "--assume-all-columns", "--allow-partial"); err != nil { t.Fatal(err) } if n := countRows(t, db, "it_imp_partial_dst"); n != 5 { t.Fatalf("partial import: %d rows", n) } } func TestIntegrationWorkersAndEnginesAgree(t *testing.T) { dsn := testDSN(t) db := openDB(t, dsn) defer db.Close() seedSimple(t, db, "it_imp_par_src") for i := 6; i <= 20; i++ { execSQL(t, db, "INSERT INTO it_imp_par_src (id, v) VALUES (?, ?)", i, fmt.Sprintf("v%d", i)) } dir := t.TempDir() exportTable(t, dsn, "it_imp_par_src", dir, "--shard-by", "count", "--shard-rows", "4", "--chunk-size", "3") for _, dst := range []string{"it_imp_par_1", "it_imp_par_4", "it_imp_par_ins"} { freshTable(t, db, dst) } if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_par_1", "--workers", "1", "--assume-all-columns"); err != nil { t.Fatal(err) } if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_par_4", "--workers", "4", "--assume-all-columns"); err != nil { t.Fatal(err) } if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_par_ins", "--engine", "insert", "--assume-all-columns"); err != nil { t.Fatal(err) } want := fingerprint(t, db, "SELECT id, v FROM it_imp_par_1 ORDER BY id") for _, dst := range []string{"it_imp_par_4", "it_imp_par_ins"} { if got := fingerprint(t, db, "SELECT id, v FROM `"+dst+"` ORDER BY id"); got != want { t.Fatalf("%s differs from serial load", dst) } } } func TestIntegrationFidelity(t *testing.T) { dsn := charsetDSN(t, testDSN(t)) db := openDB(t, dsn) defer db.Close() execSQL(t, db, "DROP TABLE IF EXISTS it_imp_fid_src") execSQL(t, db, `CREATE TABLE it_imp_fid_src ( id BIGINT NOT NULL PRIMARY KEY, s VARCHAR(128) NULL, b BLOB NULL, n INT NULL) DEFAULT CHARSET=utf8mb4`) execSQL(t, db, "INSERT INTO it_imp_fid_src (id, s, b, n) VALUES (?,?,?,?)", 1, nil, nil, nil) execSQL(t, db, "INSERT INTO it_imp_fid_src (id, s, b, n) VALUES (?,?,?,?)", 2, "tab\there\nnewline\\back", []byte{0x00, 0x01, 0x02, 0x03, 0xff}, 0) execSQL(t, db, "INSERT INTO it_imp_fid_src (id, s, b, n) VALUES (?,?,?,?)", 3, "δΈ­ζ–‡πŸ˜€", []byte("hi"), 42) dir := t.TempDir() exportTable(t, dsn, "it_imp_fid_src", dir) execSQL(t, db, "DROP TABLE IF EXISTS it_imp_fid_dst") execSQL(t, db, `CREATE TABLE it_imp_fid_dst ( id BIGINT NOT NULL PRIMARY KEY, s VARCHAR(128) NULL, b BLOB NULL, n INT NULL) DEFAULT CHARSET=utf8mb4`) if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_fid_dst", "--assume-all-columns", "--charset", "binary"); err != nil { t.Fatal(err) } q := "SELECT id, s, HEX(b), n FROM %s ORDER BY id" got := fingerprint(t, db, fmt.Sprintf(q, "it_imp_fid_dst")) want := fingerprint(t, db, fmt.Sprintf(q, "it_imp_fid_src")) if got != want { t.Fatalf("fidelity mismatch:\n got: %s\nwant: %s", got, want) } } func TestIntegrationColumnMappings(t *testing.T) { dsn := charsetDSN(t, testDSN(t)) db := openDB(t, dsn) defer db.Close() // (a) archive-recorded column list execSQL(t, db, "DROP TABLE IF EXISTS it_imp_cols_src") execSQL(t, db, "CREATE TABLE it_imp_cols_src (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))") execSQL(t, db, "INSERT INTO it_imp_cols_src (id, a, b) VALUES (1,'a1','b1'),(2,'a2','b2')") dir := t.TempDir() exportTable(t, dsn, "it_imp_cols_src", dir, "--columns", "id,a") execSQL(t, db, "DROP TABLE IF EXISTS it_imp_cols_dst") execSQL(t, db, "CREATE TABLE it_imp_cols_dst (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))") if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_cols_dst"); err != nil { t.Fatal(err) } if got := fingerprint(t, db, "SELECT id, a, b FROM it_imp_cols_dst ORDER BY id"); got != "31|6131|\n32|6132|\n" { t.Fatalf("archive-recorded mapping: %q", got) } if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_cols_dst", "--columns", "id,a"); err == nil { t.Fatal("archive column list plus --columns must be rejected") } // (b) explicit --columns on a default (all-columns) archive execSQL(t, db, "DROP TABLE IF EXISTS it_imp_all_src") execSQL(t, db, "CREATE TABLE it_imp_all_src (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))") execSQL(t, db, "INSERT INTO it_imp_all_src (id, a, b) VALUES (1,'a1','b1')") dir = t.TempDir() exportTable(t, dsn, "it_imp_all_src", dir) execSQL(t, db, "DROP TABLE IF EXISTS it_imp_all_dst") execSQL(t, db, "CREATE TABLE it_imp_all_dst (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))") if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst"); err == nil { t.Fatal("default all-columns archive without a mapping must be refused") } if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst", "--columns", "id,a,b"); err != nil { t.Fatal(err) } if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst", "--columns", "id,nope"); err == nil || !strings.Contains(err.Error(), "does not exist") { t.Fatalf("missing column: %v", err) } if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst", "--columns", "id"); err == nil || !strings.Contains(err.Error(), "fields") { t.Fatalf("field count mismatch: %v", err) } // (c) positional mapping with a generated column execSQL(t, db, "DROP TABLE IF EXISTS it_imp_gen_src") execSQL(t, db, `CREATE TABLE it_imp_gen_src ( id BIGINT NOT NULL PRIMARY KEY, a INT NOT NULL, b INT NOT NULL, total INT AS (a + b) STORED)`) execSQL(t, db, "INSERT INTO it_imp_gen_src (id, a, b) VALUES (1,2,3),(2,10,20)") dir = t.TempDir() exportTable(t, dsn, "it_imp_gen_src", dir) execSQL(t, db, "DROP TABLE IF EXISTS it_imp_gen_dst") execSQL(t, db, `CREATE TABLE it_imp_gen_dst ( id BIGINT NOT NULL PRIMARY KEY, a INT NOT NULL, b INT NOT NULL, total INT AS (a + b) STORED)`) if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_gen_dst", "--assume-all-columns"); err != nil { t.Fatal(err) } for _, want := range []struct{ id, a, b, total int }{{1, 2, 3, 5}, {2, 10, 20, 30}} { var gotID, gotA, gotB, gotTotal int if err := db.QueryRow("SELECT id, a, b, total FROM it_imp_gen_dst WHERE id=?", want.id). Scan(&gotID, &gotA, &gotB, &gotTotal); err != nil { t.Fatal(err) } if gotID != want.id || gotA != want.a || gotB != want.b || gotTotal != want.total { t.Fatalf("row %d = (%d,%d,%d,%d), want %+v", want.id, gotID, gotA, gotB, gotTotal, want) } } }