Files
voidandClaude Sonnet 4.6 366d61ca25 Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-11-08 21:14:23 +08:00

430 lines
14 KiB
Go

package importer
import (
"context"
"database/sql"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"testing"
"github.com/go-sql-driver/mysql"
"archer/internal/app"
"archer/internal/cli"
)
func TestMain(m *testing.M) {
if dsn := os.Getenv("ARCHER_TEST_DSN"); dsn != "" {
if cfg, err := mysql.ParseDSN(dsn); err == nil && cfg.DBName != "" {
dbName := cfg.DBName
cfg.DBName = ""
if db, err := sql.Open("mysql", cfg.FormatDSN()); err == nil {
_, _ = db.Exec("CREATE DATABASE IF NOT EXISTS `" + dbName + "`")
_, _ = db.Exec("SET GLOBAL local_infile=ON")
_, _ = db.Exec("SET GLOBAL sql_mode=''")
db.Close()
}
}
}
os.Exit(m.Run())
}
func testDSN(t *testing.T) string {
t.Helper()
d := os.Getenv("ARCHER_TEST_DSN")
if d == "" {
t.Skip("ARCHER_TEST_DSN is not set; skipping MySQL integration test")
}
return d
}
func charsetDSN(t *testing.T, dsn string) string {
t.Helper()
cfg, err := mysql.ParseDSN(dsn)
if err != nil {
t.Fatal(err)
}
if cfg.Params == nil {
cfg.Params = map[string]string{}
}
cfg.Params["charset"] = "utf8mb4"
return cfg.FormatDSN()
}
func openDB(t *testing.T, dsn string) *sql.DB {
t.Helper()
db, err := sql.Open("mysql", dsn)
if err != nil {
t.Fatal(err)
}
if err := db.Ping(); err != nil {
t.Fatal(err)
}
return db
}
func execSQL(t *testing.T, db *sql.DB, q string, args ...any) {
t.Helper()
if _, err := db.Exec(q, args...); err != nil {
t.Fatalf("exec %q: %v", q, err)
}
}
func exportTable(t *testing.T, dsn, table, dir string, extra ...string) {
t.Helper()
args := []string{"--source", dsn, "--table", table, "--where", "1=1", "--file", dir, "--progress", "off"}
args = append(args, extra...)
cfg, err := cli.Parse(args, io.Discard)
if err != nil {
t.Fatal(err)
}
if err := app.Run(context.Background(), cfg); err != nil {
t.Fatalf("export %s: %v", table, err)
}
}
func importArchive(t *testing.T, dsn, dir string, extra ...string) error {
t.Helper()
args := []string{"--file", dir, "--dest", dsn, "--progress", "off"}
args = append(args, extra...)
cfg, err := cli.ParseImport(args, io.Discard)
if err != nil {
return err
}
return Run(context.Background(), cfg)
}
func countRows(t *testing.T, db *sql.DB, table string) int {
t.Helper()
var n int
if err := db.QueryRow("SELECT COUNT(*) FROM `" + table + "`").Scan(&n); err != nil {
t.Fatal(err)
}
return n
}
// fingerprint renders an ordered result set byte-for-byte, using hex so NULL,
// tabs, newlines and binary values are all distinguishable.
func fingerprint(t *testing.T, db *sql.DB, q string) string {
t.Helper()
rows, err := db.Query(q)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
cols, err := rows.Columns()
if err != nil {
t.Fatal(err)
}
var b strings.Builder
for rows.Next() {
raw := make([]sql.RawBytes, len(cols))
ptrs := make([]any, len(cols))
for i := range raw {
ptrs[i] = &raw[i]
}
if err := rows.Scan(ptrs...); err != nil {
t.Fatal(err)
}
for i, v := range raw {
if i > 0 {
b.WriteByte('|')
}
if v == nil {
b.WriteString("<NULL>")
} else {
fmt.Fprintf(&b, "%x", []byte(v))
}
}
b.WriteByte('\n')
}
if err := rows.Err(); err != nil {
t.Fatal(err)
}
return b.String()
}
func seedSimple(t *testing.T, db *sql.DB, table string) {
t.Helper()
execSQL(t, db, "DROP TABLE IF EXISTS `"+table+"`")
execSQL(t, db, "CREATE TABLE `"+table+"` (id BIGINT NOT NULL, v VARCHAR(64) NULL, PRIMARY KEY (id)) DEFAULT CHARSET=utf8mb4")
execSQL(t, db, "INSERT INTO `"+table+"` (id, v) VALUES (1,'a'),(2,'b'),(3,'c'),(4,'tab\\there'),(5,NULL)")
}
func freshTable(t *testing.T, db *sql.DB, table string) {
t.Helper()
execSQL(t, db, "DROP TABLE IF EXISTS `"+table+"`")
execSQL(t, db, "CREATE TABLE `"+table+"` (id BIGINT NOT NULL, v VARCHAR(64) NULL, PRIMARY KEY (id)) DEFAULT CHARSET=utf8mb4")
}
func TestIntegrationExportImport(t *testing.T) {
dsn := testDSN(t)
db := openDB(t, dsn)
defer db.Close()
seedSimple(t, db, "it_imp_src")
dir := t.TempDir()
exportTable(t, dsn, "it_imp_src", dir)
freshTable(t, db, "it_imp_dst")
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_dst", "--assume-all-columns"); err != nil {
t.Fatal(err)
}
if n := countRows(t, db, "it_imp_dst"); n != 5 {
t.Fatalf("imported %d rows", n)
}
if got, want := fingerprint(t, db, "SELECT id, v FROM it_imp_dst ORDER BY id"),
fingerprint(t, db, "SELECT id, v FROM it_imp_src ORDER BY id"); got != want {
t.Fatalf("rows differ:\n%s\nwant:\n%s", got, want)
}
}
func TestIntegrationCompression(t *testing.T) {
dsn := testDSN(t)
db := openDB(t, dsn)
defer db.Close()
seedSimple(t, db, "it_imp_comp_src")
for _, comp := range []string{"none", "gzip", "zstd", "xz"} {
dir := t.TempDir()
exportTable(t, dsn, "it_imp_comp_src", dir, "--compress", comp)
if files, _ := filepath.Glob(filepath.Join(dir, "*.dump*")); len(files) != 1 {
t.Fatalf("%s: shard files %v", comp, files)
}
dst := "it_imp_comp_" + comp
freshTable(t, db, dst)
if err := importArchive(t, dsn, dir, "--dest-table", dst, "--assume-all-columns"); err != nil {
t.Fatalf("%s: %v", comp, err)
}
if n := countRows(t, db, dst); n != 5 {
t.Fatalf("%s: %d rows", comp, n)
}
}
}
func TestIntegrationModes(t *testing.T) {
dsn := testDSN(t)
db := openDB(t, dsn)
defer db.Close()
seedSimple(t, db, "it_imp_modes_src")
dir := t.TempDir()
exportTable(t, dsn, "it_imp_modes_src", dir)
freshTable(t, db, "it_imp_modes_dst")
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns"); err != nil {
t.Fatal(err)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns"); err != nil {
t.Fatalf("ignore re-import must be idempotent: %v", err)
}
if n := countRows(t, db, "it_imp_modes_dst"); n != 5 {
t.Fatalf("ignore re-import changed row count: %d", n)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns", "--mode", "insert"); err == nil {
t.Fatal("insert mode must fail on duplicate keys")
}
execSQL(t, db, "UPDATE it_imp_modes_dst SET v='changed' WHERE id=2")
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns", "--mode", "replace"); err != nil {
t.Fatal(err)
}
var v string
if err := db.QueryRow("SELECT v FROM it_imp_modes_dst WHERE id=2").Scan(&v); err != nil {
t.Fatal(err)
}
if v != "b" {
t.Fatalf("replace did not restore value: %q", v)
}
}
func TestIntegrationVerifyAndPartial(t *testing.T) {
dsn := testDSN(t)
db := openDB(t, dsn)
defer db.Close()
seedSimple(t, db, "it_imp_verify_src")
dir := t.TempDir()
exportTable(t, dsn, "it_imp_verify_src", dir)
files, err := filepath.Glob(filepath.Join(dir, "*.dump"))
if err != nil || len(files) != 1 {
t.Fatalf("files %v err %v", files, err)
}
f, err := os.OpenFile(files[0], os.O_APPEND|os.O_WRONLY, 0o644)
if err != nil {
t.Fatal(err)
}
if _, err := f.WriteString("tampered\n"); err != nil {
t.Fatal(err)
}
f.Close()
freshTable(t, db, "it_imp_verify_dst")
err = importArchive(t, dsn, dir, "--dest-table", "it_imp_verify_dst", "--assume-all-columns")
if err == nil || !strings.Contains(err.Error(), "failed verification") {
t.Fatalf("got %v", err)
}
// Re-export a pristine archive for the completion-gate half.
dir = t.TempDir()
exportTable(t, dsn, "it_imp_verify_src", dir)
statePath := filepath.Join(dir, "state.json")
body, err := os.ReadFile(statePath)
if err != nil {
t.Fatal(err)
}
body = []byte(strings.Replace(string(body), `"complete": true`, `"complete": false`, 1))
if err := os.WriteFile(statePath, body, 0o644); err != nil {
t.Fatal(err)
}
freshTable(t, db, "it_imp_partial_dst")
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_partial_dst", "--assume-all-columns"); err == nil || !strings.Contains(err.Error(), "incomplete") {
t.Fatalf("incomplete archive must be refused, got %v", err)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_partial_dst", "--assume-all-columns", "--allow-partial"); err != nil {
t.Fatal(err)
}
if n := countRows(t, db, "it_imp_partial_dst"); n != 5 {
t.Fatalf("partial import: %d rows", n)
}
}
func TestIntegrationWorkersAndEnginesAgree(t *testing.T) {
dsn := testDSN(t)
db := openDB(t, dsn)
defer db.Close()
seedSimple(t, db, "it_imp_par_src")
for i := 6; i <= 20; i++ {
execSQL(t, db, "INSERT INTO it_imp_par_src (id, v) VALUES (?, ?)", i, fmt.Sprintf("v%d", i))
}
dir := t.TempDir()
exportTable(t, dsn, "it_imp_par_src", dir, "--shard-by", "count", "--shard-rows", "4", "--chunk-size", "3")
for _, dst := range []string{"it_imp_par_1", "it_imp_par_4", "it_imp_par_ins"} {
freshTable(t, db, dst)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_par_1", "--workers", "1", "--assume-all-columns"); err != nil {
t.Fatal(err)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_par_4", "--workers", "4", "--assume-all-columns"); err != nil {
t.Fatal(err)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_par_ins", "--engine", "insert", "--assume-all-columns"); err != nil {
t.Fatal(err)
}
want := fingerprint(t, db, "SELECT id, v FROM it_imp_par_1 ORDER BY id")
for _, dst := range []string{"it_imp_par_4", "it_imp_par_ins"} {
if got := fingerprint(t, db, "SELECT id, v FROM `"+dst+"` ORDER BY id"); got != want {
t.Fatalf("%s differs from serial load", dst)
}
}
}
func TestIntegrationFidelity(t *testing.T) {
dsn := charsetDSN(t, testDSN(t))
db := openDB(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_fid_src")
execSQL(t, db, `CREATE TABLE it_imp_fid_src (
id BIGINT NOT NULL PRIMARY KEY,
s VARCHAR(128) NULL,
b BLOB NULL,
n INT NULL) DEFAULT CHARSET=utf8mb4`)
execSQL(t, db, "INSERT INTO it_imp_fid_src (id, s, b, n) VALUES (?,?,?,?)", 1, nil, nil, nil)
execSQL(t, db, "INSERT INTO it_imp_fid_src (id, s, b, n) VALUES (?,?,?,?)", 2, "tab\there\nnewline\\back", []byte{0x00, 0x01, 0x02, 0x03, 0xff}, 0)
execSQL(t, db, "INSERT INTO it_imp_fid_src (id, s, b, n) VALUES (?,?,?,?)", 3, "中文😀", []byte("hi"), 42)
dir := t.TempDir()
exportTable(t, dsn, "it_imp_fid_src", dir)
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_fid_dst")
execSQL(t, db, `CREATE TABLE it_imp_fid_dst (
id BIGINT NOT NULL PRIMARY KEY,
s VARCHAR(128) NULL,
b BLOB NULL,
n INT NULL) DEFAULT CHARSET=utf8mb4`)
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_fid_dst", "--assume-all-columns", "--charset", "binary"); err != nil {
t.Fatal(err)
}
q := "SELECT id, s, HEX(b), n FROM %s ORDER BY id"
got := fingerprint(t, db, fmt.Sprintf(q, "it_imp_fid_dst"))
want := fingerprint(t, db, fmt.Sprintf(q, "it_imp_fid_src"))
if got != want {
t.Fatalf("fidelity mismatch:\n got: %s\nwant: %s", got, want)
}
}
func TestIntegrationColumnMappings(t *testing.T) {
dsn := charsetDSN(t, testDSN(t))
db := openDB(t, dsn)
defer db.Close()
// (a) archive-recorded column list
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_cols_src")
execSQL(t, db, "CREATE TABLE it_imp_cols_src (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "INSERT INTO it_imp_cols_src (id, a, b) VALUES (1,'a1','b1'),(2,'a2','b2')")
dir := t.TempDir()
exportTable(t, dsn, "it_imp_cols_src", dir, "--columns", "id,a")
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_cols_dst")
execSQL(t, db, "CREATE TABLE it_imp_cols_dst (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))")
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_cols_dst"); err != nil {
t.Fatal(err)
}
if got := fingerprint(t, db, "SELECT id, a, b FROM it_imp_cols_dst ORDER BY id"); got != "31|6131|<NULL>\n32|6132|<NULL>\n" {
t.Fatalf("archive-recorded mapping: %q", got)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_cols_dst", "--columns", "id,a"); err == nil {
t.Fatal("archive column list plus --columns must be rejected")
}
// (b) explicit --columns on a default (all-columns) archive
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_all_src")
execSQL(t, db, "CREATE TABLE it_imp_all_src (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "INSERT INTO it_imp_all_src (id, a, b) VALUES (1,'a1','b1')")
dir = t.TempDir()
exportTable(t, dsn, "it_imp_all_src", dir)
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_all_dst")
execSQL(t, db, "CREATE TABLE it_imp_all_dst (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))")
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst"); err == nil {
t.Fatal("default all-columns archive without a mapping must be refused")
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst", "--columns", "id,a,b"); err != nil {
t.Fatal(err)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst", "--columns", "id,nope"); err == nil || !strings.Contains(err.Error(), "does not exist") {
t.Fatalf("missing column: %v", err)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst", "--columns", "id"); err == nil || !strings.Contains(err.Error(), "fields") {
t.Fatalf("field count mismatch: %v", err)
}
// (c) positional mapping with a generated column
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_gen_src")
execSQL(t, db, `CREATE TABLE it_imp_gen_src (
id BIGINT NOT NULL PRIMARY KEY,
a INT NOT NULL,
b INT NOT NULL,
total INT AS (a + b) STORED)`)
execSQL(t, db, "INSERT INTO it_imp_gen_src (id, a, b) VALUES (1,2,3),(2,10,20)")
dir = t.TempDir()
exportTable(t, dsn, "it_imp_gen_src", dir)
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_gen_dst")
execSQL(t, db, `CREATE TABLE it_imp_gen_dst (
id BIGINT NOT NULL PRIMARY KEY,
a INT NOT NULL,
b INT NOT NULL,
total INT AS (a + b) STORED)`)
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_gen_dst", "--assume-all-columns"); err != nil {
t.Fatal(err)
}
for _, want := range []struct{ id, a, b, total int }{{1, 2, 3, 5}, {2, 10, 20, 30}} {
var gotID, gotA, gotB, gotTotal int
if err := db.QueryRow("SELECT id, a, b, total FROM it_imp_gen_dst WHERE id=?", want.id).
Scan(&gotID, &gotA, &gotB, &gotTotal); err != nil {
t.Fatal(err)
}
if gotID != want.id || gotA != want.a || gotB != want.b || gotTotal != want.total {
t.Fatalf("row %d = (%d,%d,%d,%d), want %+v", want.id, gotID, gotA, gotB, gotTotal, want)
}
}
}