package app import ( "bytes" "compress/gzip" "context" "database/sql" "encoding/csv" "encoding/json" "fmt" "io" "os" "path/filepath" "sort" "strconv" "strings" "sync" "testing" "github.com/go-sql-driver/mysql" "github.com/klauspost/compress/zstd" "github.com/ulikunitz/xz" "archer/internal/cli" ) func testDSN(t *testing.T) string { d := os.Getenv("ARCHER_TEST_DSN") if d == "" { t.Skip("ARCHER_TEST_DSN is not set; skipping MySQL integration test") } return d } func TestMain(m *testing.M) { if dsn := os.Getenv("ARCHER_TEST_DSN"); dsn != "" { cfg, err := mysql.ParseDSN(dsn) if err == nil && cfg.DBName != "" { dbName := cfg.DBName cfg.DBName = "" db, err := sql.Open("mysql", cfg.FormatDSN()) if err == nil { _, _ = db.Exec("CREATE DATABASE IF NOT EXISTS `" + dbName + "`") _, _ = db.Exec("SET GLOBAL local_infile=ON") _, _ = db.Exec("SET GLOBAL sql_mode=''") db.Close() } } } os.Exit(m.Run()) } func openRoot(t *testing.T, dsn string) *sql.DB { t.Helper() cfg, err := mysql.ParseDSN(dsn) if err != nil { t.Fatal(err) } db, err := sql.Open("mysql", cfg.FormatDSN()) if err != nil { t.Fatal(err) } if err := db.Ping(); err != nil { t.Fatal(err) } return db } func execSQL(t *testing.T, db *sql.DB, q string, args ...any) { t.Helper() if _, err := db.Exec(q, args...); err != nil { t.Fatalf("exec %q: %v", q, err) } } var stdoutMu sync.Mutex var stderrMu sync.Mutex func captureStdout(fn func() error) (string, error) { stdoutMu.Lock() defer stdoutMu.Unlock() old := os.Stdout r, w, err := os.Pipe() if err != nil { return "", err } os.Stdout = w var buf bytes.Buffer done := make(chan struct{}) go func() { _, _ = io.Copy(&buf, r) close(done) }() runErr := fn() _ = w.Close() os.Stdout = old <-done return buf.String(), runErr } func captureStderr(fn func() error) (string, error) { stderrMu.Lock() defer stderrMu.Unlock() old := os.Stderr r, w, err := os.Pipe() if err != nil { return "", err } os.Stderr = w var buf bytes.Buffer done := make(chan struct{}) go func() { _, _ = io.Copy(&buf, r) close(done) }() runErr := fn() _ = w.Close() os.Stderr = old <-done return buf.String(), runErr } func runCLI(t *testing.T, ctx context.Context, args ...string) (string, error) { t.Helper() cfg, err := cli.Parse(args, io.Discard) if err != nil { return "", err } return captureStdout(func() error { return Run(ctx, cfg) }) } func runCLIStderr(t *testing.T, ctx context.Context, args ...string) (string, error) { t.Helper() cfg, err := cli.Parse(args, io.Discard) if err != nil { return "", err } return captureStderr(func() error { return Run(ctx, cfg) }) } func baseCLI(dsn, table, out string, extra ...string) []string { args := []string{ "--source", dsn, "--table", table, "--where", "1=1", "--file", out, } return append(args, extra...) } func listFiles(t *testing.T, dir, pattern string) []string { t.Helper() matches, err := filepath.Glob(filepath.Join(dir, pattern)) if err != nil { t.Fatal(err) } sort.Strings(matches) return matches } func dumpIDs(t *testing.T, path string) []int { t.Helper() b, err := os.ReadFile(path) if err != nil { t.Fatal(err) } var ids []int for _, line := range strings.Split(strings.TrimRight(string(b), "\n"), "\n") { if line == "" { continue } id, err := strconv.Atoi(strings.SplitN(line, "\t", 2)[0]) if err != nil { t.Fatalf("%s: bad line %q", path, line) } ids = append(ids, id) } return ids } func TestIntegrationCountShards(t *testing.T) { dsn := testDSN(t) db := openRoot(t, dsn) defer db.Close() execSQL(t, db, "DROP TABLE IF EXISTS it_count") execSQL(t, db, "CREATE TABLE it_count (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))") var b strings.Builder b.WriteString("INSERT INTO it_count (id, v) VALUES ") for i := 1; i <= 25; i++ { if i > 1 { b.WriteString(",") } fmt.Fprintf(&b, "(%d,'v%d')", i, i) } execSQL(t, db, b.String()) out := t.TempDir() _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_count", out, "--shard-by", "count", "--shard-rows", "10", "--chunk-size", "7", "--format", "dump")...) if err != nil { t.Fatal(err) } files := listFiles(t, out, "*.dump") if len(files) != 3 { t.Fatalf("got %d shard files: %v", len(files), files) } wantNames := []string{"000001.1-10.dump", "000002.11-20.dump", "000003.21-25.dump"} var got []int for i, f := range files { if filepath.Base(f) != wantNames[i] { t.Fatalf("file %d = %s want %s", i, filepath.Base(f), wantNames[i]) } got = append(got, dumpIDs(t, f)...) } if len(got) != 25 { t.Fatalf("got %d rows", len(got)) } for i, id := range got { if id != i+1 { t.Fatalf("row %d = %d, shards are not contiguous", i, id) } } entries := readManifest(t, out) if len(entries) != 3 { t.Fatalf("manifest entries %d", len(entries)) } for i, e := range entries { if e.Seq != int64(i+1) || e.Rows != int64(len(dumpIDs(t, filepath.Join(out, e.Path)))) { t.Fatalf("entry %d %+v", i, e) } if e.Bytes == 0 || e.SHA == "" { t.Fatalf("entry %d missing bytes/hash", i) } } if _, err := os.Stat(filepath.Join(out, "state.json")); err != nil { t.Fatal(err) } var complete bool st, err := os.ReadFile(filepath.Join(out, "state.json")) if err != nil { t.Fatal(err) } complete = bytes.Contains(st, []byte(`"complete": true`)) if !complete { t.Fatalf("state not complete: %s", st) } } type manifestEntry struct { Seq int64 `json:"seq"` Path string `json:"path"` Date string `json:"date"` Rows int64 `json:"rows"` Bytes int64 `json:"bytes"` SHA string `json:"sha256"` PKLo string `json:"pk_lo"` PKHi string `json:"pk_hi"` } func readManifest(t *testing.T, dir string) []manifestEntry { t.Helper() b, err := os.ReadFile(filepath.Join(dir, "manifest.jsonl")) if err != nil { t.Fatal(err) } var out []manifestEntry for _, line := range strings.Split(strings.TrimSpace(string(b)), "\n") { if line == "" { continue } var e manifestEntry if err := json.Unmarshal([]byte(line), &e); err != nil { t.Fatal(err) } out = append(out, e) } return out } func TestIntegrationDatePathsAgree(t *testing.T) { dsn := testDSN(t) db := openRoot(t, dsn) defer db.Close() for _, spec := range []struct{ name, index string }{ {"it_date_a", "KEY da (created_at, id)"}, {"it_date_b", "KEY db (created_at)"}, } { execSQL(t, db, "DROP TABLE IF EXISTS "+spec.name) execSQL(t, db, "CREATE TABLE "+spec.name+` ( id BIGINT NOT NULL, v VARCHAR(32), created_at DATETIME NULL, PRIMARY KEY (id), `+spec.index+`)`) execSQL(t, db, `INSERT INTO `+spec.name+` (id, v, created_at) VALUES (1,'a','2025-01-01 10:00:00'), (2,'b','2025-01-01 12:00:00'), (3,'c','2025-01-02 01:00:00'), (4,'d',NULL), (5,'e','2025-01-01 11:00:00'), (6,'f','2025-01-03 00:00:00')`) } outA := t.TempDir() outB := t.TempDir() if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_a", outA, "--shard-by", "date", "--shard-column", "created_at", "--format", "dump", "--chunk-size", "2")...); err != nil { t.Fatal(err) } if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_b", outB, "--shard-by", "date", "--shard-column", "created_at", "--format", "dump", "--chunk-size", "2")...); err != nil { t.Fatal(err) } a := readTree(t, outA) b := readTree(t, outB) if len(a) != len(b) { t.Fatalf("file counts differ: %d vs %d (%v / %v)", len(a), len(b), a, b) } for k, va := range a { if b[k] != va { t.Fatalf("%s:\nA=%q\nB=%q", k, va, b[k]) } } if _, ok := a["date=_null/000001.4-4.dump"]; !ok { t.Fatalf("NULL row not routed to _null: %v", a) } if len(a) != 4 { t.Fatalf("expected 4 shards (null + 3 days), got %v", a) } } func readTree(t *testing.T, dir string) map[string]string { t.Helper() out := map[string]string{} err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { if err != nil || d.IsDir() { return err } if !strings.HasSuffix(d.Name(), ".dump") { return nil } rel, _ := filepath.Rel(dir, path) b, err := os.ReadFile(path) if err != nil { return err } out[filepath.ToSlash(rel)] = string(b) return nil }) if err != nil { t.Fatal(err) } return out } func TestIntegrationZeroDateAndNull(t *testing.T) { dsn := testDSN(t) db := openRoot(t, dsn) defer db.Close() execSQL(t, db, "DROP TABLE IF EXISTS it_zero") execSQL(t, db, `CREATE TABLE it_zero ( id BIGINT NOT NULL, d DATE NULL, PRIMARY KEY (id), KEY dz (d, id))`) execSQL(t, db, `INSERT INTO it_zero (id, d) VALUES (1, NULL), (2, '0000-00-00'), (3, '2024-01-01'), (4, '2024-01-01')`) out := t.TempDir() if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_zero", out, "--shard-by", "date", "--shard-column", "d")...); err == nil { t.Fatal("expected zero-date error") } else if !strings.Contains(err.Error(), "zero date") { t.Fatalf("unexpected error: %v", err) } out = t.TempDir() if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_zero", out, "--shard-by", "date", "--shard-column", "d", "--zero-date", "bucket")...); err != nil { t.Fatal(err) } tree := readTree(t, out) for _, want := range []string{ "date=_null/000001.1-1.dump", "date=0000-00-00/000001.2-2.dump", "date=2024-01-01/000001.3-4.dump", } { if _, ok := tree[want]; !ok { t.Fatalf("missing %s in %v", want, tree) } } } func TestIntegrationEpochUnits(t *testing.T) { dsn := testDSN(t) db := openRoot(t, dsn) defer db.Close() execSQL(t, db, "DROP TABLE IF EXISTS it_epoch") execSQL(t, db, `CREATE TABLE it_epoch ( id BIGINT NOT NULL, ts BIGINT NOT NULL, PRIMARY KEY (id), KEY e (ts, id))`) execSQL(t, db, `INSERT INTO it_epoch (id, ts) VALUES (1, 1717200000), (2, 1717200001)`) for _, tc := range []struct { unit string vals string }{ {"epoch-seconds", "(3, 1717200000), (4, 1717200001)"}, {"epoch-milliseconds", "(3, 1717200000000), (4, 1717200001000)"}, {"epoch-microseconds", "(3, 1717200000000000), (4, 1717200001000000)"}, } { table := "it_epoch_" + strings.Split(tc.unit, "-")[1] execSQL(t, db, "DROP TABLE IF EXISTS "+table) execSQL(t, db, `CREATE TABLE `+table+` ( id BIGINT NOT NULL, ts BIGINT NOT NULL, PRIMARY KEY (id), KEY e (ts, id))`) execSQL(t, db, "INSERT INTO "+table+" (id, ts) VALUES "+tc.vals) out := t.TempDir() if _, err := runCLI(t, context.Background(), baseCLI(dsn, table, out, "--shard-by", "date", "--shard-column", "ts", "--shard-column-type", tc.unit, "--shard-time-zone", "UTC")...); err != nil { t.Fatalf("%s: %v", tc.unit, err) } tree := readTree(t, out) if len(tree) != 1 { t.Fatalf("%s: expected one date bucket, got %v", tc.unit, tree) } for k := range tree { if !strings.Contains(k, "date=2024-06-01") { t.Fatalf("%s: wrong bucket %s", tc.unit, k) } } } } func TestIntegrationNoIndexFailFast(t *testing.T) { dsn := testDSN(t) db := openRoot(t, dsn) defer db.Close() execSQL(t, db, "DROP TABLE IF EXISTS it_noindex") execSQL(t, db, `CREATE TABLE it_noindex ( id BIGINT NOT NULL, d DATETIME NULL, PRIMARY KEY (id))`) execSQL(t, db, `INSERT INTO it_noindex (id, d) VALUES (1,'2024-01-01 00:00:00')`) out := t.TempDir() if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_noindex", out, "--shard-by", "date", "--shard-column", "d")...); err == nil { t.Fatal("expected no-index error") } else if !strings.Contains(err.Error(), "no usable index") { t.Fatalf("unexpected error: %v", err) } out = t.TempDir() if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_noindex", out, "--shard-by", "date", "--shard-column", "d", "--allow-full-scan")...); err != nil { t.Fatal(err) } if _, err := os.Stat(filepath.Join(out, "date=2024-01-01")); err != nil { t.Fatal(err) } } func TestIntegrationDryRunCreatesNothing(t *testing.T) { dsn := testDSN(t) db := openRoot(t, dsn) defer db.Close() execSQL(t, db, "DROP TABLE IF EXISTS it_dry") execSQL(t, db, "CREATE TABLE it_dry (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))") execSQL(t, db, "INSERT INTO it_dry (id, v) VALUES (1,'a')") out := filepath.Join(t.TempDir(), "never-created") stdout, err := runCLI(t, context.Background(), baseCLI(dsn, "it_dry", out, "--dry-run")...) if err != nil { t.Fatal(err) } if _, statErr := os.Stat(out); !os.IsNotExist(statErr) { t.Fatal("dry-run created the output directory") } if !strings.Contains(stdout, "== first SELECT ==") || !strings.Contains(stdout, "config hash") { t.Fatalf("dry-run output incomplete:\n%s", stdout) } } func TestIntegrationResumeAfterInterrupt(t *testing.T) { dsn := testDSN(t) db := openRoot(t, dsn) defer db.Close() execSQL(t, db, "DROP TABLE IF EXISTS it_resume") execSQL(t, db, "CREATE TABLE it_resume (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))") var b strings.Builder b.WriteString("INSERT INTO it_resume (id, v) VALUES ") const total = 20000 for i := 1; i <= total; i++ { if i > 1 { b.WriteString(",") } fmt.Fprintf(&b, "(%d,'v')", i) } execSQL(t, db, b.String()) out := t.TempDir() args := baseCLI(dsn, "it_resume", out, "--shard-by", "count", "--shard-rows", "137", "--chunk-size", "50", "--progress", "100") ctx, cancel := context.WithCancel(context.Background()) defer cancel() stderrMu.Lock() old := os.Stderr r, w, _ := os.Pipe() os.Stderr = w cfg, err := cli.Parse(args, io.Discard) if err != nil { os.Stderr = old stderrMu.Unlock() t.Fatal(err) } firstErr := make(chan error, 1) go func() { firstErr <- Run(ctx, cfg) }() go func() { buf := make([]byte, 4096) for { n, err := r.Read(buf) if n > 0 && strings.Contains(string(buf[:n]), "progress:") { cancel() } if err != nil { return } } }() err = <-firstErr _ = w.Close() os.Stderr = old stderrMu.Unlock() if err == nil { t.Fatal("first run should have been interrupted") } if _, err := runCLI(t, context.Background(), args...); err != nil { t.Fatal(err) } seen := map[int]bool{} for _, f := range listFiles(t, out, "*.dump") { for _, id := range dumpIDs(t, f) { seen[id] = true } } for i := 1; i <= total; i++ { if !seen[i] { t.Fatalf("row %d missing after resume", i) } } st, err := os.ReadFile(filepath.Join(out, "state.json")) if err != nil { t.Fatal(err) } if !bytes.Contains(st, []byte(`"complete": true`)) { t.Fatalf("state not complete: %s", st) } var saved struct { RowsDone int64 `json:"rows_done"` } if err := json.Unmarshal(st, &saved); err != nil { t.Fatal(err) } if saved.RowsDone != total { t.Fatalf("rows_done = %d after resume, want cumulative %d", saved.RowsDone, total) } if _, err := filepath.Glob(filepath.Join(out, "*.partial")); err != nil { t.Fatal(err) } } func TestIntegrationProgressModes(t *testing.T) { dsn := testDSN(t) db := openRoot(t, dsn) defer db.Close() execSQL(t, db, "DROP TABLE IF EXISTS it_progress") execSQL(t, db, "CREATE TABLE it_progress (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))") execSQL(t, db, "INSERT INTO it_progress (id, v) VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e')") for _, extra := range [][]string{ nil, {"--progress", "off"}, {"--progress", "0"}, } { out := t.TempDir() args := baseCLI(dsn, "it_progress", out, extra...) errOut, err := runCLIStderr(t, context.Background(), args...) if err != nil { t.Fatalf("%v: %v", extra, err) } if strings.Contains(errOut, "progress:") || strings.Contains(errOut, "done rows=") { t.Fatalf("%v: unexpected progress output %q", extra, errOut) } } out := t.TempDir() errOut, err := runCLIStderr(t, context.Background(), baseCLI(dsn, "it_progress", out, "--progress", "on")...) if err != nil { t.Fatal(err) } if !strings.Contains(errOut, "done rows=5") { t.Fatalf("forced progress missing done summary: %q", errOut) } if strings.Contains(errOut, "\r") { t.Fatalf("redirected progress must not use carriage returns: %q", errOut) } out = t.TempDir() errOut, err = runCLIStderr(t, context.Background(), baseCLI(dsn, "it_progress", out, "--progress", "1")...) if err != nil { t.Fatal(err) } if !strings.Contains(errOut, "progress: rows=5") { t.Fatalf("row-threshold progress line missing: %q", errOut) } } func TestIntegrationEstimateMetrics(t *testing.T) { dsn := testDSN(t) db := openRoot(t, dsn) defer db.Close() execSQL(t, db, "DROP TABLE IF EXISTS it_estimate") execSQL(t, db, `CREATE TABLE it_estimate ( id BIGINT NOT NULL, v VARCHAR(32), created_at DATETIME NULL, PRIMARY KEY (id), KEY de (created_at, id))`) execSQL(t, db, `INSERT INTO it_estimate (id, v, created_at) VALUES (1,'a','2025-01-01 01:00:00'), (2,'b','2025-01-01 02:00:00'), (3,'c','2025-01-02 01:00:00'), (4,'d','2025-01-03 01:00:00'), (5,'e','2025-01-03 02:00:00')`) out := t.TempDir() errOut, err := runCLIStderr(t, context.Background(), baseCLI(dsn, "it_estimate", out, "--progress", "1")...) if err != nil { t.Fatal(err) } if !strings.Contains(errOut, "estimate=key") { t.Fatalf("primary key estimate missing: %q", errOut) } out = t.TempDir() errOut, err = runCLIStderr(t, context.Background(), baseCLI(dsn, "it_estimate", out, "--progress", "1", "--shard-by", "date", "--shard-column", "created_at")...) if err != nil { t.Fatal(err) } if !strings.Contains(errOut, "estimate=day") { t.Fatalf("day estimate missing: %q", errOut) } } func TestIntegrationImportDump(t *testing.T) { dsn := testDSN(t) db := openRoot(t, dsn) defer db.Close() execSQL(t, db, "DROP TABLE IF EXISTS it_import_src") execSQL(t, db, "CREATE TABLE it_import_src (id BIGINT NOT NULL, v VARCHAR(64), PRIMARY KEY (id))") execSQL(t, db, `INSERT INTO it_import_src (id, v) VALUES (1,'plain'), (2,'tab\there'), (3,'line\nbreak'), (4,'back\\slash')`) for _, comp := range []string{"none", "gzip", "zstd", "xz"} { out := t.TempDir() args := baseCLI(dsn, "it_import_src", out, "--format", "dump", "--compress", comp) if _, err := runCLI(t, context.Background(), args...); err != nil { t.Fatalf("%s: %v", comp, err) } files := listFiles(t, out, "*.dump*") if len(files) != 1 { t.Fatalf("%s: files %v", comp, files) } loadPath := files[0] if comp != "none" { loadPath = filepath.Join(t.TempDir(), "decompressed.dump") if err := decompressFile(files[0], loadPath, comp); err != nil { t.Fatal(err) } } execSQL(t, db, "DROP TABLE IF EXISTS it_import_dst") execSQL(t, db, "CREATE TABLE it_import_dst (id BIGINT NOT NULL, v VARCHAR(64), PRIMARY KEY (id))") loadLocal(t, dsn, loadPath, "it_import_dst") var n int if err := db.QueryRow("SELECT COUNT(*) FROM it_import_dst").Scan(&n); err != nil { t.Fatal(err) } if n != 4 { t.Fatalf("%s: loaded %d rows", comp, n) } var v string if err := db.QueryRow("SELECT v FROM it_import_dst WHERE id=3").Scan(&v); err != nil { t.Fatal(err) } if v != "line\nbreak" { t.Fatalf("%s: round trip mismatch %q", comp, v) } } } func decompressFile(src, dst, comp string) error { in, err := os.Open(src) if err != nil { return err } defer in.Close() var r io.Reader switch comp { case "gzip": gr, err := gzip.NewReader(in) if err != nil { return err } defer gr.Close() r = gr case "zstd": zr, err := zstd.NewReader(in) if err != nil { return err } defer zr.Close() r = zr case "xz": xr, err := xz.NewReader(in) if err != nil { return err } r = xr default: r = in } out, err := os.Create(dst) if err != nil { return err } defer out.Close() _, err = io.Copy(out, r) return err } func loadLocal(t *testing.T, dsn, path, table string) { t.Helper() cfg, err := mysql.ParseDSN(dsn) if err != nil { t.Fatal(err) } cfg.AllowAllFiles = true db, err := sql.Open("mysql", cfg.FormatDSN()) if err != nil { t.Fatal(err) } defer db.Close() p := filepath.ToSlash(path) p = strings.ReplaceAll(p, `'`, `''`) q := fmt.Sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE %s", p, table) if _, err := db.Exec(q); err != nil { t.Fatalf("load data: %v", err) } } func TestIntegrationCSVAndJSONL(t *testing.T) { dsn := testDSN(t) db := openRoot(t, dsn) defer db.Close() execSQL(t, db, "DROP TABLE IF EXISTS it_fmt") execSQL(t, db, `CREATE TABLE it_fmt ( id BIGINT NOT NULL, name VARCHAR(64), price DECIMAL(10,2), created DATETIME, data BLOB, PRIMARY KEY (id))`) execSQL(t, db, `INSERT INTO it_fmt (id, name, price, created, data) VALUES (1,'a,\"b\"',12.30,'2024-01-02 03:04:05',X'6869'), (2,'',NULL,NULL,NULL)`) // CSV out := t.TempDir() if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_fmt", out, "--format", "csv")...); err != nil { t.Fatal(err) } files := listFiles(t, out, "*.csv") if len(files) != 1 { t.Fatalf("csv files %v", files) } f, err := os.Open(files[0]) if err != nil { t.Fatal(err) } recs, err := csv.NewReader(f).ReadAll() f.Close() if err != nil { t.Fatal(err) } if len(recs) != 3 { t.Fatalf("csv records %d", len(recs)) } if recs[0][0] != "id" || recs[0][1] != "name" { t.Fatalf("header %v", recs[0]) } if recs[1][1] != `a,"b"` || recs[1][2] != "12.30" { t.Fatalf("row %v", recs[1]) } if recs[2][2] != `\N` { t.Fatalf("NULL decimal %q", recs[2][2]) } // JSONL out = t.TempDir() if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_fmt", out, "--format", "jsonl")...); err != nil { t.Fatal(err) } files = listFiles(t, out, "*.jsonl") var shardFiles []string for _, f := range files { if filepath.Base(f) != "manifest.jsonl" { shardFiles = append(shardFiles, f) } } files = shardFiles if len(files) != 1 { t.Fatalf("jsonl files %v", files) } b, err := os.ReadFile(files[0]) if err != nil { t.Fatal(err) } lines := strings.Split(strings.TrimSpace(string(b)), "\n") if len(lines) != 2 { t.Fatalf("jsonl lines %d", len(lines)) } var obj map[string]any if err := json.Unmarshal([]byte(lines[0]), &obj); err != nil { t.Fatal(err) } if obj["price"] != "12.30" { t.Fatalf("price %v", obj["price"]) } if obj["created"] != "2024-01-02T03:04:05Z" { t.Fatalf("created %v", obj["created"]) } if obj["data"] != "aGk=" { t.Fatalf("data %v", obj["data"]) } var second map[string]any if err := json.Unmarshal([]byte(lines[1]), &second); err != nil { t.Fatal(err) } if second["price"] != nil || second["created"] != nil { t.Fatalf("null mapping %v", second) } } func TestIntegrationPurgeAndDest(t *testing.T) { dsn := testDSN(t) db := openRoot(t, dsn) defer db.Close() execSQL(t, db, "DROP TABLE IF EXISTS it_purge_src") execSQL(t, db, "DROP TABLE IF EXISTS it_purge_dst") execSQL(t, db, "CREATE TABLE it_purge_src (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))") execSQL(t, db, "CREATE TABLE it_purge_dst (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))") execSQL(t, db, "INSERT INTO it_purge_src (id, v) VALUES (1,'a'),(2,'b'),(3,'c')") out := t.TempDir() args := []string{ "--source", dsn, "--table", "it_purge_src", "--where", "1=1", "--file", out, "--dest", dsn, "--dest-table", "it_purge_dst", "--purge", "--chunk-size", "2", } if _, err := runCLI(t, context.Background(), args...); err != nil { t.Fatal(err) } var srcCount, dstCount int db.QueryRow("SELECT COUNT(*) FROM it_purge_src").Scan(&srcCount) db.QueryRow("SELECT COUNT(*) FROM it_purge_dst").Scan(&dstCount) if srcCount != 0 || dstCount != 3 { t.Fatalf("src=%d dst=%d", srcCount, dstCount) } if len(listFiles(t, out, "*.dump")) != 1 { t.Fatal("expected one shard") } } func TestIntegrationDateBulkDelete(t *testing.T) { dsn := testDSN(t) db := openRoot(t, dsn) defer db.Close() seed := func(table string) { execSQL(t, db, "DROP TABLE IF EXISTS "+table) execSQL(t, db, `CREATE TABLE `+table+` ( id BIGINT NOT NULL AUTO_INCREMENT, v VARCHAR(32), created_at DATETIME NOT NULL, PRIMARY KEY (id), KEY d (created_at, id))`) execSQL(t, db, `INSERT INTO `+table+` (id, v, created_at) VALUES (1,'a','2025-01-01 00:00:00'), (2,'b','2025-01-02 00:00:00'), (3,'c','2025-01-03 00:00:00'), (4,'d','2025-01-04 00:00:00')`) } seed("it_date_bulk") out := t.TempDir() if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_bulk", out, "--shard-by", "date", "--shard-column", "created_at", "--purge", "--bulk-delete", "--safe-auto-increment=false", "--chunk-size", "2")...); err != nil { t.Fatal(err) } var count int db.QueryRow("SELECT COUNT(*) FROM it_date_bulk").Scan(&count) if count != 0 { t.Fatalf("safe off: %d rows left", count) } if got := len(readTree(t, out)); got != 4 { t.Fatalf("expected 4 shards, got %d", got) } seed("it_date_bulk_safe") out = t.TempDir() if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_bulk_safe", out, "--shard-by", "date", "--shard-column", "created_at", "--purge", "--bulk-delete", "--chunk-size", "2")...); err != nil { t.Fatal(err) } db.QueryRow("SELECT COUNT(*) FROM it_date_bulk_safe").Scan(&count) var maxID int64 db.QueryRow("SELECT MAX(id) FROM it_date_bulk_safe").Scan(&maxID) if count != 1 || maxID != 4 { t.Fatalf("safe on: count=%d max=%d", count, maxID) } } func TestIntegrationCheckpointConfigMismatch(t *testing.T) { dsn := testDSN(t) db := openRoot(t, dsn) defer db.Close() execSQL(t, db, "DROP TABLE IF EXISTS it_mismatch") execSQL(t, db, "CREATE TABLE it_mismatch (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))") execSQL(t, db, "INSERT INTO it_mismatch (id, v) VALUES (1,'a'),(2,'b')") out := t.TempDir() if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_mismatch", out)...); err != nil { t.Fatal(err) } // Remove the complete flag so resume logic runs. stPath := filepath.Join(out, "state.json") b, _ := os.ReadFile(stPath) b = bytes.Replace(b, []byte(`"complete": true`), []byte(`"complete": false`), 1) if err := os.WriteFile(stPath, b, 0o644); err != nil { t.Fatal(err) } args := baseCLI(dsn, "it_mismatch", out) for i := range args { if args[i] == "1=1" { args[i] = "id > 0" } } if _, err := runCLI(t, context.Background(), args...); err == nil { t.Fatal("expected config mismatch error") } else if !strings.Contains(err.Error(), "different configuration") { t.Fatalf("unexpected error: %v", err) } args = append(args, "--force-resume") if _, err := runCLI(t, context.Background(), args...); err != nil { t.Fatalf("force resume: %v", err) } }