Files
archer/internal/app/integration_test.go
T
voidandClaude Sonnet 4.6 366d61ca25 Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-11-08 21:14:23 +08:00

970 lines
26 KiB
Go

package app
import (
"bytes"
"compress/gzip"
"context"
"database/sql"
"encoding/csv"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"testing"
"github.com/go-sql-driver/mysql"
"github.com/klauspost/compress/zstd"
"github.com/ulikunitz/xz"
"archer/internal/cli"
)
func testDSN(t *testing.T) string {
d := os.Getenv("ARCHER_TEST_DSN")
if d == "" {
t.Skip("ARCHER_TEST_DSN is not set; skipping MySQL integration test")
}
return d
}
func TestMain(m *testing.M) {
if dsn := os.Getenv("ARCHER_TEST_DSN"); dsn != "" {
cfg, err := mysql.ParseDSN(dsn)
if err == nil && cfg.DBName != "" {
dbName := cfg.DBName
cfg.DBName = ""
db, err := sql.Open("mysql", cfg.FormatDSN())
if err == nil {
_, _ = db.Exec("CREATE DATABASE IF NOT EXISTS `" + dbName + "`")
_, _ = db.Exec("SET GLOBAL local_infile=ON")
_, _ = db.Exec("SET GLOBAL sql_mode=''")
db.Close()
}
}
}
os.Exit(m.Run())
}
func openRoot(t *testing.T, dsn string) *sql.DB {
t.Helper()
cfg, err := mysql.ParseDSN(dsn)
if err != nil {
t.Fatal(err)
}
db, err := sql.Open("mysql", cfg.FormatDSN())
if err != nil {
t.Fatal(err)
}
if err := db.Ping(); err != nil {
t.Fatal(err)
}
return db
}
func execSQL(t *testing.T, db *sql.DB, q string, args ...any) {
t.Helper()
if _, err := db.Exec(q, args...); err != nil {
t.Fatalf("exec %q: %v", q, err)
}
}
var stdoutMu sync.Mutex
var stderrMu sync.Mutex
func captureStdout(fn func() error) (string, error) {
stdoutMu.Lock()
defer stdoutMu.Unlock()
old := os.Stdout
r, w, err := os.Pipe()
if err != nil {
return "", err
}
os.Stdout = w
var buf bytes.Buffer
done := make(chan struct{})
go func() {
_, _ = io.Copy(&buf, r)
close(done)
}()
runErr := fn()
_ = w.Close()
os.Stdout = old
<-done
return buf.String(), runErr
}
func captureStderr(fn func() error) (string, error) {
stderrMu.Lock()
defer stderrMu.Unlock()
old := os.Stderr
r, w, err := os.Pipe()
if err != nil {
return "", err
}
os.Stderr = w
var buf bytes.Buffer
done := make(chan struct{})
go func() {
_, _ = io.Copy(&buf, r)
close(done)
}()
runErr := fn()
_ = w.Close()
os.Stderr = old
<-done
return buf.String(), runErr
}
func runCLI(t *testing.T, ctx context.Context, args ...string) (string, error) {
t.Helper()
cfg, err := cli.Parse(args, io.Discard)
if err != nil {
return "", err
}
return captureStdout(func() error { return Run(ctx, cfg) })
}
func runCLIStderr(t *testing.T, ctx context.Context, args ...string) (string, error) {
t.Helper()
cfg, err := cli.Parse(args, io.Discard)
if err != nil {
return "", err
}
return captureStderr(func() error { return Run(ctx, cfg) })
}
func baseCLI(dsn, table, out string, extra ...string) []string {
args := []string{
"--source", dsn,
"--table", table,
"--where", "1=1",
"--file", out,
}
return append(args, extra...)
}
func listFiles(t *testing.T, dir, pattern string) []string {
t.Helper()
matches, err := filepath.Glob(filepath.Join(dir, pattern))
if err != nil {
t.Fatal(err)
}
sort.Strings(matches)
return matches
}
func dumpIDs(t *testing.T, path string) []int {
t.Helper()
b, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var ids []int
for _, line := range strings.Split(strings.TrimRight(string(b), "\n"), "\n") {
if line == "" {
continue
}
id, err := strconv.Atoi(strings.SplitN(line, "\t", 2)[0])
if err != nil {
t.Fatalf("%s: bad line %q", path, line)
}
ids = append(ids, id)
}
return ids
}
func TestIntegrationCountShards(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_count")
execSQL(t, db, "CREATE TABLE it_count (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
var b strings.Builder
b.WriteString("INSERT INTO it_count (id, v) VALUES ")
for i := 1; i <= 25; i++ {
if i > 1 {
b.WriteString(",")
}
fmt.Fprintf(&b, "(%d,'v%d')", i, i)
}
execSQL(t, db, b.String())
out := t.TempDir()
_, err := runCLI(t, context.Background(), baseCLI(dsn, "it_count", out,
"--shard-by", "count", "--shard-rows", "10", "--chunk-size", "7", "--format", "dump")...)
if err != nil {
t.Fatal(err)
}
files := listFiles(t, out, "*.dump")
if len(files) != 3 {
t.Fatalf("got %d shard files: %v", len(files), files)
}
wantNames := []string{"000001.1-10.dump", "000002.11-20.dump", "000003.21-25.dump"}
var got []int
for i, f := range files {
if filepath.Base(f) != wantNames[i] {
t.Fatalf("file %d = %s want %s", i, filepath.Base(f), wantNames[i])
}
got = append(got, dumpIDs(t, f)...)
}
if len(got) != 25 {
t.Fatalf("got %d rows", len(got))
}
for i, id := range got {
if id != i+1 {
t.Fatalf("row %d = %d, shards are not contiguous", i, id)
}
}
entries := readManifest(t, out)
if len(entries) != 3 {
t.Fatalf("manifest entries %d", len(entries))
}
for i, e := range entries {
if e.Seq != int64(i+1) || e.Rows != int64(len(dumpIDs(t, filepath.Join(out, e.Path)))) {
t.Fatalf("entry %d %+v", i, e)
}
if e.Bytes == 0 || e.SHA == "" {
t.Fatalf("entry %d missing bytes/hash", i)
}
}
if _, err := os.Stat(filepath.Join(out, "state.json")); err != nil {
t.Fatal(err)
}
var complete bool
st, err := os.ReadFile(filepath.Join(out, "state.json"))
if err != nil {
t.Fatal(err)
}
complete = bytes.Contains(st, []byte(`"complete": true`))
if !complete {
t.Fatalf("state not complete: %s", st)
}
}
type manifestEntry struct {
Seq int64 `json:"seq"`
Path string `json:"path"`
Date string `json:"date"`
Rows int64 `json:"rows"`
Bytes int64 `json:"bytes"`
SHA string `json:"sha256"`
PKLo string `json:"pk_lo"`
PKHi string `json:"pk_hi"`
}
func readManifest(t *testing.T, dir string) []manifestEntry {
t.Helper()
b, err := os.ReadFile(filepath.Join(dir, "manifest.jsonl"))
if err != nil {
t.Fatal(err)
}
var out []manifestEntry
for _, line := range strings.Split(strings.TrimSpace(string(b)), "\n") {
if line == "" {
continue
}
var e manifestEntry
if err := json.Unmarshal([]byte(line), &e); err != nil {
t.Fatal(err)
}
out = append(out, e)
}
return out
}
func TestIntegrationDatePathsAgree(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
for _, spec := range []struct{ name, index string }{
{"it_date_a", "KEY da (created_at, id)"},
{"it_date_b", "KEY db (created_at)"},
} {
execSQL(t, db, "DROP TABLE IF EXISTS "+spec.name)
execSQL(t, db, "CREATE TABLE "+spec.name+` (
id BIGINT NOT NULL,
v VARCHAR(32),
created_at DATETIME NULL,
PRIMARY KEY (id),
`+spec.index+`)`)
execSQL(t, db, `INSERT INTO `+spec.name+` (id, v, created_at) VALUES
(1,'a','2025-01-01 10:00:00'),
(2,'b','2025-01-01 12:00:00'),
(3,'c','2025-01-02 01:00:00'),
(4,'d',NULL),
(5,'e','2025-01-01 11:00:00'),
(6,'f','2025-01-03 00:00:00')`)
}
outA := t.TempDir()
outB := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_a", outA,
"--shard-by", "date", "--shard-column", "created_at", "--format", "dump", "--chunk-size", "2")...); err != nil {
t.Fatal(err)
}
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_b", outB,
"--shard-by", "date", "--shard-column", "created_at", "--format", "dump", "--chunk-size", "2")...); err != nil {
t.Fatal(err)
}
a := readTree(t, outA)
b := readTree(t, outB)
if len(a) != len(b) {
t.Fatalf("file counts differ: %d vs %d (%v / %v)", len(a), len(b), a, b)
}
for k, va := range a {
if b[k] != va {
t.Fatalf("%s:\nA=%q\nB=%q", k, va, b[k])
}
}
if _, ok := a["date=_null/000001.4-4.dump"]; !ok {
t.Fatalf("NULL row not routed to _null: %v", a)
}
if len(a) != 4 {
t.Fatalf("expected 4 shards (null + 3 days), got %v", a)
}
}
func readTree(t *testing.T, dir string) map[string]string {
t.Helper()
out := map[string]string{}
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
if err != nil || d.IsDir() {
return err
}
if !strings.HasSuffix(d.Name(), ".dump") {
return nil
}
rel, _ := filepath.Rel(dir, path)
b, err := os.ReadFile(path)
if err != nil {
return err
}
out[filepath.ToSlash(rel)] = string(b)
return nil
})
if err != nil {
t.Fatal(err)
}
return out
}
func TestIntegrationZeroDateAndNull(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_zero")
execSQL(t, db, `CREATE TABLE it_zero (
id BIGINT NOT NULL,
d DATE NULL,
PRIMARY KEY (id),
KEY dz (d, id))`)
execSQL(t, db, `INSERT INTO it_zero (id, d) VALUES
(1, NULL), (2, '0000-00-00'), (3, '2024-01-01'), (4, '2024-01-01')`)
out := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_zero", out,
"--shard-by", "date", "--shard-column", "d")...); err == nil {
t.Fatal("expected zero-date error")
} else if !strings.Contains(err.Error(), "zero date") {
t.Fatalf("unexpected error: %v", err)
}
out = t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_zero", out,
"--shard-by", "date", "--shard-column", "d", "--zero-date", "bucket")...); err != nil {
t.Fatal(err)
}
tree := readTree(t, out)
for _, want := range []string{
"date=_null/000001.1-1.dump",
"date=0000-00-00/000001.2-2.dump",
"date=2024-01-01/000001.3-4.dump",
} {
if _, ok := tree[want]; !ok {
t.Fatalf("missing %s in %v", want, tree)
}
}
}
func TestIntegrationEpochUnits(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_epoch")
execSQL(t, db, `CREATE TABLE it_epoch (
id BIGINT NOT NULL,
ts BIGINT NOT NULL,
PRIMARY KEY (id),
KEY e (ts, id))`)
execSQL(t, db, `INSERT INTO it_epoch (id, ts) VALUES
(1, 1717200000), (2, 1717200001)`)
for _, tc := range []struct {
unit string
vals string
}{
{"epoch-seconds", "(3, 1717200000), (4, 1717200001)"},
{"epoch-milliseconds", "(3, 1717200000000), (4, 1717200001000)"},
{"epoch-microseconds", "(3, 1717200000000000), (4, 1717200001000000)"},
} {
table := "it_epoch_" + strings.Split(tc.unit, "-")[1]
execSQL(t, db, "DROP TABLE IF EXISTS "+table)
execSQL(t, db, `CREATE TABLE `+table+` (
id BIGINT NOT NULL, ts BIGINT NOT NULL,
PRIMARY KEY (id), KEY e (ts, id))`)
execSQL(t, db, "INSERT INTO "+table+" (id, ts) VALUES "+tc.vals)
out := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, table, out,
"--shard-by", "date", "--shard-column", "ts", "--shard-column-type", tc.unit,
"--shard-time-zone", "UTC")...); err != nil {
t.Fatalf("%s: %v", tc.unit, err)
}
tree := readTree(t, out)
if len(tree) != 1 {
t.Fatalf("%s: expected one date bucket, got %v", tc.unit, tree)
}
for k := range tree {
if !strings.Contains(k, "date=2024-06-01") {
t.Fatalf("%s: wrong bucket %s", tc.unit, k)
}
}
}
}
func TestIntegrationNoIndexFailFast(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_noindex")
execSQL(t, db, `CREATE TABLE it_noindex (
id BIGINT NOT NULL, d DATETIME NULL, PRIMARY KEY (id))`)
execSQL(t, db, `INSERT INTO it_noindex (id, d) VALUES (1,'2024-01-01 00:00:00')`)
out := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_noindex", out,
"--shard-by", "date", "--shard-column", "d")...); err == nil {
t.Fatal("expected no-index error")
} else if !strings.Contains(err.Error(), "no usable index") {
t.Fatalf("unexpected error: %v", err)
}
out = t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_noindex", out,
"--shard-by", "date", "--shard-column", "d", "--allow-full-scan")...); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(out, "date=2024-01-01")); err != nil {
t.Fatal(err)
}
}
func TestIntegrationDryRunCreatesNothing(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_dry")
execSQL(t, db, "CREATE TABLE it_dry (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "INSERT INTO it_dry (id, v) VALUES (1,'a')")
out := filepath.Join(t.TempDir(), "never-created")
stdout, err := runCLI(t, context.Background(), baseCLI(dsn, "it_dry", out, "--dry-run")...)
if err != nil {
t.Fatal(err)
}
if _, statErr := os.Stat(out); !os.IsNotExist(statErr) {
t.Fatal("dry-run created the output directory")
}
if !strings.Contains(stdout, "== first SELECT ==") || !strings.Contains(stdout, "config hash") {
t.Fatalf("dry-run output incomplete:\n%s", stdout)
}
}
func TestIntegrationResumeAfterInterrupt(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_resume")
execSQL(t, db, "CREATE TABLE it_resume (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
var b strings.Builder
b.WriteString("INSERT INTO it_resume (id, v) VALUES ")
const total = 20000
for i := 1; i <= total; i++ {
if i > 1 {
b.WriteString(",")
}
fmt.Fprintf(&b, "(%d,'v')", i)
}
execSQL(t, db, b.String())
out := t.TempDir()
args := baseCLI(dsn, "it_resume", out, "--shard-by", "count", "--shard-rows", "137",
"--chunk-size", "50", "--progress", "100")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
stderrMu.Lock()
old := os.Stderr
r, w, _ := os.Pipe()
os.Stderr = w
cfg, err := cli.Parse(args, io.Discard)
if err != nil {
os.Stderr = old
stderrMu.Unlock()
t.Fatal(err)
}
firstErr := make(chan error, 1)
go func() { firstErr <- Run(ctx, cfg) }()
go func() {
buf := make([]byte, 4096)
for {
n, err := r.Read(buf)
if n > 0 && strings.Contains(string(buf[:n]), "progress:") {
cancel()
}
if err != nil {
return
}
}
}()
err = <-firstErr
_ = w.Close()
os.Stderr = old
stderrMu.Unlock()
if err == nil {
t.Fatal("first run should have been interrupted")
}
if _, err := runCLI(t, context.Background(), args...); err != nil {
t.Fatal(err)
}
seen := map[int]bool{}
for _, f := range listFiles(t, out, "*.dump") {
for _, id := range dumpIDs(t, f) {
seen[id] = true
}
}
for i := 1; i <= total; i++ {
if !seen[i] {
t.Fatalf("row %d missing after resume", i)
}
}
st, err := os.ReadFile(filepath.Join(out, "state.json"))
if err != nil {
t.Fatal(err)
}
if !bytes.Contains(st, []byte(`"complete": true`)) {
t.Fatalf("state not complete: %s", st)
}
var saved struct {
RowsDone int64 `json:"rows_done"`
}
if err := json.Unmarshal(st, &saved); err != nil {
t.Fatal(err)
}
if saved.RowsDone != total {
t.Fatalf("rows_done = %d after resume, want cumulative %d", saved.RowsDone, total)
}
if _, err := filepath.Glob(filepath.Join(out, "*.partial")); err != nil {
t.Fatal(err)
}
}
func TestIntegrationProgressModes(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_progress")
execSQL(t, db, "CREATE TABLE it_progress (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "INSERT INTO it_progress (id, v) VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e')")
for _, extra := range [][]string{
nil,
{"--progress", "off"},
{"--progress", "0"},
} {
out := t.TempDir()
args := baseCLI(dsn, "it_progress", out, extra...)
errOut, err := runCLIStderr(t, context.Background(), args...)
if err != nil {
t.Fatalf("%v: %v", extra, err)
}
if strings.Contains(errOut, "progress:") || strings.Contains(errOut, "done rows=") {
t.Fatalf("%v: unexpected progress output %q", extra, errOut)
}
}
out := t.TempDir()
errOut, err := runCLIStderr(t, context.Background(), baseCLI(dsn, "it_progress", out, "--progress", "on")...)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(errOut, "done rows=5") {
t.Fatalf("forced progress missing done summary: %q", errOut)
}
if strings.Contains(errOut, "\r") {
t.Fatalf("redirected progress must not use carriage returns: %q", errOut)
}
out = t.TempDir()
errOut, err = runCLIStderr(t, context.Background(), baseCLI(dsn, "it_progress", out, "--progress", "1")...)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(errOut, "progress: rows=5") {
t.Fatalf("row-threshold progress line missing: %q", errOut)
}
}
func TestIntegrationEstimateMetrics(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_estimate")
execSQL(t, db, `CREATE TABLE it_estimate (
id BIGINT NOT NULL,
v VARCHAR(32),
created_at DATETIME NULL,
PRIMARY KEY (id),
KEY de (created_at, id))`)
execSQL(t, db, `INSERT INTO it_estimate (id, v, created_at) VALUES
(1,'a','2025-01-01 01:00:00'),
(2,'b','2025-01-01 02:00:00'),
(3,'c','2025-01-02 01:00:00'),
(4,'d','2025-01-03 01:00:00'),
(5,'e','2025-01-03 02:00:00')`)
out := t.TempDir()
errOut, err := runCLIStderr(t, context.Background(), baseCLI(dsn, "it_estimate", out, "--progress", "1")...)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(errOut, "estimate=key") {
t.Fatalf("primary key estimate missing: %q", errOut)
}
out = t.TempDir()
errOut, err = runCLIStderr(t, context.Background(),
baseCLI(dsn, "it_estimate", out, "--progress", "1", "--shard-by", "date", "--shard-column", "created_at")...)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(errOut, "estimate=day") {
t.Fatalf("day estimate missing: %q", errOut)
}
}
func TestIntegrationImportDump(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_import_src")
execSQL(t, db, "CREATE TABLE it_import_src (id BIGINT NOT NULL, v VARCHAR(64), PRIMARY KEY (id))")
execSQL(t, db, `INSERT INTO it_import_src (id, v) VALUES
(1,'plain'), (2,'tab\there'), (3,'line\nbreak'), (4,'back\\slash')`)
for _, comp := range []string{"none", "gzip", "zstd", "xz"} {
out := t.TempDir()
args := baseCLI(dsn, "it_import_src", out, "--format", "dump", "--compress", comp)
if _, err := runCLI(t, context.Background(), args...); err != nil {
t.Fatalf("%s: %v", comp, err)
}
files := listFiles(t, out, "*.dump*")
if len(files) != 1 {
t.Fatalf("%s: files %v", comp, files)
}
loadPath := files[0]
if comp != "none" {
loadPath = filepath.Join(t.TempDir(), "decompressed.dump")
if err := decompressFile(files[0], loadPath, comp); err != nil {
t.Fatal(err)
}
}
execSQL(t, db, "DROP TABLE IF EXISTS it_import_dst")
execSQL(t, db, "CREATE TABLE it_import_dst (id BIGINT NOT NULL, v VARCHAR(64), PRIMARY KEY (id))")
loadLocal(t, dsn, loadPath, "it_import_dst")
var n int
if err := db.QueryRow("SELECT COUNT(*) FROM it_import_dst").Scan(&n); err != nil {
t.Fatal(err)
}
if n != 4 {
t.Fatalf("%s: loaded %d rows", comp, n)
}
var v string
if err := db.QueryRow("SELECT v FROM it_import_dst WHERE id=3").Scan(&v); err != nil {
t.Fatal(err)
}
if v != "line\nbreak" {
t.Fatalf("%s: round trip mismatch %q", comp, v)
}
}
}
func decompressFile(src, dst, comp string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
var r io.Reader
switch comp {
case "gzip":
gr, err := gzip.NewReader(in)
if err != nil {
return err
}
defer gr.Close()
r = gr
case "zstd":
zr, err := zstd.NewReader(in)
if err != nil {
return err
}
defer zr.Close()
r = zr
case "xz":
xr, err := xz.NewReader(in)
if err != nil {
return err
}
r = xr
default:
r = in
}
out, err := os.Create(dst)
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, r)
return err
}
func loadLocal(t *testing.T, dsn, path, table string) {
t.Helper()
cfg, err := mysql.ParseDSN(dsn)
if err != nil {
t.Fatal(err)
}
cfg.AllowAllFiles = true
db, err := sql.Open("mysql", cfg.FormatDSN())
if err != nil {
t.Fatal(err)
}
defer db.Close()
p := filepath.ToSlash(path)
p = strings.ReplaceAll(p, `'`, `''`)
q := fmt.Sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE %s", p, table)
if _, err := db.Exec(q); err != nil {
t.Fatalf("load data: %v", err)
}
}
func TestIntegrationCSVAndJSONL(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_fmt")
execSQL(t, db, `CREATE TABLE it_fmt (
id BIGINT NOT NULL, name VARCHAR(64), price DECIMAL(10,2), created DATETIME,
data BLOB, PRIMARY KEY (id))`)
execSQL(t, db, `INSERT INTO it_fmt (id, name, price, created, data) VALUES
(1,'a,\"b\"',12.30,'2024-01-02 03:04:05',X'6869'), (2,'',NULL,NULL,NULL)`)
// CSV
out := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_fmt", out, "--format", "csv")...); err != nil {
t.Fatal(err)
}
files := listFiles(t, out, "*.csv")
if len(files) != 1 {
t.Fatalf("csv files %v", files)
}
f, err := os.Open(files[0])
if err != nil {
t.Fatal(err)
}
recs, err := csv.NewReader(f).ReadAll()
f.Close()
if err != nil {
t.Fatal(err)
}
if len(recs) != 3 {
t.Fatalf("csv records %d", len(recs))
}
if recs[0][0] != "id" || recs[0][1] != "name" {
t.Fatalf("header %v", recs[0])
}
if recs[1][1] != `a,"b"` || recs[1][2] != "12.30" {
t.Fatalf("row %v", recs[1])
}
if recs[2][2] != `\N` {
t.Fatalf("NULL decimal %q", recs[2][2])
}
// JSONL
out = t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_fmt", out, "--format", "jsonl")...); err != nil {
t.Fatal(err)
}
files = listFiles(t, out, "*.jsonl")
var shardFiles []string
for _, f := range files {
if filepath.Base(f) != "manifest.jsonl" {
shardFiles = append(shardFiles, f)
}
}
files = shardFiles
if len(files) != 1 {
t.Fatalf("jsonl files %v", files)
}
b, err := os.ReadFile(files[0])
if err != nil {
t.Fatal(err)
}
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
if len(lines) != 2 {
t.Fatalf("jsonl lines %d", len(lines))
}
var obj map[string]any
if err := json.Unmarshal([]byte(lines[0]), &obj); err != nil {
t.Fatal(err)
}
if obj["price"] != "12.30" {
t.Fatalf("price %v", obj["price"])
}
if obj["created"] != "2024-01-02T03:04:05Z" {
t.Fatalf("created %v", obj["created"])
}
if obj["data"] != "aGk=" {
t.Fatalf("data %v", obj["data"])
}
var second map[string]any
if err := json.Unmarshal([]byte(lines[1]), &second); err != nil {
t.Fatal(err)
}
if second["price"] != nil || second["created"] != nil {
t.Fatalf("null mapping %v", second)
}
}
func TestIntegrationPurgeAndDest(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_purge_src")
execSQL(t, db, "DROP TABLE IF EXISTS it_purge_dst")
execSQL(t, db, "CREATE TABLE it_purge_src (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "CREATE TABLE it_purge_dst (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "INSERT INTO it_purge_src (id, v) VALUES (1,'a'),(2,'b'),(3,'c')")
out := t.TempDir()
args := []string{
"--source", dsn, "--table", "it_purge_src", "--where", "1=1",
"--file", out, "--dest", dsn, "--dest-table", "it_purge_dst",
"--purge", "--chunk-size", "2",
}
if _, err := runCLI(t, context.Background(), args...); err != nil {
t.Fatal(err)
}
var srcCount, dstCount int
db.QueryRow("SELECT COUNT(*) FROM it_purge_src").Scan(&srcCount)
db.QueryRow("SELECT COUNT(*) FROM it_purge_dst").Scan(&dstCount)
if srcCount != 0 || dstCount != 3 {
t.Fatalf("src=%d dst=%d", srcCount, dstCount)
}
if len(listFiles(t, out, "*.dump")) != 1 {
t.Fatal("expected one shard")
}
}
func TestIntegrationDateBulkDelete(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
seed := func(table string) {
execSQL(t, db, "DROP TABLE IF EXISTS "+table)
execSQL(t, db, `CREATE TABLE `+table+` (
id BIGINT NOT NULL AUTO_INCREMENT,
v VARCHAR(32),
created_at DATETIME NOT NULL,
PRIMARY KEY (id),
KEY d (created_at, id))`)
execSQL(t, db, `INSERT INTO `+table+` (id, v, created_at) VALUES
(1,'a','2025-01-01 00:00:00'),
(2,'b','2025-01-02 00:00:00'),
(3,'c','2025-01-03 00:00:00'),
(4,'d','2025-01-04 00:00:00')`)
}
seed("it_date_bulk")
out := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_bulk", out,
"--shard-by", "date", "--shard-column", "created_at",
"--purge", "--bulk-delete", "--safe-auto-increment=false", "--chunk-size", "2")...); err != nil {
t.Fatal(err)
}
var count int
db.QueryRow("SELECT COUNT(*) FROM it_date_bulk").Scan(&count)
if count != 0 {
t.Fatalf("safe off: %d rows left", count)
}
if got := len(readTree(t, out)); got != 4 {
t.Fatalf("expected 4 shards, got %d", got)
}
seed("it_date_bulk_safe")
out = t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_bulk_safe", out,
"--shard-by", "date", "--shard-column", "created_at",
"--purge", "--bulk-delete", "--chunk-size", "2")...); err != nil {
t.Fatal(err)
}
db.QueryRow("SELECT COUNT(*) FROM it_date_bulk_safe").Scan(&count)
var maxID int64
db.QueryRow("SELECT MAX(id) FROM it_date_bulk_safe").Scan(&maxID)
if count != 1 || maxID != 4 {
t.Fatalf("safe on: count=%d max=%d", count, maxID)
}
}
func TestIntegrationCheckpointConfigMismatch(t *testing.T) {
dsn := testDSN(t)
db := openRoot(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_mismatch")
execSQL(t, db, "CREATE TABLE it_mismatch (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "INSERT INTO it_mismatch (id, v) VALUES (1,'a'),(2,'b')")
out := t.TempDir()
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_mismatch", out)...); err != nil {
t.Fatal(err)
}
// Remove the complete flag so resume logic runs.
stPath := filepath.Join(out, "state.json")
b, _ := os.ReadFile(stPath)
b = bytes.Replace(b, []byte(`"complete": true`), []byte(`"complete": false`), 1)
if err := os.WriteFile(stPath, b, 0o644); err != nil {
t.Fatal(err)
}
args := baseCLI(dsn, "it_mismatch", out)
for i := range args {
if args[i] == "1=1" {
args[i] = "id > 0"
}
}
if _, err := runCLI(t, context.Background(), args...); err == nil {
t.Fatal("expected config mismatch error")
} else if !strings.Contains(err.Error(), "different configuration") {
t.Fatalf("unexpected error: %v", err)
}
args = append(args, "--force-resume")
if _, err := runCLI(t, context.Background(), args...); err != nil {
t.Fatalf("force resume: %v", err)
}
}