Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
919 lines
24 KiB
Go
919 lines
24 KiB
Go
package app
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"database/sql"
|
|
"encoding/csv"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/go-sql-driver/mysql"
|
|
"github.com/klauspost/compress/zstd"
|
|
"github.com/ulikunitz/xz"
|
|
|
|
"archer/internal/cli"
|
|
)
|
|
|
|
func testDSN(t *testing.T) string {
|
|
d := os.Getenv("ARCHER_TEST_DSN")
|
|
if d == "" {
|
|
t.Skip("ARCHER_TEST_DSN is not set; skipping MySQL integration test")
|
|
}
|
|
return d
|
|
}
|
|
|
|
func TestMain(m *testing.M) {
|
|
if dsn := os.Getenv("ARCHER_TEST_DSN"); dsn != "" {
|
|
cfg, err := mysql.ParseDSN(dsn)
|
|
if err == nil && cfg.DBName != "" {
|
|
dbName := cfg.DBName
|
|
cfg.DBName = ""
|
|
db, err := sql.Open("mysql", cfg.FormatDSN())
|
|
if err == nil {
|
|
_, _ = db.Exec("CREATE DATABASE IF NOT EXISTS `" + dbName + "`")
|
|
_, _ = db.Exec("SET GLOBAL local_infile=ON")
|
|
_, _ = db.Exec("SET GLOBAL sql_mode=''")
|
|
db.Close()
|
|
}
|
|
}
|
|
}
|
|
os.Exit(m.Run())
|
|
}
|
|
|
|
func openRoot(t *testing.T, dsn string) *sql.DB {
|
|
t.Helper()
|
|
cfg, err := mysql.ParseDSN(dsn)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
db, err := sql.Open("mysql", cfg.FormatDSN())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.Ping(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return db
|
|
}
|
|
|
|
func execSQL(t *testing.T, db *sql.DB, q string, args ...any) {
|
|
t.Helper()
|
|
if _, err := db.Exec(q, args...); err != nil {
|
|
t.Fatalf("exec %q: %v", q, err)
|
|
}
|
|
}
|
|
|
|
var stdoutMu sync.Mutex
|
|
var stderrMu sync.Mutex
|
|
|
|
func captureStdout(fn func() error) (string, error) {
|
|
stdoutMu.Lock()
|
|
defer stdoutMu.Unlock()
|
|
old := os.Stdout
|
|
r, w, err := os.Pipe()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
os.Stdout = w
|
|
var buf bytes.Buffer
|
|
done := make(chan struct{})
|
|
go func() {
|
|
_, _ = io.Copy(&buf, r)
|
|
close(done)
|
|
}()
|
|
runErr := fn()
|
|
_ = w.Close()
|
|
os.Stdout = old
|
|
<-done
|
|
return buf.String(), runErr
|
|
}
|
|
|
|
func captureStderr(fn func() error) (string, error) {
|
|
stderrMu.Lock()
|
|
defer stderrMu.Unlock()
|
|
old := os.Stderr
|
|
r, w, err := os.Pipe()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
os.Stderr = w
|
|
var buf bytes.Buffer
|
|
done := make(chan struct{})
|
|
go func() {
|
|
_, _ = io.Copy(&buf, r)
|
|
close(done)
|
|
}()
|
|
runErr := fn()
|
|
_ = w.Close()
|
|
os.Stderr = old
|
|
<-done
|
|
return buf.String(), runErr
|
|
}
|
|
|
|
func runCLI(t *testing.T, ctx context.Context, args ...string) (string, error) {
|
|
t.Helper()
|
|
cfg, err := cli.Parse(args, io.Discard)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return captureStdout(func() error { return Run(ctx, cfg) })
|
|
}
|
|
|
|
func runCLIStderr(t *testing.T, ctx context.Context, args ...string) (string, error) {
|
|
t.Helper()
|
|
cfg, err := cli.Parse(args, io.Discard)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return captureStderr(func() error { return Run(ctx, cfg) })
|
|
}
|
|
|
|
func baseCLI(dsn, table, out string, extra ...string) []string {
|
|
args := []string{
|
|
"--source", dsn,
|
|
"--table", table,
|
|
"--where", "1=1",
|
|
"--file", out,
|
|
}
|
|
return append(args, extra...)
|
|
}
|
|
|
|
func listFiles(t *testing.T, dir, pattern string) []string {
|
|
t.Helper()
|
|
matches, err := filepath.Glob(filepath.Join(dir, pattern))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sort.Strings(matches)
|
|
return matches
|
|
}
|
|
|
|
func dumpIDs(t *testing.T, path string) []int {
|
|
t.Helper()
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var ids []int
|
|
for _, line := range strings.Split(strings.TrimRight(string(b), "\n"), "\n") {
|
|
if line == "" {
|
|
continue
|
|
}
|
|
id, err := strconv.Atoi(strings.SplitN(line, "\t", 2)[0])
|
|
if err != nil {
|
|
t.Fatalf("%s: bad line %q", path, line)
|
|
}
|
|
ids = append(ids, id)
|
|
}
|
|
return ids
|
|
}
|
|
|
|
func TestIntegrationCountShards(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openRoot(t, dsn)
|
|
defer db.Close()
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_count")
|
|
execSQL(t, db, "CREATE TABLE it_count (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
|
var b strings.Builder
|
|
b.WriteString("INSERT INTO it_count (id, v) VALUES ")
|
|
for i := 1; i <= 25; i++ {
|
|
if i > 1 {
|
|
b.WriteString(",")
|
|
}
|
|
fmt.Fprintf(&b, "(%d,'v%d')", i, i)
|
|
}
|
|
execSQL(t, db, b.String())
|
|
|
|
out := t.TempDir()
|
|
_, err := runCLI(t, context.Background(), baseCLI(dsn, "it_count", out,
|
|
"--shard-by", "count", "--shard-rows", "10", "--chunk-size", "7", "--format", "dump")...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
files := listFiles(t, out, "*.dump")
|
|
if len(files) != 3 {
|
|
t.Fatalf("got %d shard files: %v", len(files), files)
|
|
}
|
|
wantNames := []string{"000001.1-10.dump", "000002.11-20.dump", "000003.21-25.dump"}
|
|
var got []int
|
|
for i, f := range files {
|
|
if filepath.Base(f) != wantNames[i] {
|
|
t.Fatalf("file %d = %s want %s", i, filepath.Base(f), wantNames[i])
|
|
}
|
|
got = append(got, dumpIDs(t, f)...)
|
|
}
|
|
if len(got) != 25 {
|
|
t.Fatalf("got %d rows", len(got))
|
|
}
|
|
for i, id := range got {
|
|
if id != i+1 {
|
|
t.Fatalf("row %d = %d, shards are not contiguous", i, id)
|
|
}
|
|
}
|
|
|
|
entries := readManifest(t, out)
|
|
if len(entries) != 3 {
|
|
t.Fatalf("manifest entries %d", len(entries))
|
|
}
|
|
for i, e := range entries {
|
|
if e.Seq != int64(i+1) || e.Rows != int64(len(dumpIDs(t, filepath.Join(out, e.Path)))) {
|
|
t.Fatalf("entry %d %+v", i, e)
|
|
}
|
|
if e.Bytes == 0 || e.SHA == "" {
|
|
t.Fatalf("entry %d missing bytes/hash", i)
|
|
}
|
|
}
|
|
if _, err := os.Stat(filepath.Join(out, "state.json")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var complete bool
|
|
st, err := os.ReadFile(filepath.Join(out, "state.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
complete = bytes.Contains(st, []byte(`"complete": true`))
|
|
if !complete {
|
|
t.Fatalf("state not complete: %s", st)
|
|
}
|
|
}
|
|
|
|
type manifestEntry struct {
|
|
Seq int64 `json:"seq"`
|
|
Path string `json:"path"`
|
|
Date string `json:"date"`
|
|
Rows int64 `json:"rows"`
|
|
Bytes int64 `json:"bytes"`
|
|
SHA string `json:"sha256"`
|
|
PKLo string `json:"pk_lo"`
|
|
PKHi string `json:"pk_hi"`
|
|
}
|
|
|
|
func readManifest(t *testing.T, dir string) []manifestEntry {
|
|
t.Helper()
|
|
b, err := os.ReadFile(filepath.Join(dir, "manifest.jsonl"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var out []manifestEntry
|
|
for _, line := range strings.Split(strings.TrimSpace(string(b)), "\n") {
|
|
if line == "" {
|
|
continue
|
|
}
|
|
var e manifestEntry
|
|
if err := json.Unmarshal([]byte(line), &e); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
out = append(out, e)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestIntegrationDatePathsAgree(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openRoot(t, dsn)
|
|
defer db.Close()
|
|
for _, spec := range []struct{ name, index string }{
|
|
{"it_date_a", "KEY da (created_at, id)"},
|
|
{"it_date_b", "KEY db (created_at)"},
|
|
} {
|
|
execSQL(t, db, "DROP TABLE IF EXISTS "+spec.name)
|
|
execSQL(t, db, "CREATE TABLE "+spec.name+` (
|
|
id BIGINT NOT NULL,
|
|
v VARCHAR(32),
|
|
created_at DATETIME NULL,
|
|
PRIMARY KEY (id),
|
|
`+spec.index+`)`)
|
|
execSQL(t, db, `INSERT INTO `+spec.name+` (id, v, created_at) VALUES
|
|
(1,'a','2025-01-01 10:00:00'),
|
|
(2,'b','2025-01-01 12:00:00'),
|
|
(3,'c','2025-01-02 01:00:00'),
|
|
(4,'d',NULL),
|
|
(5,'e','2025-01-01 11:00:00'),
|
|
(6,'f','2025-01-03 00:00:00')`)
|
|
}
|
|
outA := t.TempDir()
|
|
outB := t.TempDir()
|
|
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_a", outA,
|
|
"--shard-by", "date", "--shard-column", "created_at", "--format", "dump", "--chunk-size", "2")...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_b", outB,
|
|
"--shard-by", "date", "--shard-column", "created_at", "--format", "dump", "--chunk-size", "2")...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a := readTree(t, outA)
|
|
b := readTree(t, outB)
|
|
if len(a) != len(b) {
|
|
t.Fatalf("file counts differ: %d vs %d (%v / %v)", len(a), len(b), a, b)
|
|
}
|
|
for k, va := range a {
|
|
if b[k] != va {
|
|
t.Fatalf("%s:\nA=%q\nB=%q", k, va, b[k])
|
|
}
|
|
}
|
|
if _, ok := a["date=_null/000001.4-4.dump"]; !ok {
|
|
t.Fatalf("NULL row not routed to _null: %v", a)
|
|
}
|
|
if len(a) != 4 {
|
|
t.Fatalf("expected 4 shards (null + 3 days), got %v", a)
|
|
}
|
|
}
|
|
|
|
func readTree(t *testing.T, dir string) map[string]string {
|
|
t.Helper()
|
|
out := map[string]string{}
|
|
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
|
|
if err != nil || d.IsDir() {
|
|
return err
|
|
}
|
|
if !strings.HasSuffix(d.Name(), ".dump") {
|
|
return nil
|
|
}
|
|
rel, _ := filepath.Rel(dir, path)
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
out[filepath.ToSlash(rel)] = string(b)
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestIntegrationZeroDateAndNull(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openRoot(t, dsn)
|
|
defer db.Close()
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_zero")
|
|
execSQL(t, db, `CREATE TABLE it_zero (
|
|
id BIGINT NOT NULL,
|
|
d DATE NULL,
|
|
PRIMARY KEY (id),
|
|
KEY dz (d, id))`)
|
|
execSQL(t, db, `INSERT INTO it_zero (id, d) VALUES
|
|
(1, NULL), (2, '0000-00-00'), (3, '2024-01-01'), (4, '2024-01-01')`)
|
|
|
|
out := t.TempDir()
|
|
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_zero", out,
|
|
"--shard-by", "date", "--shard-column", "d")...); err == nil {
|
|
t.Fatal("expected zero-date error")
|
|
} else if !strings.Contains(err.Error(), "zero date") {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
|
|
out = t.TempDir()
|
|
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_zero", out,
|
|
"--shard-by", "date", "--shard-column", "d", "--zero-date", "bucket")...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tree := readTree(t, out)
|
|
for _, want := range []string{
|
|
"date=_null/000001.1-1.dump",
|
|
"date=0000-00-00/000001.2-2.dump",
|
|
"date=2024-01-01/000001.3-4.dump",
|
|
} {
|
|
if _, ok := tree[want]; !ok {
|
|
t.Fatalf("missing %s in %v", want, tree)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestIntegrationEpochUnits(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openRoot(t, dsn)
|
|
defer db.Close()
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_epoch")
|
|
execSQL(t, db, `CREATE TABLE it_epoch (
|
|
id BIGINT NOT NULL,
|
|
ts BIGINT NOT NULL,
|
|
PRIMARY KEY (id),
|
|
KEY e (ts, id))`)
|
|
execSQL(t, db, `INSERT INTO it_epoch (id, ts) VALUES
|
|
(1, 1717200000), (2, 1717200001)`)
|
|
|
|
for _, tc := range []struct {
|
|
unit string
|
|
vals string
|
|
}{
|
|
{"epoch-seconds", "(3, 1717200000), (4, 1717200001)"},
|
|
{"epoch-milliseconds", "(3, 1717200000000), (4, 1717200001000)"},
|
|
{"epoch-microseconds", "(3, 1717200000000000), (4, 1717200001000000)"},
|
|
} {
|
|
table := "it_epoch_" + strings.Split(tc.unit, "-")[1]
|
|
execSQL(t, db, "DROP TABLE IF EXISTS "+table)
|
|
execSQL(t, db, `CREATE TABLE `+table+` (
|
|
id BIGINT NOT NULL, ts BIGINT NOT NULL,
|
|
PRIMARY KEY (id), KEY e (ts, id))`)
|
|
execSQL(t, db, "INSERT INTO "+table+" (id, ts) VALUES "+tc.vals)
|
|
out := t.TempDir()
|
|
if _, err := runCLI(t, context.Background(), baseCLI(dsn, table, out,
|
|
"--shard-by", "date", "--shard-column", "ts", "--shard-column-type", tc.unit,
|
|
"--shard-time-zone", "UTC")...); err != nil {
|
|
t.Fatalf("%s: %v", tc.unit, err)
|
|
}
|
|
tree := readTree(t, out)
|
|
if len(tree) != 1 {
|
|
t.Fatalf("%s: expected one date bucket, got %v", tc.unit, tree)
|
|
}
|
|
for k := range tree {
|
|
if !strings.Contains(k, "date=2024-06-01") {
|
|
t.Fatalf("%s: wrong bucket %s", tc.unit, k)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestIntegrationNoIndexFailFast(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openRoot(t, dsn)
|
|
defer db.Close()
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_noindex")
|
|
execSQL(t, db, `CREATE TABLE it_noindex (
|
|
id BIGINT NOT NULL, d DATETIME NULL, PRIMARY KEY (id))`)
|
|
execSQL(t, db, `INSERT INTO it_noindex (id, d) VALUES (1,'2024-01-01 00:00:00')`)
|
|
|
|
out := t.TempDir()
|
|
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_noindex", out,
|
|
"--shard-by", "date", "--shard-column", "d")...); err == nil {
|
|
t.Fatal("expected no-index error")
|
|
} else if !strings.Contains(err.Error(), "no usable index") {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
out = t.TempDir()
|
|
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_noindex", out,
|
|
"--shard-by", "date", "--shard-column", "d", "--allow-full-scan")...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(out, "date=2024-01-01")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestIntegrationDryRunCreatesNothing(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openRoot(t, dsn)
|
|
defer db.Close()
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_dry")
|
|
execSQL(t, db, "CREATE TABLE it_dry (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
|
execSQL(t, db, "INSERT INTO it_dry (id, v) VALUES (1,'a')")
|
|
|
|
out := filepath.Join(t.TempDir(), "never-created")
|
|
stdout, err := runCLI(t, context.Background(), baseCLI(dsn, "it_dry", out, "--dry-run")...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, statErr := os.Stat(out); !os.IsNotExist(statErr) {
|
|
t.Fatal("dry-run created the output directory")
|
|
}
|
|
if !strings.Contains(stdout, "== first SELECT ==") || !strings.Contains(stdout, "config hash") {
|
|
t.Fatalf("dry-run output incomplete:\n%s", stdout)
|
|
}
|
|
}
|
|
|
|
func TestIntegrationResumeAfterInterrupt(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openRoot(t, dsn)
|
|
defer db.Close()
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_resume")
|
|
execSQL(t, db, "CREATE TABLE it_resume (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
|
var b strings.Builder
|
|
b.WriteString("INSERT INTO it_resume (id, v) VALUES ")
|
|
const total = 20000
|
|
for i := 1; i <= total; i++ {
|
|
if i > 1 {
|
|
b.WriteString(",")
|
|
}
|
|
fmt.Fprintf(&b, "(%d,'v')", i)
|
|
}
|
|
execSQL(t, db, b.String())
|
|
|
|
out := t.TempDir()
|
|
args := baseCLI(dsn, "it_resume", out, "--shard-by", "count", "--shard-rows", "137",
|
|
"--chunk-size", "50", "--progress", "100")
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
stderrMu.Lock()
|
|
old := os.Stderr
|
|
r, w, _ := os.Pipe()
|
|
os.Stderr = w
|
|
cfg, err := cli.Parse(args, io.Discard)
|
|
if err != nil {
|
|
os.Stderr = old
|
|
stderrMu.Unlock()
|
|
t.Fatal(err)
|
|
}
|
|
firstErr := make(chan error, 1)
|
|
go func() { firstErr <- Run(ctx, cfg) }()
|
|
go func() {
|
|
buf := make([]byte, 4096)
|
|
for {
|
|
n, err := r.Read(buf)
|
|
if n > 0 && strings.Contains(string(buf[:n]), "progress:") {
|
|
cancel()
|
|
}
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
err = <-firstErr
|
|
_ = w.Close()
|
|
os.Stderr = old
|
|
stderrMu.Unlock()
|
|
if err == nil {
|
|
t.Fatal("first run should have been interrupted")
|
|
}
|
|
|
|
if _, err := runCLI(t, context.Background(), args...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
seen := map[int]bool{}
|
|
for _, f := range listFiles(t, out, "*.dump") {
|
|
for _, id := range dumpIDs(t, f) {
|
|
seen[id] = true
|
|
}
|
|
}
|
|
for i := 1; i <= total; i++ {
|
|
if !seen[i] {
|
|
t.Fatalf("row %d missing after resume", i)
|
|
}
|
|
}
|
|
st, err := os.ReadFile(filepath.Join(out, "state.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Contains(st, []byte(`"complete": true`)) {
|
|
t.Fatalf("state not complete: %s", st)
|
|
}
|
|
var saved struct {
|
|
RowsDone int64 `json:"rows_done"`
|
|
}
|
|
if err := json.Unmarshal(st, &saved); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if saved.RowsDone != total {
|
|
t.Fatalf("rows_done = %d after resume, want cumulative %d", saved.RowsDone, total)
|
|
}
|
|
if _, err := filepath.Glob(filepath.Join(out, "*.partial")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestIntegrationProgressModes(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openRoot(t, dsn)
|
|
defer db.Close()
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_progress")
|
|
execSQL(t, db, "CREATE TABLE it_progress (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
|
execSQL(t, db, "INSERT INTO it_progress (id, v) VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e')")
|
|
|
|
for _, extra := range [][]string{
|
|
nil,
|
|
{"--progress", "off"},
|
|
{"--progress", "0"},
|
|
} {
|
|
out := t.TempDir()
|
|
args := baseCLI(dsn, "it_progress", out, extra...)
|
|
errOut, err := runCLIStderr(t, context.Background(), args...)
|
|
if err != nil {
|
|
t.Fatalf("%v: %v", extra, err)
|
|
}
|
|
if strings.Contains(errOut, "progress:") || strings.Contains(errOut, "done rows=") {
|
|
t.Fatalf("%v: unexpected progress output %q", extra, errOut)
|
|
}
|
|
}
|
|
|
|
out := t.TempDir()
|
|
errOut, err := runCLIStderr(t, context.Background(), baseCLI(dsn, "it_progress", out, "--progress", "on")...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(errOut, "done rows=5") {
|
|
t.Fatalf("forced progress missing done summary: %q", errOut)
|
|
}
|
|
if strings.Contains(errOut, "\r") {
|
|
t.Fatalf("redirected progress must not use carriage returns: %q", errOut)
|
|
}
|
|
|
|
out = t.TempDir()
|
|
errOut, err = runCLIStderr(t, context.Background(), baseCLI(dsn, "it_progress", out, "--progress", "1")...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(errOut, "progress: rows=5") {
|
|
t.Fatalf("row-threshold progress line missing: %q", errOut)
|
|
}
|
|
}
|
|
|
|
func TestIntegrationEstimateMetrics(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openRoot(t, dsn)
|
|
defer db.Close()
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_estimate")
|
|
execSQL(t, db, `CREATE TABLE it_estimate (
|
|
id BIGINT NOT NULL,
|
|
v VARCHAR(32),
|
|
created_at DATETIME NULL,
|
|
PRIMARY KEY (id),
|
|
KEY de (created_at, id))`)
|
|
execSQL(t, db, `INSERT INTO it_estimate (id, v, created_at) VALUES
|
|
(1,'a','2025-01-01 01:00:00'),
|
|
(2,'b','2025-01-01 02:00:00'),
|
|
(3,'c','2025-01-02 01:00:00'),
|
|
(4,'d','2025-01-03 01:00:00'),
|
|
(5,'e','2025-01-03 02:00:00')`)
|
|
|
|
out := t.TempDir()
|
|
errOut, err := runCLIStderr(t, context.Background(), baseCLI(dsn, "it_estimate", out, "--progress", "1")...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(errOut, "estimate=key") {
|
|
t.Fatalf("primary key estimate missing: %q", errOut)
|
|
}
|
|
|
|
out = t.TempDir()
|
|
errOut, err = runCLIStderr(t, context.Background(),
|
|
baseCLI(dsn, "it_estimate", out, "--progress", "1", "--shard-by", "date", "--shard-column", "created_at")...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(errOut, "estimate=day") {
|
|
t.Fatalf("day estimate missing: %q", errOut)
|
|
}
|
|
}
|
|
|
|
func TestIntegrationImportDump(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openRoot(t, dsn)
|
|
defer db.Close()
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_import_src")
|
|
execSQL(t, db, "CREATE TABLE it_import_src (id BIGINT NOT NULL, v VARCHAR(64), PRIMARY KEY (id))")
|
|
execSQL(t, db, `INSERT INTO it_import_src (id, v) VALUES
|
|
(1,'plain'), (2,'tab\there'), (3,'line\nbreak'), (4,'back\\slash')`)
|
|
|
|
for _, comp := range []string{"none", "gzip", "zstd", "xz"} {
|
|
out := t.TempDir()
|
|
args := baseCLI(dsn, "it_import_src", out, "--format", "dump", "--compress", comp)
|
|
if _, err := runCLI(t, context.Background(), args...); err != nil {
|
|
t.Fatalf("%s: %v", comp, err)
|
|
}
|
|
files := listFiles(t, out, "*.dump*")
|
|
if len(files) != 1 {
|
|
t.Fatalf("%s: files %v", comp, files)
|
|
}
|
|
loadPath := files[0]
|
|
if comp != "none" {
|
|
loadPath = filepath.Join(t.TempDir(), "decompressed.dump")
|
|
if err := decompressFile(files[0], loadPath, comp); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_import_dst")
|
|
execSQL(t, db, "CREATE TABLE it_import_dst (id BIGINT NOT NULL, v VARCHAR(64), PRIMARY KEY (id))")
|
|
loadLocal(t, dsn, loadPath, "it_import_dst")
|
|
var n int
|
|
if err := db.QueryRow("SELECT COUNT(*) FROM it_import_dst").Scan(&n); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 4 {
|
|
t.Fatalf("%s: loaded %d rows", comp, n)
|
|
}
|
|
var v string
|
|
if err := db.QueryRow("SELECT v FROM it_import_dst WHERE id=3").Scan(&v); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if v != "line\nbreak" {
|
|
t.Fatalf("%s: round trip mismatch %q", comp, v)
|
|
}
|
|
}
|
|
}
|
|
|
|
func decompressFile(src, dst, comp string) error {
|
|
in, err := os.Open(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer in.Close()
|
|
var r io.Reader
|
|
switch comp {
|
|
case "gzip":
|
|
gr, err := gzip.NewReader(in)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer gr.Close()
|
|
r = gr
|
|
case "zstd":
|
|
zr, err := zstd.NewReader(in)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer zr.Close()
|
|
r = zr
|
|
case "xz":
|
|
xr, err := xz.NewReader(in)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
r = xr
|
|
default:
|
|
r = in
|
|
}
|
|
out, err := os.Create(dst)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer out.Close()
|
|
_, err = io.Copy(out, r)
|
|
return err
|
|
}
|
|
|
|
func loadLocal(t *testing.T, dsn, path, table string) {
|
|
t.Helper()
|
|
cfg, err := mysql.ParseDSN(dsn)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cfg.AllowAllFiles = true
|
|
db, err := sql.Open("mysql", cfg.FormatDSN())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
p := filepath.ToSlash(path)
|
|
p = strings.ReplaceAll(p, `'`, `''`)
|
|
q := fmt.Sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE %s", p, table)
|
|
if _, err := db.Exec(q); err != nil {
|
|
t.Fatalf("load data: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestIntegrationCSVAndJSONL(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openRoot(t, dsn)
|
|
defer db.Close()
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_fmt")
|
|
execSQL(t, db, `CREATE TABLE it_fmt (
|
|
id BIGINT NOT NULL, name VARCHAR(64), price DECIMAL(10,2), created DATETIME,
|
|
data BLOB, PRIMARY KEY (id))`)
|
|
execSQL(t, db, `INSERT INTO it_fmt (id, name, price, created, data) VALUES
|
|
(1,'a,\"b\"',12.30,'2024-01-02 03:04:05',X'6869'), (2,'',NULL,NULL,NULL)`)
|
|
|
|
// CSV
|
|
out := t.TempDir()
|
|
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_fmt", out, "--format", "csv")...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
files := listFiles(t, out, "*.csv")
|
|
if len(files) != 1 {
|
|
t.Fatalf("csv files %v", files)
|
|
}
|
|
f, err := os.Open(files[0])
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
recs, err := csv.NewReader(f).ReadAll()
|
|
f.Close()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(recs) != 3 {
|
|
t.Fatalf("csv records %d", len(recs))
|
|
}
|
|
if recs[0][0] != "id" || recs[0][1] != "name" {
|
|
t.Fatalf("header %v", recs[0])
|
|
}
|
|
if recs[1][1] != `a,"b"` || recs[1][2] != "12.30" {
|
|
t.Fatalf("row %v", recs[1])
|
|
}
|
|
if recs[2][2] != `\N` {
|
|
t.Fatalf("NULL decimal %q", recs[2][2])
|
|
}
|
|
|
|
// JSONL
|
|
out = t.TempDir()
|
|
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_fmt", out, "--format", "jsonl")...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
files = listFiles(t, out, "*.jsonl")
|
|
var shardFiles []string
|
|
for _, f := range files {
|
|
if filepath.Base(f) != "manifest.jsonl" {
|
|
shardFiles = append(shardFiles, f)
|
|
}
|
|
}
|
|
files = shardFiles
|
|
if len(files) != 1 {
|
|
t.Fatalf("jsonl files %v", files)
|
|
}
|
|
b, err := os.ReadFile(files[0])
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
|
|
if len(lines) != 2 {
|
|
t.Fatalf("jsonl lines %d", len(lines))
|
|
}
|
|
var obj map[string]any
|
|
if err := json.Unmarshal([]byte(lines[0]), &obj); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if obj["price"] != "12.30" {
|
|
t.Fatalf("price %v", obj["price"])
|
|
}
|
|
if obj["created"] != "2024-01-02T03:04:05Z" {
|
|
t.Fatalf("created %v", obj["created"])
|
|
}
|
|
if obj["data"] != "aGk=" {
|
|
t.Fatalf("data %v", obj["data"])
|
|
}
|
|
var second map[string]any
|
|
if err := json.Unmarshal([]byte(lines[1]), &second); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if second["price"] != nil || second["created"] != nil {
|
|
t.Fatalf("null mapping %v", second)
|
|
}
|
|
}
|
|
|
|
func TestIntegrationPurgeAndDest(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openRoot(t, dsn)
|
|
defer db.Close()
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_purge_src")
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_purge_dst")
|
|
execSQL(t, db, "CREATE TABLE it_purge_src (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
|
execSQL(t, db, "CREATE TABLE it_purge_dst (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
|
execSQL(t, db, "INSERT INTO it_purge_src (id, v) VALUES (1,'a'),(2,'b'),(3,'c')")
|
|
|
|
out := t.TempDir()
|
|
args := []string{
|
|
"--source", dsn, "--table", "it_purge_src", "--where", "1=1",
|
|
"--file", out, "--dest", dsn, "--dest-table", "it_purge_dst",
|
|
"--purge", "--chunk-size", "2",
|
|
}
|
|
if _, err := runCLI(t, context.Background(), args...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var srcCount, dstCount int
|
|
db.QueryRow("SELECT COUNT(*) FROM it_purge_src").Scan(&srcCount)
|
|
db.QueryRow("SELECT COUNT(*) FROM it_purge_dst").Scan(&dstCount)
|
|
if srcCount != 0 || dstCount != 3 {
|
|
t.Fatalf("src=%d dst=%d", srcCount, dstCount)
|
|
}
|
|
if len(listFiles(t, out, "*.dump")) != 1 {
|
|
t.Fatal("expected one shard")
|
|
}
|
|
}
|
|
|
|
func TestIntegrationCheckpointConfigMismatch(t *testing.T) {
|
|
dsn := testDSN(t)
|
|
db := openRoot(t, dsn)
|
|
defer db.Close()
|
|
execSQL(t, db, "DROP TABLE IF EXISTS it_mismatch")
|
|
execSQL(t, db, "CREATE TABLE it_mismatch (id BIGINT NOT NULL, v VARCHAR(32), PRIMARY KEY (id))")
|
|
execSQL(t, db, "INSERT INTO it_mismatch (id, v) VALUES (1,'a'),(2,'b')")
|
|
out := t.TempDir()
|
|
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_mismatch", out)...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Remove the complete flag so resume logic runs.
|
|
stPath := filepath.Join(out, "state.json")
|
|
b, _ := os.ReadFile(stPath)
|
|
b = bytes.Replace(b, []byte(`"complete": true`), []byte(`"complete": false`), 1)
|
|
if err := os.WriteFile(stPath, b, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
args := baseCLI(dsn, "it_mismatch", out)
|
|
for i := range args {
|
|
if args[i] == "1=1" {
|
|
args[i] = "id > 0"
|
|
}
|
|
}
|
|
if _, err := runCLI(t, context.Background(), args...); err == nil {
|
|
t.Fatal("expected config mismatch error")
|
|
} else if !strings.Contains(err.Error(), "different configuration") {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
args = append(args, "--force-resume")
|
|
if _, err := runCLI(t, context.Background(), args...); err != nil {
|
|
t.Fatalf("force resume: %v", err)
|
|
}
|
|
}
|