Add archer import and allow date-mode bulk delete

archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-08 21:14:23 +08:00
1 parent bc4a6b504f
commit 366d61ca25
49 files changed
+5099 -342

No files matched your search

+214
View File
@@ -0,0 +1,214 @@
// Package importer replays a dump archive into an existing MySQL table.
//
// The archive's manifest.jsonl is the only authority: files on disk that are
// not registered are never imported, and unparseable manifest lines abort the
// run before a database connection is opened. Each shard is a complete,
// independently loadable file, so shards are the unit of selection, atomicity
// and failure reporting.
package importer
import (
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"archer/internal/archererr"
"archer/internal/checkpoint"
"archer/internal/shard"
)
// Archive is a loaded, validated-by-reading view of an export directory.
type Archive struct {
Root string
Entries []shard.Entry
State *checkpoint.State
Complete bool
SourceTable string
ArchivedColumns []string
Format string
Compress string
Compressor *shard.Compressor
}
// LoadArchive reads manifest.jsonl strictly and state.json when present. It
// never writes to the archive directory.
func LoadArchive(root string) (*Archive, error) {
st, err := os.Stat(root)
if err != nil {
return nil, archererr.Newf(archererr.KindConfig, "--file %s: %v", root, err)
}
if !st.IsDir() {
return nil, archererr.Newf(archererr.KindConfig, "--file %s is not a directory", root)
}
entries, err := shard.LoadManifest(root)
if err != nil {
return nil, archererr.Wrap(archererr.KindValidation, err)
}
state, err := checkpoint.Load(root)
if err != nil {
return nil, archererr.Wrap(archererr.KindValidation, err)
}
a := &Archive{Root: root, Entries: entries, State: state}
if state != nil {
a.Complete = state.Complete
a.SourceTable = strings.TrimSpace(state.Config["table"])
if cols := strings.TrimSpace(state.Config["columns"]); cols != "" {
for _, part := range strings.Split(cols, ",") {
if part = strings.TrimSpace(part); part != "" {
a.ArchivedColumns = append(a.ArchivedColumns, part)
}
}
}
}
return a, nil
}
// CheckComplete enforces the completion gate. A manifest only proves which
// shards are sealed, not that the export covered the whole table, so an
// incomplete archive is refused unless the operator opts in explicitly.
func (a *Archive) CheckComplete(allowPartial bool) error {
if allowPartial {
return nil
}
if a.State == nil {
return archererr.Newf(archererr.KindValidation,
"archive %s has no state.json; cannot confirm the export finished (pass --allow-partial to import its sealed shards)", a.Root)
}
if !a.State.Complete {
return archererr.Newf(archererr.KindValidation,
"archive %s is incomplete; it may contain only part of the table (pass --allow-partial to import its sealed shards)", a.Root)
}
return nil
}
// CheckMappingChoice rejects an explicit mapping when the archive already
// recorded its own column list: two sources of truth would silently disagree.
func (a *Archive) CheckMappingChoice(columns []string, assumeAll bool) error {
if len(a.ArchivedColumns) == 0 {
return nil
}
if len(columns) > 0 {
return archererr.Newf(archererr.KindConfig,
"archive records its own column list (%s); --columns cannot be combined with it", strings.Join(a.ArchivedColumns, ", "))
}
if assumeAll {
return archererr.Newf(archererr.KindConfig,
"archive records its own column list (%s); --assume-all-columns cannot be combined with it", strings.Join(a.ArchivedColumns, ", "))
}
return nil
}
var shardFileRE = regexp.MustCompile(`^\d+\..*\.(dump|csv|jsonl)(\.gz|\.zst|\.xz)?$`)
// UnregisteredFiles counts shard-looking files that the manifest does not
// list. They are ignored by design; the count is surfaced in --statistics so
// they are not mistaken for imported data.
func UnregisteredFiles(root string, entries []shard.Entry) int {
registered := make(map[string]bool, len(entries))
for _, e := range entries {
registered[filepath.ToSlash(e.Path)] = true
}
n := 0
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil || d.IsDir() {
return nil
}
name := d.Name()
if name == "manifest.jsonl" || name == "state.json" || strings.HasSuffix(name, ".partial") ||
strings.HasSuffix(name, ".partial.meta") || !shardFileRE.MatchString(name) {
return nil
}
rel, err := filepath.Rel(root, path)
if err != nil {
return nil
}
if !registered[filepath.ToSlash(rel)] {
n++
}
return nil
})
return n
}
// ShardFormat derives the output format and compression from a manifest path
// such as 000003.100-200.dump.gz.
func ShardFormat(p string) (format, compress string, err error) {
name := filepath.Base(p)
switch {
case strings.HasSuffix(name, ".gz"):
compress, name = "gzip", strings.TrimSuffix(name, ".gz")
case strings.HasSuffix(name, ".zst"):
compress, name = "zstd", strings.TrimSuffix(name, ".zst")
case strings.HasSuffix(name, ".xz"):
compress, name = "xz", strings.TrimSuffix(name, ".xz")
default:
compress = "none"
}
ext := strings.ToLower(filepath.Ext(name))
if ext == "" {
return "", "", fmt.Errorf("shard %s has no format extension", p)
}
format = strings.TrimPrefix(ext, ".")
switch format {
case "dump", "csv", "jsonl":
return format, compress, nil
default:
return "", "", fmt.Errorf("shard %s has unknown format %q", p, format)
}
}
// ResolveFormat checks that every selected shard agrees on format and
// compression, and that the format is dump (the only one v1 can replay).
func ResolveFormat(entries []shard.Entry) (format, compress string, comp *shard.Compressor, err error) {
for i, e := range entries {
f, c, ferr := ShardFormat(e.Path)
if ferr != nil {
return "", "", nil, archererr.Wrap(archererr.KindValidation, ferr)
}
if i == 0 {
format, compress = f, c
continue
}
if f != format {
return "", "", nil, archererr.Newf(archererr.KindValidation,
"selected shards mix formats: %s is %s but %s is %s", entries[0].Path, format, e.Path, f)
}
if c != compress {
return "", "", nil, archererr.Newf(archererr.KindValidation,
"selected shards mix compression: %s is %s but %s is %s", entries[0].Path, compress, e.Path, c)
}
}
if format != "dump" {
return "", "", nil, archererr.Newf(archererr.KindValidation,
"import supports only dump archives; the selected shards are %s", format)
}
comp, err = shard.NewCompressor(compress, shard.DefaultLevel)
if err != nil {
return "", "", nil, archererr.Wrap(archererr.KindValidation, err)
}
return format, compress, comp, nil
}
// CheckShardFiles verifies every selected manifest entry has a readable file
// before any connection is opened.
func CheckShardFiles(root string, entries []shard.Entry) error {
for _, e := range entries {
path := filepath.Join(root, filepath.FromSlash(e.Path))
st, err := os.Stat(path)
if err != nil {
return archererr.Newf(archererr.KindValidation,
"manifest entry seq %d points to missing shard %s: %v", e.Seq, e.Path, err)
}
if st.IsDir() {
return archererr.Newf(archererr.KindValidation,
"manifest entry seq %d points to a directory, not a shard: %s", e.Seq, e.Path)
}
}
return nil
}
func shardPath(root string, e shard.Entry) string {
return filepath.Join(root, filepath.FromSlash(e.Path))
}
+211
View File
@@ -0,0 +1,211 @@
package importer
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"archer/internal/checkpoint"
"archer/internal/shard"
)
func writeManifest(t *testing.T, root string, entries []shard.Entry) {
t.Helper()
var b strings.Builder
for _, e := range entries {
j, err := json.Marshal(e)
if err != nil {
t.Fatal(err)
}
b.Write(j)
b.WriteByte('\n')
}
if err := os.WriteFile(filepath.Join(root, "manifest.jsonl"), []byte(b.String()), 0o644); err != nil {
t.Fatal(err)
}
}
func writeShard(t *testing.T, root, rel, body string) {
t.Helper()
path := filepath.Join(root, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
func saveState(t *testing.T, root string, complete bool, config map[string]string) {
t.Helper()
if err := checkpoint.Save(root, &checkpoint.State{Complete: complete, Config: config}); err != nil {
t.Fatal(err)
}
}
func TestLoadArchiveReadsStateAndRecords(t *testing.T) {
root := t.TempDir()
entries := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump", PKLo: "1", PKHi: "1", Rows: 1, Bytes: 3, SHA256: "aa"}}
writeManifest(t, root, entries)
writeShard(t, root, entries[0].Path, "1\tx\n")
saveState(t, root, true, map[string]string{"table": "orders", "columns": "id,title"})
a, err := LoadArchive(root)
if err != nil {
t.Fatal(err)
}
if !a.Complete || a.SourceTable != "orders" {
t.Fatalf("archive = %+v", a)
}
if strings.Join(a.ArchivedColumns, ",") != "id,title" {
t.Fatalf("columns = %v", a.ArchivedColumns)
}
}
func TestLoadArchiveRejectsCorruptManifest(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "manifest.jsonl"), []byte("{not json\n"), 0o644); err != nil {
t.Fatal(err)
}
_, err := LoadArchive(root)
if err == nil || !strings.Contains(err.Error(), "line 1") {
t.Fatalf("got %v", err)
}
}
func TestLoadArchiveMissingManifest(t *testing.T) {
_, err := LoadArchive(t.TempDir())
if err == nil || !strings.Contains(err.Error(), "manifest") {
t.Fatalf("got %v", err)
}
}
func TestLoadArchiveNotADirectory(t *testing.T) {
path := filepath.Join(t.TempDir(), "file")
if err := os.WriteFile(path, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := LoadArchive(path); err == nil {
t.Fatal("expected an error")
}
}
func TestCompletionGate(t *testing.T) {
root := t.TempDir()
writeManifest(t, root, []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}})
a, err := LoadArchive(root)
if err != nil {
t.Fatal(err)
}
if err := a.CheckComplete(false); err == nil || !strings.Contains(err.Error(), "state.json") {
t.Fatalf("missing state: %v", err)
}
if err := a.CheckComplete(true); err != nil {
t.Fatalf("--allow-partial: %v", err)
}
saveState(t, root, false, nil)
a, _ = LoadArchive(root)
if err := a.CheckComplete(false); err == nil || !strings.Contains(err.Error(), "incomplete") {
t.Fatalf("incomplete: %v", err)
}
if err := a.CheckComplete(true); err != nil {
t.Fatalf("--allow-partial: %v", err)
}
saveState(t, root, true, nil)
a, _ = LoadArchive(root)
if err := a.CheckComplete(false); err != nil {
t.Fatal(err)
}
}
func TestMappingChoiceConflict(t *testing.T) {
root := t.TempDir()
writeManifest(t, root, []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}})
saveState(t, root, true, map[string]string{"columns": "id,title"})
a, _ := LoadArchive(root)
if err := a.CheckMappingChoice([]string{"id"}, false); err == nil || !strings.Contains(err.Error(), "--columns") {
t.Fatalf("got %v", err)
}
if err := a.CheckMappingChoice(nil, true); err == nil || !strings.Contains(err.Error(), "--assume-all-columns") {
t.Fatalf("got %v", err)
}
if err := a.CheckMappingChoice(nil, false); err != nil {
t.Fatal(err)
}
}
func TestCheckShardFiles(t *testing.T) {
root := t.TempDir()
writeShard(t, root, "a/000001.1-1.dump", "1\n")
ok := []shard.Entry{{Seq: 1, Path: "a/000001.1-1.dump"}}
if err := CheckShardFiles(root, ok); err != nil {
t.Fatal(err)
}
missing := []shard.Entry{{Seq: 7, Path: "a/000007.1-1.dump"}}
if err := CheckShardFiles(root, missing); err == nil || !strings.Contains(err.Error(), "seq 7") {
t.Fatalf("got %v", err)
}
}
func TestResolveFormat(t *testing.T) {
dump := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}, {Seq: 2, Path: "000002.2-2.dump.gz"}}
if _, _, _, err := ResolveFormat(dump); err == nil || !strings.Contains(err.Error(), "mix compression") {
t.Fatalf("got %v", err)
}
gz := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump.gz"}, {Seq: 2, Path: "000002.2-2.dump.gz"}}
format, compress, comp, err := ResolveFormat(gz)
if err != nil {
t.Fatal(err)
}
if format != "dump" || compress != "gzip" || comp.Name != "gzip" {
t.Fatalf("format=%s compress=%s comp=%+v", format, compress, comp)
}
for _, bad := range []string{"000001.1-1.csv", "000001.1-1.jsonl", "000001.1-1.csv.gz"} {
_, _, _, err := ResolveFormat([]shard.Entry{{Seq: 1, Path: bad}})
if err == nil || !strings.Contains(err.Error(), "only dump") {
t.Fatalf("%s: got %v", bad, err)
}
}
mixed := []shard.Entry{{Seq: 1, Path: "a/000001.1-1.dump"}, {Seq: 2, Path: "b/000002.2-2.csv"}}
if _, _, _, err := ResolveFormat(mixed); err == nil || !strings.Contains(err.Error(), "mix formats") {
t.Fatalf("got %v", err)
}
}
func TestShardFormat(t *testing.T) {
for _, tc := range []struct{ path, format, compress string }{
{"000001.1-2.dump", "dump", "none"},
{"000001.1-2.dump.gz", "dump", "gzip"},
{"000001.1-2.dump.zst", "dump", "zstd"},
{"000001.1-2.dump.xz", "dump", "xz"},
{"date=2025-01-01/000001.1-2.jsonl", "jsonl", "none"},
} {
f, c, err := ShardFormat(tc.path)
if err != nil || f != tc.format || c != tc.compress {
t.Fatalf("%s: f=%s c=%s err=%v", tc.path, f, c, err)
}
}
if _, _, err := ShardFormat("000001.1-2"); err == nil {
t.Fatal("expected an error for a missing extension")
}
}
func TestUnregisteredFiles(t *testing.T) {
root := t.TempDir()
entries := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}}
writeManifest(t, root, entries)
writeShard(t, root, "000001.1-1.dump", "1\n")
writeShard(t, root, "000002.2-2.dump", "2\n")
writeShard(t, root, "notes.txt", "ignored")
if got := UnregisteredFiles(root, entries); got != 1 {
t.Fatalf("got %d", got)
}
}
+163
View File
@@ -0,0 +1,163 @@
package importer
import (
"bufio"
"bytes"
"fmt"
"io"
"os"
"strings"
"archer/internal/archererr"
"archer/internal/schema"
"archer/internal/shard"
"archer/internal/sqlutil"
)
// Mapping translates file fields to target columns. It is the single place
// that decides whether a field is loaded into a named column or discarded into
// a user variable (generated columns under positional mapping).
type Mapping struct {
// FileColumns has one entry per file field: the canonical target column
// name, or "" when the field is intentionally discarded.
FileColumns []string
// InsertColumns and InsertFieldIdx describe the parameterized INSERT used
// by the insert engine. InsertFieldIdx points into FileColumns.
InsertColumns []string
InsertFieldIdx []int
// LoadList is the LOAD DATA column list: quoted identifiers and
// @_archer_skip_N variables.
LoadList []string
// Positional is true for --assume-all-columns; callers must warn.
Positional bool
}
// BuildMapping resolves the three permitted mapping sources. Columns recorded
// in the archive win over an explicit --columns by construction (the conflict
// is rejected earlier); assumeAll is only consulted when neither list exists.
func BuildMapping(table *schema.Table, archived, explicit []string, assumeAll bool) (*Mapping, error) {
return buildMapping(table.Name, table.Columns, archived, explicit, assumeAll)
}
func buildMapping(tableName string, columns []schema.Column, archived, explicit []string, assumeAll bool) (*Mapping, error) {
names := archived
if len(names) == 0 {
names = explicit
}
switch {
case len(names) > 0:
return mappingByName(tableName, columns, names)
case assumeAll:
return mappingPositional(tableName, columns)
default:
return nil, archererr.Newf(archererr.KindConfig,
"the archive did not record a column list; provide --columns in file field order or accept positional mapping with --assume-all-columns")
}
}
func mappingByName(tableName string, columns []schema.Column, names []string) (*Mapping, error) {
byName := make(map[string]schema.Column, len(columns))
for _, c := range columns {
byName[strings.ToLower(c.Name)] = c
}
m := &Mapping{}
seen := map[string]bool{}
for _, requested := range names {
col, ok := byName[strings.ToLower(requested)]
if !ok {
return nil, archererr.Newf(archererr.KindValidation,
"column %q from the file field mapping does not exist in target table %s", requested, tableName)
}
if isGenerated(col) {
return nil, archererr.Newf(archererr.KindValidation,
"column %q is a generated column and cannot be written from an archive", col.Name)
}
if seen[col.Name] {
return nil, archererr.Newf(archererr.KindValidation,
"column %q appears more than once in the file field mapping", col.Name)
}
seen[col.Name] = true
idx := len(m.FileColumns)
m.FileColumns = append(m.FileColumns, col.Name)
m.InsertColumns = append(m.InsertColumns, col.Name)
m.InsertFieldIdx = append(m.InsertFieldIdx, idx)
m.LoadList = append(m.LoadList, sqlutil.Ident(col.Name))
}
return m, nil
}
func mappingPositional(tableName string, columns []schema.Column) (*Mapping, error) {
m := &Mapping{Positional: true}
for i, col := range columns {
if isGenerated(col) {
// The file still contains this field, so keep a placeholder and
// discard the value; the target table recomputes it.
m.FileColumns = append(m.FileColumns, "")
m.LoadList = append(m.LoadList, fmt.Sprintf("@_archer_skip_%d", col.Ordinal))
continue
}
m.FileColumns = append(m.FileColumns, col.Name)
m.InsertColumns = append(m.InsertColumns, col.Name)
m.InsertFieldIdx = append(m.InsertFieldIdx, i)
m.LoadList = append(m.LoadList, sqlutil.Ident(col.Name))
}
if len(m.InsertColumns) == 0 {
return nil, archererr.Newf(archererr.KindValidation,
"target table %s has no writable columns", tableName)
}
return m, nil
}
func isGenerated(c schema.Column) bool {
return strings.Contains(strings.ToLower(c.Extra), "generated")
}
// CountFields counts dump fields in one raw line. Literal tabs are escaped by
// the dump encoder, so every 0x09 byte is a separator. An empty line is a
// single empty, non-NULL field.
func CountFields(line []byte) int {
return strings.Count(string(line), "\t") + 1
}
// FirstLineFieldCount reads the first decompressed line of a shard.
func FirstLineFieldCount(path string, comp *shard.Compressor) (fields int, hasRow bool, err error) {
f, err := os.Open(path)
if err != nil {
return 0, false, err
}
defer f.Close()
r, err := comp.Reader(f)
if err != nil {
return 0, false, err
}
defer r.Close()
br := bufio.NewReaderSize(r, 64*1024)
line, err := br.ReadBytes('\n')
if err == io.EOF {
if len(line) == 0 {
return 0, false, nil
}
} else if err != nil {
return 0, false, err
}
line = bytes.TrimSuffix(line, []byte{'\n'})
return CountFields(line), true, nil
}
// CheckFieldCount compares the first row's field count with the load list
// length. It is a sanity check only: it cannot prove that same-width columns
// are in the right order.
func CheckFieldCount(path string, comp *shard.Compressor, want int) error {
got, hasRow, err := FirstLineFieldCount(path, comp)
if err != nil {
return archererr.Wrap(archererr.KindValidation, fmt.Errorf("read first row of %s: %w", path, err))
}
if !hasRow {
return nil
}
if got != want {
return archererr.Newf(archererr.KindValidation,
"first row of %s has %d fields, but the load column list has %d; refusing to load", path, got, want)
}
return nil
}
+151
View File
@@ -0,0 +1,151 @@
package importer
import (
"os"
"path/filepath"
"strings"
"testing"
"archer/internal/schema"
"archer/internal/shard"
)
func targetColumns() []schema.Column {
return []schema.Column{
{Name: "id", Ordinal: 1},
{Name: "title", Ordinal: 2},
{Name: "full_name", Ordinal: 3, Extra: "STORED GENERATED"},
{Name: "created_at", Ordinal: 4},
}
}
func TestMappingByNamePreservesOrder(t *testing.T) {
m, err := buildMapping("orders", targetColumns(), []string{"title", "ID"}, nil, false)
if err != nil {
t.Fatal(err)
}
if strings.Join(m.FileColumns, ",") != "title,id" {
t.Fatalf("file columns = %v", m.FileColumns)
}
if strings.Join(m.InsertColumns, ",") != "title,id" {
t.Fatalf("insert columns = %v", m.InsertColumns)
}
if strings.Join(m.LoadList, ",") != "`title`,`id`" {
t.Fatalf("load list = %v", m.LoadList)
}
}
func TestMappingByNameMissingColumn(t *testing.T) {
_, err := buildMapping("orders", targetColumns(), []string{"id", "missing"}, nil, false)
if err == nil || !strings.Contains(err.Error(), "missing") {
t.Fatalf("got %v", err)
}
}
func TestMappingByNameRejectsGenerated(t *testing.T) {
_, err := buildMapping("orders", targetColumns(), []string{"id", "full_name"}, nil, false)
if err == nil || !strings.Contains(err.Error(), "generated") {
t.Fatalf("got %v", err)
}
}
func TestMappingByNameRejectsDuplicate(t *testing.T) {
_, err := buildMapping("orders", targetColumns(), []string{"id", "Id"}, nil, false)
if err == nil || !strings.Contains(err.Error(), "more than once") {
t.Fatalf("got %v", err)
}
}
func TestMappingPositionalDropsGenerated(t *testing.T) {
m, err := buildMapping("orders", targetColumns(), nil, nil, true)
if err != nil {
t.Fatal(err)
}
if !m.Positional {
t.Fatal("expected positional mapping")
}
if len(m.FileColumns) != 4 {
t.Fatalf("file fields = %v", m.FileColumns)
}
if m.FileColumns[2] != "" {
t.Fatalf("generated field should be empty, got %q", m.FileColumns[2])
}
if strings.Join(m.InsertColumns, ",") != "id,title,created_at" {
t.Fatalf("insert columns = %v", m.InsertColumns)
}
if len(m.InsertFieldIdx) != 3 || m.InsertFieldIdx[2] != 3 {
t.Fatalf("insert indices = %v", m.InsertFieldIdx)
}
if m.LoadList[2] != "@_archer_skip_3" {
t.Fatalf("skip variable = %q", m.LoadList[2])
}
}
func TestMappingRequiresSource(t *testing.T) {
_, err := buildMapping("orders", targetColumns(), nil, nil, false)
if err == nil || !strings.Contains(err.Error(), "--assume-all-columns") {
t.Fatalf("got %v", err)
}
}
func TestMappingExplicitColumns(t *testing.T) {
m, err := buildMapping("orders", targetColumns(), nil, []string{"id", "title"}, false)
if err != nil {
t.Fatal(err)
}
if len(m.FileColumns) != 2 || m.FileColumns[1] != "title" {
t.Fatalf("file columns = %v", m.FileColumns)
}
}
func TestCountFields(t *testing.T) {
cases := []struct {
line string
want int
}{
{"", 1},
{"a", 1},
{"a\tb", 2},
{"a\tb\tc\t", 4},
{`a\tb`, 1}, // escaped tab is not a separator
}
for _, tc := range cases {
if got := CountFields([]byte(tc.line)); got != tc.want {
t.Fatalf("%q: got %d want %d", tc.line, got, tc.want)
}
}
}
func TestFirstLineFieldCount(t *testing.T) {
root := t.TempDir()
path := filepath.Join(root, "x.dump")
if err := os.WriteFile(path, []byte("a\tb\tc\n1\t2\t3\n"), 0o644); err != nil {
t.Fatal(err)
}
comp, _ := shard.NewCompressor("none", shard.DefaultLevel)
n, ok, err := FirstLineFieldCount(path, comp)
if err != nil || !ok || n != 3 {
t.Fatalf("n=%d ok=%t err=%v", n, ok, err)
}
if err := CheckFieldCount(path, comp, 3); err != nil {
t.Fatal(err)
}
if err := CheckFieldCount(path, comp, 4); err == nil || !strings.Contains(err.Error(), "3 fields") {
t.Fatalf("got %v", err)
}
}
func TestFirstLineFieldCountEmptyFile(t *testing.T) {
path := filepath.Join(t.TempDir(), "empty.dump")
if err := os.WriteFile(path, nil, 0o644); err != nil {
t.Fatal(err)
}
comp, _ := shard.NewCompressor("none", shard.DefaultLevel)
_, ok, err := FirstLineFieldCount(path, comp)
if err != nil || ok {
t.Fatalf("ok=%t err=%v", ok, err)
}
if err := CheckFieldCount(path, comp, 9); err != nil {
t.Fatalf("empty shard must not fail the field check: %v", err)
}
}
+83
View File
@@ -0,0 +1,83 @@
package importer
import (
"bufio"
"bytes"
"io"
"archer/internal/row"
)
// DumpReader decodes the dump format produced by internal/row.dumpEncoder:
// TAB-separated fields, LF-terminated rows, \N for NULL and the MySQL escape
// table. It exists for the insert fallback engine; the load engine hands the
// bytes to the server unchanged.
type DumpReader struct {
br *bufio.Reader
}
func NewDumpReader(r io.Reader) *DumpReader {
return &DumpReader{br: bufio.NewReaderSize(r, 256*1024)}
}
// ReadRow returns the next row. It returns io.EOF exactly when the stream ends.
func (d *DumpReader) ReadRow() (row.Row, error) {
line, err := d.br.ReadBytes('\n')
if len(line) == 0 && err == io.EOF {
return row.Row{}, io.EOF
}
if err != nil && err != io.EOF {
return row.Row{}, err
}
line = bytes.TrimSuffix(line, []byte{'\n'})
fields := bytes.Split(line, []byte{'\t'})
out := row.Row{Values: make([]row.Value, len(fields))}
for i, f := range fields {
out.Values[i] = unescapeField(f)
}
return out, nil
}
// unescapeField mirrors MySQL's LOAD DATA escape handling. A field is NULL only
// when it is exactly \N; unknown escapes yield the escaped character, matching
// the server's default FIELDS ESCAPED BY '\\' behavior.
func unescapeField(f []byte) row.Value {
if len(f) == 2 && f[0] == '\\' && f[1] == 'N' {
return row.Value{}
}
if bytes.IndexByte(f, '\\') < 0 {
return row.Value{Raw: append([]byte{}, f...)}
}
out := make([]byte, 0, len(f))
for i := 0; i < len(f); i++ {
if f[i] != '\\' {
out = append(out, f[i])
continue
}
if i+1 >= len(f) {
out = append(out, '\\')
break
}
i++
switch f[i] {
case '\\':
out = append(out, '\\')
case 't':
out = append(out, '\t')
case 'n':
out = append(out, '\n')
case 'r':
out = append(out, '\r')
case '0':
out = append(out, 0)
case 'b':
out = append(out, '\b')
case 'Z':
out = append(out, 0x1a)
default:
// Includes \N in a mixed field: only a whole-field \N is NULL.
out = append(out, f[i])
}
}
return row.Value{Raw: out}
}
+96
View File
@@ -0,0 +1,96 @@
package importer
import (
"bytes"
"io"
"strings"
"testing"
"archer/internal/row"
)
func TestDumpReaderEmptyFieldAndNull(t *testing.T) {
input := "a\t\\N\t\tx\\Ny\n"
dr := NewDumpReader(strings.NewReader(input))
got, err := dr.ReadRow()
if err != nil {
t.Fatal(err)
}
if len(got.Values) != 4 {
t.Fatalf("got %d fields", len(got.Values))
}
if got.Values[0].IsNull() || got.Values[0].String() != "a" {
t.Fatalf("field 0 = %+v", got.Values[0])
}
if !got.Values[1].IsNull() {
t.Fatalf("field 1 should be NULL")
}
if got.Values[2].IsNull() || len(got.Values[2].Raw) != 0 {
t.Fatalf("field 2 should be an empty string, got %+v", got.Values[2])
}
if got.Values[3].String() != "xNy" {
t.Fatalf("\\N inside a field should yield N, got %q", got.Values[3].String())
}
if _, err := dr.ReadRow(); err != io.EOF {
t.Fatalf("expected EOF, got %v", err)
}
}
func TestDumpReaderEscapeTable(t *testing.T) {
input := "a\t\\N\t\t\\t\\n\\r\\0\\b\\Z\\\\\t中文😀\n"
dr := NewDumpReader(strings.NewReader(input))
got, err := dr.ReadRow()
if err != nil {
t.Fatal(err)
}
if len(got.Values) != 5 {
t.Fatalf("got %d fields: %+v", len(got.Values), got.Values)
}
want := []byte{'\t', '\n', '\r', 0, '\b', 0x1a, '\\'}
if !bytes.Equal(got.Values[3].Raw, want) {
t.Fatalf("escapes = %q, want %q", got.Values[3].Raw, want)
}
if got.Values[4].String() != "中文😀" {
t.Fatalf("unicode = %q", got.Values[4].String())
}
}
func TestDumpReaderRoundTripWithEncoder(t *testing.T) {
cols := []row.Column{{Name: "a"}, {Name: "b"}, {Name: "c"}}
enc, err := row.NewEncoder("dump", cols, []int{0, 1, 2}, row.Options{})
if err != nil {
t.Fatal(err)
}
rows := []row.Row{
{Values: []row.Value{{Raw: []byte("")}, {}, {Raw: []byte{0, 1, 2, 255}}}},
{Values: []row.Value{{Raw: []byte("tab\there")}, {Raw: []byte("line\nbreak")}, {Raw: []byte("back\\slash")}}},
{Values: []row.Value{{Raw: []byte("中文")}, {Raw: []byte("😀")}, {Raw: []byte("")}}},
}
var buf bytes.Buffer
for _, r := range rows {
b, err := enc.Encode(nil, r)
if err != nil {
t.Fatal(err)
}
buf.Write(b)
}
dr := NewDumpReader(&buf)
for i, want := range rows {
got, err := dr.ReadRow()
if err != nil {
t.Fatalf("row %d: %v", i, err)
}
if len(got.Values) != len(want.Values) {
t.Fatalf("row %d: %d fields", i, len(got.Values))
}
for j := range want.Values {
w, g := want.Values[j], got.Values[j]
if w.IsNull() != g.IsNull() || !bytes.Equal(w.Raw, g.Raw) {
t.Fatalf("row %d field %d: got %+v want %+v", i, j, g, w)
}
}
}
if _, err := dr.ReadRow(); err != io.EOF {
t.Fatalf("expected EOF, got %v", err)
}
}
+214
View File
@@ -0,0 +1,214 @@
package importer
import (
"context"
"database/sql"
"fmt"
"io"
"os"
"sync/atomic"
"github.com/go-sql-driver/mysql"
"archer/internal/archererr"
"archer/internal/cli"
"archer/internal/row"
"archer/internal/shard"
"archer/internal/sink"
)
// insertBatchRows bounds the decoder's memory independently of
// max_allowed_packet; sink.DB still splits each batch by packet size.
const insertBatchRows = 1000
type worker struct {
conn *sql.Conn
sink *sink.DB
cfg *cli.ImportConfig
archive *Archive
mapping *Mapping
dbName string
}
func newWorker(ctx context.Context, db *sql.DB, cfg *cli.ImportConfig, archive *Archive, mapping *Mapping, dbName string) (*worker, error) {
conn, err := db.Conn(ctx)
if err != nil {
return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("acquire connection: %w", err))
}
if err := SetSession(ctx, conn, cfg.Charset, cfg.TimeZone); err != nil {
conn.Close()
return nil, err
}
w := &worker{conn: conn, cfg: cfg, archive: archive, mapping: mapping, dbName: dbName}
if cfg.Engine == "insert" {
s, err := sink.NewDB(ctx, db, dbName, cfg.DestTable, mapping.InsertColumns, cfg.Mode, mapping.InsertFieldIdx)
if err != nil {
conn.Close()
return nil, archererr.Wrap(archererr.KindValidation, err)
}
w.sink = s
}
return w, nil
}
func (w *worker) close() {
if w.conn != nil {
_ = w.conn.Close()
}
}
// importShard loads one shard and returns affected rows plus, for ignore mode,
// the number of rows the server skipped.
func (w *worker) importShard(ctx context.Context, e shard.Entry) (affected, skipped int64, err error) {
path := shardPath(w.archive.Root, e)
if w.cfg.Verify == "pre" {
if err := VerifyShard(path, e); err != nil {
return 0, 0, err
}
}
if w.cfg.Engine == "insert" {
return w.insertShard(ctx, e, path)
}
return w.loadShard(ctx, e, path)
}
func (w *worker) loadShard(ctx context.Context, e shard.Entry, path string) (int64, int64, error) {
name := newReaderName(e.Seq)
mysql.RegisterReaderHandler(name, func() io.Reader {
return openShardReader(path, w.archive.Compressor)
})
defer mysql.DeregisterReaderHandler(name)
q := BuildLoadSQL(w.dbName, w.cfg.DestTable, w.cfg.Charset, w.mapping, w.cfg.Mode, name)
res, err := w.conn.ExecContext(ctx, q)
if err != nil {
return 0, 0, archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: LOAD DATA: %w", e.Seq, err))
}
affected, _ := res.RowsAffected()
if err := w.checkWarnings(ctx, e.Seq); err != nil {
return affected, 0, err
}
skipped, err := CheckRowRule(w.cfg.Mode, e.Rows, affected)
if err != nil {
return affected, 0, fmt.Errorf("shard %d: %w", e.Seq, err)
}
return affected, skipped, nil
}
func (w *worker) insertShard(ctx context.Context, e shard.Entry, path string) (int64, int64, error) {
f, err := os.Open(path)
if err != nil {
return 0, 0, archererr.Wrap(archererr.KindValidation, fmt.Errorf("shard %d: open: %w", e.Seq, err))
}
defer f.Close()
r, err := w.archive.Compressor.Reader(f)
if err != nil {
return 0, 0, archererr.Wrap(archererr.KindValidation, fmt.Errorf("shard %d: decompress: %w", e.Seq, err))
}
defer r.Close()
tx, err := w.conn.BeginTx(ctx, nil)
if err != nil {
return 0, 0, archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: begin transaction: %w", e.Seq, err))
}
committed := false
defer func() {
if !committed {
_ = tx.Rollback()
}
}()
dr := NewDumpReader(r)
chunk := &row.Chunk{}
var affected int64
flush := func() error {
if len(chunk.Rows) == 0 {
return nil
}
n, err := w.sink.InsertChunk(ctx, tx, chunk)
if err != nil {
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: %w", e.Seq, err))
}
affected += n
chunk = &row.Chunk{}
return w.checkWarnings(ctx, e.Seq)
}
for {
rw, err := dr.ReadRow()
if err == io.EOF {
break
}
if err != nil {
return affected, 0, archererr.Wrap(archererr.KindValidation, fmt.Errorf("shard %d: decode: %w", e.Seq, err))
}
if len(rw.Values) != len(w.mapping.FileColumns) {
return affected, 0, archererr.Newf(archererr.KindValidation,
"shard %d: row has %d fields but the mapping expects %d", e.Seq, len(rw.Values), len(w.mapping.FileColumns))
}
chunk.Rows = append(chunk.Rows, rw)
if len(chunk.Rows) >= insertBatchRows {
if err := flush(); err != nil {
return affected, 0, err
}
}
}
if err := flush(); err != nil {
return affected, 0, err
}
skipped, err := CheckRowRule(w.cfg.Mode, e.Rows, affected)
if err != nil {
return affected, 0, fmt.Errorf("shard %d: %w", e.Seq, err)
}
if err := tx.Commit(); err != nil {
return affected, 0, archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: commit: %w", e.Seq, err))
}
committed = true
return affected, skipped, nil
}
func (w *worker) checkWarnings(ctx context.Context, seq int64) error {
warnings, err := ShowWarnings(ctx, w.conn)
if err != nil {
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: SHOW WARNINGS: %w", seq, err))
}
return CheckWarnings(warnings, w.cfg.AllowWarnings, w.cfg.Mode, seq)
}
var readerCounter atomic.Uint64
func newReaderName(seq int64) string {
return fmt.Sprintf("archer-import-%d-%d-%d", os.Getpid(), seq, readerCounter.Add(1))
}
type errReader struct{ err error }
func (e errReader) Read([]byte) (int, error) { return 0, e.err }
type readCloserPair struct {
r io.ReadCloser
f *os.File
}
func (p *readCloserPair) Read(b []byte) (int, error) { return p.r.Read(b) }
func (p *readCloserPair) Close() error {
err := p.r.Close()
if ferr := p.f.Close(); err == nil {
err = ferr
}
return err
}
func openShardReader(path string, comp *shard.Compressor) io.Reader {
f, err := os.Open(path)
if err != nil {
return errReader{err}
}
r, err := comp.Reader(f)
if err != nil {
_ = f.Close()
return errReader{err}
}
return &readCloserPair{r: r, f: f}
}
+429
View File
@@ -0,0 +1,429 @@
package importer
import (
"context"
"database/sql"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"testing"
"github.com/go-sql-driver/mysql"
"archer/internal/app"
"archer/internal/cli"
)
func TestMain(m *testing.M) {
if dsn := os.Getenv("ARCHER_TEST_DSN"); dsn != "" {
if cfg, err := mysql.ParseDSN(dsn); err == nil && cfg.DBName != "" {
dbName := cfg.DBName
cfg.DBName = ""
if db, err := sql.Open("mysql", cfg.FormatDSN()); err == nil {
_, _ = db.Exec("CREATE DATABASE IF NOT EXISTS `" + dbName + "`")
_, _ = db.Exec("SET GLOBAL local_infile=ON")
_, _ = db.Exec("SET GLOBAL sql_mode=''")
db.Close()
}
}
}
os.Exit(m.Run())
}
func testDSN(t *testing.T) string {
t.Helper()
d := os.Getenv("ARCHER_TEST_DSN")
if d == "" {
t.Skip("ARCHER_TEST_DSN is not set; skipping MySQL integration test")
}
return d
}
func charsetDSN(t *testing.T, dsn string) string {
t.Helper()
cfg, err := mysql.ParseDSN(dsn)
if err != nil {
t.Fatal(err)
}
if cfg.Params == nil {
cfg.Params = map[string]string{}
}
cfg.Params["charset"] = "utf8mb4"
return cfg.FormatDSN()
}
func openDB(t *testing.T, dsn string) *sql.DB {
t.Helper()
db, err := sql.Open("mysql", dsn)
if err != nil {
t.Fatal(err)
}
if err := db.Ping(); err != nil {
t.Fatal(err)
}
return db
}
func execSQL(t *testing.T, db *sql.DB, q string, args ...any) {
t.Helper()
if _, err := db.Exec(q, args...); err != nil {
t.Fatalf("exec %q: %v", q, err)
}
}
func exportTable(t *testing.T, dsn, table, dir string, extra ...string) {
t.Helper()
args := []string{"--source", dsn, "--table", table, "--where", "1=1", "--file", dir, "--progress", "off"}
args = append(args, extra...)
cfg, err := cli.Parse(args, io.Discard)
if err != nil {
t.Fatal(err)
}
if err := app.Run(context.Background(), cfg); err != nil {
t.Fatalf("export %s: %v", table, err)
}
}
func importArchive(t *testing.T, dsn, dir string, extra ...string) error {
t.Helper()
args := []string{"--file", dir, "--dest", dsn, "--progress", "off"}
args = append(args, extra...)
cfg, err := cli.ParseImport(args, io.Discard)
if err != nil {
return err
}
return Run(context.Background(), cfg)
}
func countRows(t *testing.T, db *sql.DB, table string) int {
t.Helper()
var n int
if err := db.QueryRow("SELECT COUNT(*) FROM `" + table + "`").Scan(&n); err != nil {
t.Fatal(err)
}
return n
}
// fingerprint renders an ordered result set byte-for-byte, using hex so NULL,
// tabs, newlines and binary values are all distinguishable.
func fingerprint(t *testing.T, db *sql.DB, q string) string {
t.Helper()
rows, err := db.Query(q)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
cols, err := rows.Columns()
if err != nil {
t.Fatal(err)
}
var b strings.Builder
for rows.Next() {
raw := make([]sql.RawBytes, len(cols))
ptrs := make([]any, len(cols))
for i := range raw {
ptrs[i] = &raw[i]
}
if err := rows.Scan(ptrs...); err != nil {
t.Fatal(err)
}
for i, v := range raw {
if i > 0 {
b.WriteByte('|')
}
if v == nil {
b.WriteString("<NULL>")
} else {
fmt.Fprintf(&b, "%x", []byte(v))
}
}
b.WriteByte('\n')
}
if err := rows.Err(); err != nil {
t.Fatal(err)
}
return b.String()
}
func seedSimple(t *testing.T, db *sql.DB, table string) {
t.Helper()
execSQL(t, db, "DROP TABLE IF EXISTS `"+table+"`")
execSQL(t, db, "CREATE TABLE `"+table+"` (id BIGINT NOT NULL, v VARCHAR(64) NULL, PRIMARY KEY (id)) DEFAULT CHARSET=utf8mb4")
execSQL(t, db, "INSERT INTO `"+table+"` (id, v) VALUES (1,'a'),(2,'b'),(3,'c'),(4,'tab\\there'),(5,NULL)")
}
func freshTable(t *testing.T, db *sql.DB, table string) {
t.Helper()
execSQL(t, db, "DROP TABLE IF EXISTS `"+table+"`")
execSQL(t, db, "CREATE TABLE `"+table+"` (id BIGINT NOT NULL, v VARCHAR(64) NULL, PRIMARY KEY (id)) DEFAULT CHARSET=utf8mb4")
}
func TestIntegrationExportImport(t *testing.T) {
dsn := testDSN(t)
db := openDB(t, dsn)
defer db.Close()
seedSimple(t, db, "it_imp_src")
dir := t.TempDir()
exportTable(t, dsn, "it_imp_src", dir)
freshTable(t, db, "it_imp_dst")
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_dst", "--assume-all-columns"); err != nil {
t.Fatal(err)
}
if n := countRows(t, db, "it_imp_dst"); n != 5 {
t.Fatalf("imported %d rows", n)
}
if got, want := fingerprint(t, db, "SELECT id, v FROM it_imp_dst ORDER BY id"),
fingerprint(t, db, "SELECT id, v FROM it_imp_src ORDER BY id"); got != want {
t.Fatalf("rows differ:\n%s\nwant:\n%s", got, want)
}
}
func TestIntegrationCompression(t *testing.T) {
dsn := testDSN(t)
db := openDB(t, dsn)
defer db.Close()
seedSimple(t, db, "it_imp_comp_src")
for _, comp := range []string{"none", "gzip", "zstd", "xz"} {
dir := t.TempDir()
exportTable(t, dsn, "it_imp_comp_src", dir, "--compress", comp)
if files, _ := filepath.Glob(filepath.Join(dir, "*.dump*")); len(files) != 1 {
t.Fatalf("%s: shard files %v", comp, files)
}
dst := "it_imp_comp_" + comp
freshTable(t, db, dst)
if err := importArchive(t, dsn, dir, "--dest-table", dst, "--assume-all-columns"); err != nil {
t.Fatalf("%s: %v", comp, err)
}
if n := countRows(t, db, dst); n != 5 {
t.Fatalf("%s: %d rows", comp, n)
}
}
}
func TestIntegrationModes(t *testing.T) {
dsn := testDSN(t)
db := openDB(t, dsn)
defer db.Close()
seedSimple(t, db, "it_imp_modes_src")
dir := t.TempDir()
exportTable(t, dsn, "it_imp_modes_src", dir)
freshTable(t, db, "it_imp_modes_dst")
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns"); err != nil {
t.Fatal(err)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns"); err != nil {
t.Fatalf("ignore re-import must be idempotent: %v", err)
}
if n := countRows(t, db, "it_imp_modes_dst"); n != 5 {
t.Fatalf("ignore re-import changed row count: %d", n)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns", "--mode", "insert"); err == nil {
t.Fatal("insert mode must fail on duplicate keys")
}
execSQL(t, db, "UPDATE it_imp_modes_dst SET v='changed' WHERE id=2")
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns", "--mode", "replace"); err != nil {
t.Fatal(err)
}
var v string
if err := db.QueryRow("SELECT v FROM it_imp_modes_dst WHERE id=2").Scan(&v); err != nil {
t.Fatal(err)
}
if v != "b" {
t.Fatalf("replace did not restore value: %q", v)
}
}
func TestIntegrationVerifyAndPartial(t *testing.T) {
dsn := testDSN(t)
db := openDB(t, dsn)
defer db.Close()
seedSimple(t, db, "it_imp_verify_src")
dir := t.TempDir()
exportTable(t, dsn, "it_imp_verify_src", dir)
files, err := filepath.Glob(filepath.Join(dir, "*.dump"))
if err != nil || len(files) != 1 {
t.Fatalf("files %v err %v", files, err)
}
f, err := os.OpenFile(files[0], os.O_APPEND|os.O_WRONLY, 0o644)
if err != nil {
t.Fatal(err)
}
if _, err := f.WriteString("tampered\n"); err != nil {
t.Fatal(err)
}
f.Close()
freshTable(t, db, "it_imp_verify_dst")
err = importArchive(t, dsn, dir, "--dest-table", "it_imp_verify_dst", "--assume-all-columns")
if err == nil || !strings.Contains(err.Error(), "failed verification") {
t.Fatalf("got %v", err)
}
// Re-export a pristine archive for the completion-gate half.
dir = t.TempDir()
exportTable(t, dsn, "it_imp_verify_src", dir)
statePath := filepath.Join(dir, "state.json")
body, err := os.ReadFile(statePath)
if err != nil {
t.Fatal(err)
}
body = []byte(strings.Replace(string(body), `"complete": true`, `"complete": false`, 1))
if err := os.WriteFile(statePath, body, 0o644); err != nil {
t.Fatal(err)
}
freshTable(t, db, "it_imp_partial_dst")
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_partial_dst", "--assume-all-columns"); err == nil || !strings.Contains(err.Error(), "incomplete") {
t.Fatalf("incomplete archive must be refused, got %v", err)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_partial_dst", "--assume-all-columns", "--allow-partial"); err != nil {
t.Fatal(err)
}
if n := countRows(t, db, "it_imp_partial_dst"); n != 5 {
t.Fatalf("partial import: %d rows", n)
}
}
func TestIntegrationWorkersAndEnginesAgree(t *testing.T) {
dsn := testDSN(t)
db := openDB(t, dsn)
defer db.Close()
seedSimple(t, db, "it_imp_par_src")
for i := 6; i <= 20; i++ {
execSQL(t, db, "INSERT INTO it_imp_par_src (id, v) VALUES (?, ?)", i, fmt.Sprintf("v%d", i))
}
dir := t.TempDir()
exportTable(t, dsn, "it_imp_par_src", dir, "--shard-by", "count", "--shard-rows", "4", "--chunk-size", "3")
for _, dst := range []string{"it_imp_par_1", "it_imp_par_4", "it_imp_par_ins"} {
freshTable(t, db, dst)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_par_1", "--workers", "1", "--assume-all-columns"); err != nil {
t.Fatal(err)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_par_4", "--workers", "4", "--assume-all-columns"); err != nil {
t.Fatal(err)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_par_ins", "--engine", "insert", "--assume-all-columns"); err != nil {
t.Fatal(err)
}
want := fingerprint(t, db, "SELECT id, v FROM it_imp_par_1 ORDER BY id")
for _, dst := range []string{"it_imp_par_4", "it_imp_par_ins"} {
if got := fingerprint(t, db, "SELECT id, v FROM `"+dst+"` ORDER BY id"); got != want {
t.Fatalf("%s differs from serial load", dst)
}
}
}
func TestIntegrationFidelity(t *testing.T) {
dsn := charsetDSN(t, testDSN(t))
db := openDB(t, dsn)
defer db.Close()
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_fid_src")
execSQL(t, db, `CREATE TABLE it_imp_fid_src (
id BIGINT NOT NULL PRIMARY KEY,
s VARCHAR(128) NULL,
b BLOB NULL,
n INT NULL) DEFAULT CHARSET=utf8mb4`)
execSQL(t, db, "INSERT INTO it_imp_fid_src (id, s, b, n) VALUES (?,?,?,?)", 1, nil, nil, nil)
execSQL(t, db, "INSERT INTO it_imp_fid_src (id, s, b, n) VALUES (?,?,?,?)", 2, "tab\there\nnewline\\back", []byte{0x00, 0x01, 0x02, 0x03, 0xff}, 0)
execSQL(t, db, "INSERT INTO it_imp_fid_src (id, s, b, n) VALUES (?,?,?,?)", 3, "中文😀", []byte("hi"), 42)
dir := t.TempDir()
exportTable(t, dsn, "it_imp_fid_src", dir)
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_fid_dst")
execSQL(t, db, `CREATE TABLE it_imp_fid_dst (
id BIGINT NOT NULL PRIMARY KEY,
s VARCHAR(128) NULL,
b BLOB NULL,
n INT NULL) DEFAULT CHARSET=utf8mb4`)
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_fid_dst", "--assume-all-columns", "--charset", "binary"); err != nil {
t.Fatal(err)
}
q := "SELECT id, s, HEX(b), n FROM %s ORDER BY id"
got := fingerprint(t, db, fmt.Sprintf(q, "it_imp_fid_dst"))
want := fingerprint(t, db, fmt.Sprintf(q, "it_imp_fid_src"))
if got != want {
t.Fatalf("fidelity mismatch:\n got: %s\nwant: %s", got, want)
}
}
func TestIntegrationColumnMappings(t *testing.T) {
dsn := charsetDSN(t, testDSN(t))
db := openDB(t, dsn)
defer db.Close()
// (a) archive-recorded column list
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_cols_src")
execSQL(t, db, "CREATE TABLE it_imp_cols_src (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "INSERT INTO it_imp_cols_src (id, a, b) VALUES (1,'a1','b1'),(2,'a2','b2')")
dir := t.TempDir()
exportTable(t, dsn, "it_imp_cols_src", dir, "--columns", "id,a")
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_cols_dst")
execSQL(t, db, "CREATE TABLE it_imp_cols_dst (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))")
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_cols_dst"); err != nil {
t.Fatal(err)
}
if got := fingerprint(t, db, "SELECT id, a, b FROM it_imp_cols_dst ORDER BY id"); got != "31|6131|<NULL>\n32|6132|<NULL>\n" {
t.Fatalf("archive-recorded mapping: %q", got)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_cols_dst", "--columns", "id,a"); err == nil {
t.Fatal("archive column list plus --columns must be rejected")
}
// (b) explicit --columns on a default (all-columns) archive
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_all_src")
execSQL(t, db, "CREATE TABLE it_imp_all_src (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))")
execSQL(t, db, "INSERT INTO it_imp_all_src (id, a, b) VALUES (1,'a1','b1')")
dir = t.TempDir()
exportTable(t, dsn, "it_imp_all_src", dir)
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_all_dst")
execSQL(t, db, "CREATE TABLE it_imp_all_dst (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))")
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst"); err == nil {
t.Fatal("default all-columns archive without a mapping must be refused")
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst", "--columns", "id,a,b"); err != nil {
t.Fatal(err)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst", "--columns", "id,nope"); err == nil || !strings.Contains(err.Error(), "does not exist") {
t.Fatalf("missing column: %v", err)
}
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst", "--columns", "id"); err == nil || !strings.Contains(err.Error(), "fields") {
t.Fatalf("field count mismatch: %v", err)
}
// (c) positional mapping with a generated column
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_gen_src")
execSQL(t, db, `CREATE TABLE it_imp_gen_src (
id BIGINT NOT NULL PRIMARY KEY,
a INT NOT NULL,
b INT NOT NULL,
total INT AS (a + b) STORED)`)
execSQL(t, db, "INSERT INTO it_imp_gen_src (id, a, b) VALUES (1,2,3),(2,10,20)")
dir = t.TempDir()
exportTable(t, dsn, "it_imp_gen_src", dir)
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_gen_dst")
execSQL(t, db, `CREATE TABLE it_imp_gen_dst (
id BIGINT NOT NULL PRIMARY KEY,
a INT NOT NULL,
b INT NOT NULL,
total INT AS (a + b) STORED)`)
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_gen_dst", "--assume-all-columns"); err != nil {
t.Fatal(err)
}
for _, want := range []struct{ id, a, b, total int }{{1, 2, 3, 5}, {2, 10, 20, 30}} {
var gotID, gotA, gotB, gotTotal int
if err := db.QueryRow("SELECT id, a, b, total FROM it_imp_gen_dst WHERE id=?", want.id).
Scan(&gotID, &gotA, &gotB, &gotTotal); err != nil {
t.Fatal(err)
}
if gotID != want.id || gotA != want.a || gotB != want.b || gotTotal != want.total {
t.Fatalf("row %d = (%d,%d,%d,%d), want %+v", want.id, gotID, gotA, gotB, gotTotal, want)
}
}
}
+418
View File
@@ -0,0 +1,418 @@
package importer
import (
"context"
"database/sql"
"fmt"
"io"
"os"
"sync"
"time"
"github.com/go-sql-driver/mysql"
"archer/internal/archererr"
"archer/internal/cli"
"archer/internal/progress"
"archer/internal/schema"
"archer/internal/shard"
)
// Stats collects the --statistics counters for one import run.
type Stats struct {
Started time.Time
Elapsed time.Duration
Selected int
NotSelected int
Succeeded int
Failed int
ExpectedRows int64
ImportedRows int64
SkippedRows int64
Bytes int64
IgnoredFiles int
Partial bool
}
// Run executes one import.
func Run(ctx context.Context, cfg *cli.ImportConfig) error {
if err := cfg.Validate(); err != nil {
return err
}
stats := &Stats{Started: time.Now()}
archive, err := LoadArchive(cfg.FileDir)
if err != nil {
return err
}
if err := archive.CheckComplete(cfg.AllowPartial); err != nil {
return err
}
if err := archive.CheckMappingChoice(cfg.Columns, cfg.AssumeAll); err != nil {
return err
}
stats.Partial = cfg.AllowPartial && (archive.State == nil || !archive.State.Complete)
selected, warnings, err := shard.ApplySelection(archive.Entries, cfg.Selectors)
if err != nil {
return archererr.Newf(archererr.KindConfig, "%v", err)
}
if err := CheckShardFiles(archive.Root, selected); err != nil {
return err
}
format, compress, comp, err := ResolveFormat(selected)
if err != nil {
return err
}
archive.Format, archive.Compress, archive.Compressor = format, compress, comp
stats.Selected = len(selected)
stats.NotSelected = len(archive.Entries) - len(selected)
for _, e := range selected {
stats.ExpectedRows += e.Rows
stats.Bytes += e.Bytes
}
if cfg.DestTable == "" {
cfg.DestTable = archive.SourceTable
}
if cfg.DestTable == "" {
return archererr.Newf(archererr.KindConfig,
"--dest-table is required: the archive does not record the source table name")
}
if cfg.DryRun {
fields, hasRow, ferr := FirstLineFieldCount(shardPath(archive.Root, selected[0]), comp)
if ferr != nil {
return archererr.Wrap(archererr.KindValidation, ferr)
}
return PrintDryRun(os.Stdout, cfg, archive, selected, warnings, compress, fields, hasRow)
}
for _, w := range warnings {
fmt.Fprintln(os.Stderr, "warning:", w)
}
if len(warnings) > 0 {
fmt.Fprintln(os.Stderr, "warning: shard selection is whole-shard; rows outside the requested key range may be imported")
}
destCfg, err := cfg.DestConfig()
if err != nil {
return archererr.Wrap(archererr.KindConfig, err)
}
db, err := openDest(destCfg, cfg.Workers)
if err != nil {
return err
}
defer db.Close()
if err := checkTarget(ctx, db, cfg.Engine); err != nil {
return err
}
if err := checkTableExists(ctx, db, destCfg.DBName, cfg.DestTable); err != nil {
return err
}
table, err := schema.Load(ctx, db, destCfg.DBName, cfg.DestTable)
if err != nil {
return archererr.Wrap(archererr.KindValidation, err)
}
mapping, err := BuildMapping(table, archive.ArchivedColumns, cfg.Columns, cfg.AssumeAll)
if err != nil {
return err
}
if mapping.Positional {
fmt.Fprintln(os.Stderr, "warning: --assume-all-columns accepts positional mapping; only the field count is checked, not column names or order")
}
if err := CheckFieldCount(shardPath(archive.Root, selected[0]), comp, len(mapping.FileColumns)); err != nil {
return err
}
rep := startReporter(cfg, stats.ExpectedRows)
if rep != nil {
defer rep.Stop()
}
if err := runWorkers(ctx, db, destCfg.DBName, cfg, archive, mapping, selected, stats, rep); err != nil {
if archererr.KindOf(err) == archererr.KindInterrupted && rep != nil {
rep.Interrupted()
}
return err
}
stats.Elapsed = time.Since(stats.Started)
stats.IgnoredFiles = UnregisteredFiles(archive.Root, archive.Entries)
if rep != nil {
rep.SetShards(int64(len(selected)))
rep.SetBytes(stats.Bytes)
rep.Done()
}
if cfg.Statistics {
PrintStats(os.Stdout, archive, stats)
}
return nil
}
func openDest(cfg *mysql.Config, workers int) (*sql.DB, error) {
db, err := sql.Open("mysql", cfg.FormatDSN())
if err != nil {
return nil, archererr.Wrap(archererr.KindConnection, err)
}
db.SetMaxOpenConns(workers + 2)
db.SetMaxIdleConns(workers + 1)
db.SetConnMaxLifetime(time.Hour)
if err := db.Ping(); err != nil {
db.Close()
return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("connect %s: %w", cfg.Addr, err))
}
return db, nil
}
func checkTarget(ctx context.Context, db *sql.DB, engine string) error {
var localInfile int
var version string
if err := db.QueryRowContext(ctx, "SELECT @@local_infile, @@version").Scan(&localInfile, &version); err != nil {
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("query local_infile: %w", err))
}
if engine == "load" && localInfile == 0 {
return archererr.Newf(archererr.KindConfig,
"target server (%s) has local_infile disabled; enable it on the server/proxy, or use --engine insert", version)
}
return nil
}
func checkTableExists(ctx context.Context, db *sql.DB, dbName, table string) error {
var n int
err := db.QueryRowContext(ctx, `
SELECT COUNT(*) FROM information_schema.tables
WHERE table_schema = ? AND table_name = ?`, dbName, table).Scan(&n)
if err != nil {
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("check target table: %w", err))
}
if n == 0 {
return archererr.Newf(archererr.KindValidation,
"target table %s.%s does not exist; create it before importing (archer does not create tables)", dbName, table)
}
return nil
}
func startReporter(cfg *cli.ImportConfig, total int64) *progress.Reporter {
tty := progress.IsTerminal(os.Stderr)
enabled := cfg.Progress.Enabled(tty)
r := progress.New(progress.Options{
Out: os.Stderr,
Enabled: enabled,
Live: enabled && tty,
Interval: time.Second,
RowEvery: cfg.Progress.Rows,
})
r.Start()
r.SetEstimate(&progress.Estimate{Kind: "rows", Lo: 0, Hi: float64(total)})
r.SetPosition(0)
return r
}
func runWorkers(ctx context.Context, db *sql.DB, dbName string, cfg *cli.ImportConfig, archive *Archive, mapping *Mapping, selected []shard.Entry, stats *Stats, rep *progress.Reporter) error {
parent := ctx
ctx, cancel := context.WithCancel(ctx)
defer cancel()
var (
wg sync.WaitGroup
mu sync.Mutex
firstErr error
failedSeq int64
processedRows int64
rowsAtFailure int64
importedRows int64
skippedRows int64
okShards int
)
recordFailure := func(e shard.Entry, err error) {
mu.Lock()
defer mu.Unlock()
if firstErr != nil {
return
}
if parent.Err() != nil {
firstErr = archererr.Wrap(archererr.KindInterrupted, parent.Err())
} else {
firstErr = err
}
failedSeq = e.Seq
rowsAtFailure = processedRows
}
ranges := splitRanges(len(selected), cfg.Workers)
for _, rg := range ranges {
if rg[0] >= rg[1] {
continue
}
part := selected[rg[0]:rg[1]]
wg.Add(1)
go func(part []shard.Entry) {
defer wg.Done()
if err := ctx.Err(); err != nil {
recordFailure(part[0], err)
return
}
w, err := newWorker(ctx, db, cfg, archive, mapping, dbName)
if err != nil {
recordFailure(part[0], err)
cancel()
return
}
defer w.close()
for _, e := range part {
if err := ctx.Err(); err != nil {
recordFailure(e, err)
return
}
affected, skipped, err := w.importShard(ctx, e)
if err != nil {
recordFailure(e, err)
cancel()
return
}
mu.Lock()
processedRows += e.Rows
importedRows += affected
skippedRows += skipped
okShards++
done := processedRows
mu.Unlock()
if rep != nil {
rep.AddRows(e.Rows)
rep.SetPosition(float64(done))
}
}
}(part)
}
wg.Wait()
mu.Lock()
stats.Succeeded = okShards
stats.ImportedRows = importedRows
stats.SkippedRows = skippedRows
err := firstErr
seq := failedSeq
atFailure := rowsAtFailure
doneRows := processedRows
mu.Unlock()
if err != nil {
if archererr.KindOf(err) == archererr.KindInterrupted {
return fmt.Errorf("interrupted after %d rows: %w", doneRows, err)
}
stats.Failed = 1
return fmt.Errorf("shard %d failed after %d rows: %w", seq, atFailure, err)
}
return nil
}
func splitRanges(n, workers int) [][2]int {
if n <= 0 || workers <= 0 {
return nil
}
if workers > n {
workers = n
}
out := make([][2]int, 0, workers)
base, rem := n/workers, n%workers
start := 0
for i := 0; i < workers; i++ {
size := base
if i < rem {
size++
}
out = append(out, [2]int{start, start + size})
start += size
}
return out
}
// PrintDryRun prints the resolved configuration and selection without opening
// a database connection.
func PrintDryRun(w io.Writer, cfg *cli.ImportConfig, archive *Archive, selected []shard.Entry, warnings []string, compress string, fields int, hasRow bool) error {
fmt.Fprintln(w, "== import configuration ==")
fmt.Fprintf(w, "archive: %s\n", archive.Root)
fmt.Fprintf(w, "dest: %s\n", redactDSN(cfg.DestDSN))
fmt.Fprintf(w, "dest table: %s\n", cfg.DestTable)
if len(archive.ArchivedColumns) > 0 {
fmt.Fprintf(w, "archive columns: %s\n", join(archive.ArchivedColumns))
} else if len(cfg.Columns) > 0 {
fmt.Fprintf(w, "columns: %s\n", join(cfg.Columns))
} else if cfg.AssumeAll {
fmt.Fprintf(w, "columns: positional (--assume-all-columns)\n")
} else {
fmt.Fprintf(w, "columns: (unresolved; --columns or --assume-all-columns is required)\n")
}
fmt.Fprintf(w, "engine: %s\n", cfg.Engine)
fmt.Fprintf(w, "mode: %s\n", cfg.Mode)
fmt.Fprintf(w, "verify: %s\n", cfg.Verify)
fmt.Fprintf(w, "workers: %d\n", cfg.Workers)
fmt.Fprintf(w, "charset: %s\n", cfg.Charset)
if cfg.TimeZone != "" {
fmt.Fprintf(w, "time zone: %s\n", cfg.TimeZone)
}
fmt.Fprintf(w, "compression: %s\n", compress)
switch {
case archive.State == nil:
fmt.Fprintln(w, "archive complete: unknown (state.json missing)")
case archive.Complete:
fmt.Fprintln(w, "archive complete: true")
default:
fmt.Fprintln(w, "archive complete: false (--allow-partial given)")
}
if hasRow {
fmt.Fprintf(w, "first row fields: %d\n", fields)
}
fmt.Fprintf(w, "\n== selected shards (%d) ==\n", len(selected))
var rows, bytes int64
for _, e := range selected {
date := e.Date
if date == "" {
date = "-"
}
fmt.Fprintf(w, " seq=%-6d pk=%-24s date=%-12s rows=%-10d bytes=%-10d %s\n",
e.Seq, e.PKLo+"-"+e.PKHi, date, e.Rows, e.Bytes, e.Path)
rows += e.Rows
bytes += e.Bytes
}
fmt.Fprintf(w, "total: %d shards, %d rows, %d bytes\n", len(selected), rows, bytes)
fmt.Fprintln(w, "\nno database connection was made and no data was written")
return nil
}
// PrintStats prints the --statistics summary.
func PrintStats(w io.Writer, archive *Archive, s *Stats) {
fmt.Fprintln(w, "import summary:")
fmt.Fprintf(w, " archive: %s\n", archive.Root)
fmt.Fprintf(w, " partial archive: %t\n", s.Partial)
fmt.Fprintf(w, " shards selected: %d\n", s.Selected)
fmt.Fprintf(w, " shards succeeded: %d\n", s.Succeeded)
fmt.Fprintf(w, " shards failed: %d\n", s.Failed)
fmt.Fprintf(w, " shards not selected: %d\n", s.NotSelected)
fmt.Fprintf(w, " rows expected: %d\n", s.ExpectedRows)
fmt.Fprintf(w, " rows affected: %d\n", s.ImportedRows)
fmt.Fprintf(w, " rows skipped: %d\n", s.SkippedRows)
fmt.Fprintf(w, " bytes: %d\n", s.Bytes)
fmt.Fprintf(w, " ignored files: %d (present on disk, not in manifest)\n", s.IgnoredFiles)
fmt.Fprintf(w, " elapsed: %s\n", s.Elapsed.Round(time.Millisecond))
}
func join(parts []string) string {
out := ""
for i, p := range parts {
if i > 0 {
out += ", "
}
out += p
}
return out
}
func redactDSN(dsn string) string {
c, err := mysql.ParseDSN(dsn)
if err != nil {
return "<invalid dsn>"
}
c.Passwd = ""
return c.FormatDSN()
}
+104
View File
@@ -0,0 +1,104 @@
package importer
import (
"bytes"
"context"
"strings"
"testing"
"archer/internal/archererr"
"archer/internal/cli"
"archer/internal/shard"
)
func TestSplitRanges(t *testing.T) {
got := splitRanges(10, 3)
want := [][2]int{{0, 4}, {4, 7}, {7, 10}}
if len(got) != len(want) {
t.Fatalf("got %v", got)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("got %v want %v", got, want)
}
}
if got := splitRanges(2, 5); len(got) != 2 || got[1] != [2]int{1, 2} {
t.Fatalf("workers beyond shards: %v", got)
}
if got := splitRanges(0, 3); got != nil {
t.Fatalf("empty selection: %v", got)
}
}
func TestRunConfigErrorExitCode(t *testing.T) {
err := Run(context.Background(), cli.ImportDefaults())
if err == nil {
t.Fatal("expected an error")
}
if got := archererr.ExitCode(err); got != 2 {
t.Fatalf("exit code = %d, want 2 (%v)", got, err)
}
}
func TestRunEmptySelectionIsConfigError(t *testing.T) {
root := t.TempDir()
writeManifest(t, root, []shard.Entry{{Seq: 1, Path: "000001.1-1.dump", Rows: 1}})
writeShard(t, root, "000001.1-1.dump", "1\n")
saveState(t, root, true, map[string]string{"table": "src"})
cfg := cli.ImportDefaults()
cfg.FileDir = root
cfg.DestDSN = "u:p@tcp(127.0.0.1:3306)/db"
cfg.Selects = []string{"seq:9-9"}
err := Run(context.Background(), cfg)
if err == nil || !strings.Contains(err.Error(), "no shards matched") {
t.Fatalf("got %v", err)
}
if got := archererr.ExitCode(err); got != 2 {
t.Fatalf("exit code = %d, want 2", got)
}
}
func TestRunDryRunDoesNotConnect(t *testing.T) {
root := t.TempDir()
writeManifest(t, root, []shard.Entry{{Seq: 1, Path: "000001.1-1.dump", PKLo: "1", PKHi: "1", Rows: 1, Bytes: 3}})
writeShard(t, root, "000001.1-1.dump", "1\tx\n")
saveState(t, root, true, map[string]string{"table": "src"})
cfg := cli.ImportDefaults()
cfg.FileDir = root
cfg.DestDSN = "u:p@tcp(127.0.0.1:1)/db" // would fail if a connection were attempted
cfg.DryRun = true
// PrintDryRun goes to stdout; Run reaching it without a connection error is
// what matters here.
if err := Run(context.Background(), cfg); err != nil {
t.Fatal(err)
}
}
func TestPrintDryRun(t *testing.T) {
root := t.TempDir()
entries := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump", PKLo: "1", PKHi: "1", Rows: 2, Bytes: 10}}
writeManifest(t, root, entries)
saveState(t, root, true, map[string]string{"table": "src", "columns": "id,title"})
archive, err := LoadArchive(root)
if err != nil {
t.Fatal(err)
}
cfg := cli.ImportDefaults()
cfg.FileDir = root
cfg.DestDSN = "u:p@tcp(h:3306)/db"
cfg.DestTable = "src"
var buf bytes.Buffer
if err := PrintDryRun(&buf, cfg, archive, entries, nil, "none", 2, true); err != nil {
t.Fatal(err)
}
out := buf.String()
for _, want := range []string{"archive:", "dest table:", "selected shards (1)", "seq=1", "total: 1 shards, 2 rows, 10 bytes", "no database connection"} {
if !strings.Contains(out, want) {
t.Fatalf("missing %q in:\n%s", want, out)
}
}
}
+167
View File
@@ -0,0 +1,167 @@
package importer
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"fmt"
"io"
"os"
"strings"
"archer/internal/archererr"
"archer/internal/shard"
"archer/internal/sqlutil"
)
// BuildLoadSQL renders the LOAD DATA statement. Field and line rules match
// internal/row.dumpEncoder exactly: TAB, LF, backslash escaping and \N.
func BuildLoadSQL(dbName, table, charset string, m *Mapping, mode, readerName string) string {
var b strings.Builder
b.WriteString("LOAD DATA LOCAL INFILE '")
b.WriteString(escapeSQLString("Reader::" + readerName))
b.WriteString("'")
switch mode {
case "ignore":
b.WriteString(" IGNORE")
case "replace":
b.WriteString(" REPLACE")
}
b.WriteString(" INTO TABLE ")
b.WriteString(sqlutil.Qualified(dbName, table))
b.WriteString(" CHARACTER SET ")
b.WriteString(charset)
b.WriteString(` FIELDS TERMINATED BY '\t' ESCAPED BY '\\' LINES TERMINATED BY '\n'`)
if len(m.LoadList) > 0 {
b.WriteString(" (")
b.WriteString(strings.Join(m.LoadList, ", "))
b.WriteString(")")
}
return b.String()
}
func escapeSQLString(s string) string {
s = strings.ReplaceAll(s, `\`, `\\`)
return strings.ReplaceAll(s, `'`, `''`)
}
// CheckRowRule applies the per-mode affected-row rule and returns the number
// of rows that were skipped (only meaningful for ignore mode).
func CheckRowRule(mode string, expected, affected int64) (int64, error) {
switch mode {
case "insert":
if affected != expected {
return 0, archererr.Newf(archererr.KindValidation,
"insert expected %d rows but wrote %d", expected, affected)
}
return 0, nil
case "ignore":
if affected > expected {
return 0, archererr.Newf(archererr.KindValidation,
"ignore wrote %d rows but the manifest records only %d", affected, expected)
}
return expected - affected, nil
default: // replace may delete+insert, so affected can exceed rows
return 0, nil
}
}
// VerifyShard streams the compressed shard through SHA-256 and compares it
// with the manifest before any bytes are sent to the server.
func VerifyShard(path string, e shard.Entry) error {
if e.SHA256 == "" {
return nil
}
f, err := os.Open(path)
if err != nil {
return archererr.Newf(archererr.KindValidation, "open shard %s: %v", e.Path, err)
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return archererr.Newf(archererr.KindValidation, "hash shard %s: %v", e.Path, err)
}
got := hex.EncodeToString(h.Sum(nil))
if !strings.EqualFold(got, e.SHA256) {
return archererr.Newf(archererr.KindValidation,
"shard %d (%s) failed verification: manifest sha256 %s, actual %s", e.Seq, e.Path, e.SHA256, got)
}
return nil
}
// Warning is one SHOW WARNINGS row.
type Warning struct {
Level string
Code int
Message string
}
func (w Warning) String() string { return fmt.Sprintf("%s %d: %s", w.Level, w.Code, w.Message) }
// ShowWarnings returns the current session warnings.
func ShowWarnings(ctx context.Context, conn *sql.Conn) ([]Warning, error) {
rows, err := conn.QueryContext(ctx, "SHOW WARNINGS")
if err != nil {
return nil, err
}
defer rows.Close()
var out []Warning
for rows.Next() {
var w Warning
if err := rows.Scan(&w.Level, &w.Code, &w.Message); err != nil {
return nil, err
}
out = append(out, w)
}
return out, rows.Err()
}
// CheckWarnings turns server warnings into a per-shard failure unless the
// operator explicitly allowed them. Duplicate-key warnings under ignore mode
// are expected skips (they are the mechanism that makes re-import idempotent),
// so they are not failures; the skipped count comes from affected rows.
func CheckWarnings(warnings []Warning, allow bool, mode string, seq int64) error {
if allow || len(warnings) == 0 {
return nil
}
fatal := warnings[:0:0]
for _, w := range warnings {
if mode == "ignore" && w.Code == 1062 {
continue
}
fatal = append(fatal, w)
}
if len(fatal) == 0 {
return nil
}
const maxShown = 5
shown := fatal
suffix := ""
if len(shown) > maxShown {
suffix = fmt.Sprintf(" (and %d more)", len(shown)-maxShown)
shown = shown[:maxShown]
}
parts := make([]string, len(shown))
for i, w := range shown {
parts[i] = w.String()
}
return archererr.Newf(archererr.KindValidation,
"shard %d produced %d server warning(s); pass --allow-warnings to continue: %s%s",
seq, len(fatal), strings.Join(parts, "; "), suffix)
}
// SetSession applies the loader's character set and optional time zone to one
// fixed connection.
func SetSession(ctx context.Context, conn *sql.Conn, charset, timeZone string) error {
if _, err := conn.ExecContext(ctx, "SET NAMES "+charset); err != nil {
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("set character set %s: %w", charset, err))
}
if timeZone != "" {
q := "SET time_zone = '" + escapeSQLString(timeZone) + "'"
if _, err := conn.ExecContext(ctx, q); err != nil {
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("set time_zone %q: %w", timeZone, err))
}
}
return nil
}
+116
View File
@@ -0,0 +1,116 @@
package importer
import (
"crypto/sha256"
"encoding/hex"
"os"
"path/filepath"
"strings"
"testing"
"archer/internal/shard"
)
func TestBuildLoadSQLShape(t *testing.T) {
m := &Mapping{LoadList: []string{"`id`", "`title`"}}
q := BuildLoadSQL("db", "orders", "utf8mb4", m, "ignore", "r1")
for _, want := range []string{
"LOAD DATA LOCAL INFILE 'Reader::r1'",
"IGNORE INTO TABLE `db`.`orders`",
"CHARACTER SET utf8mb4",
`FIELDS TERMINATED BY '\t' ESCAPED BY '\\' LINES TERMINATED BY '\n'`,
"(`id`, `title`)",
} {
if !strings.Contains(q, want) {
t.Fatalf("missing %q in %s", want, q)
}
}
}
func TestBuildLoadSQLModes(t *testing.T) {
m := &Mapping{}
for _, tc := range []struct{ mode, want string }{
{"insert", "INFILE 'Reader::r' INTO TABLE"},
{"ignore", "INFILE 'Reader::r' IGNORE INTO TABLE"},
{"replace", "INFILE 'Reader::r' REPLACE INTO TABLE"},
} {
q := BuildLoadSQL("db", "t", "binary", m, tc.mode, "r")
if !strings.Contains(q, tc.want) {
t.Fatalf("%s: %s", tc.mode, q)
}
}
}
func TestBuildLoadSQLNoColumnList(t *testing.T) {
q := BuildLoadSQL("db", "t", "utf8mb4", &Mapping{}, "insert", "r")
if strings.Contains(q, "(") {
t.Fatalf("unexpected column list: %s", q)
}
}
func TestEscapeSQLString(t *testing.T) {
if got := escapeSQLString(`a'b\c`); got != `a''b\\c` {
t.Fatalf("got %q", got)
}
}
func TestCheckRowRule(t *testing.T) {
if _, err := CheckRowRule("insert", 10, 10); err != nil {
t.Fatal(err)
}
if _, err := CheckRowRule("insert", 10, 9); err == nil || !strings.Contains(err.Error(), "expected 10") {
t.Fatalf("got %v", err)
}
skipped, err := CheckRowRule("ignore", 10, 7)
if err != nil || skipped != 3 {
t.Fatalf("skipped=%d err=%v", skipped, err)
}
if _, err := CheckRowRule("ignore", 10, 11); err == nil {
t.Fatal("ignore must fail when affected exceeds rows")
}
if _, err := CheckRowRule("replace", 10, 15); err != nil {
t.Fatalf("replace only reports: %v", err)
}
}
func TestVerifyShard(t *testing.T) {
root := t.TempDir()
path := filepath.Join(root, "000001.1-1.dump")
body := []byte("1\ta\n")
if err := os.WriteFile(path, body, 0o644); err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(body)
good := shard.Entry{Seq: 1, Path: "000001.1-1.dump", SHA256: hex.EncodeToString(sum[:])}
if err := VerifyShard(path, good); err != nil {
t.Fatal(err)
}
bad := good
bad.SHA256 = strings.Repeat("0", 64)
if err := VerifyShard(path, bad); err == nil || !strings.Contains(err.Error(), "failed verification") {
t.Fatalf("got %v", err)
}
if err := VerifyShard(path, shard.Entry{Seq: 1, Path: path}); err != nil {
t.Fatalf("missing hash should be skipped: %v", err)
}
}
func TestCheckWarnings(t *testing.T) {
if err := CheckWarnings(nil, false, "insert", 1); err != nil {
t.Fatal(err)
}
w := []Warning{{Level: "Warning", Code: 1265, Message: "data truncated"}}
if err := CheckWarnings(w, true, "insert", 1); err != nil {
t.Fatal(err)
}
if err := CheckWarnings(w, false, "insert", 3); err == nil || !strings.Contains(err.Error(), "shard 3") {
t.Fatalf("got %v", err)
}
dup := []Warning{{Level: "Warning", Code: 1062, Message: "Duplicate entry '1'"}}
if err := CheckWarnings(dup, false, "ignore", 1); err != nil {
t.Fatalf("duplicate-key warnings are expected skips under ignore: %v", err)
}
if err := CheckWarnings(dup, false, "insert", 1); err == nil {
t.Fatal("duplicate warnings must fail outside ignore mode")
}
}