Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4a6b504f
commit
366d61ca25
49 files changed
+5099
-342
No files matched your search
@@ -366,6 +366,9 @@ func (a *App) execute(ctx context.Context) error {
|
||||
if a.cfg.BulkDelete {
|
||||
a.message("warning: --bulk-delete removes all rows in the scanned key range that match --where, including concurrent inserts")
|
||||
}
|
||||
if a.cfg.BulkDelete && a.cfg.IsDateShard() {
|
||||
a.message("warning: date mode --bulk-delete assumes the shard column never moves backwards relative to the scan key; a late or backfilled row can be deleted before it is archived")
|
||||
}
|
||||
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
|
||||
@@ -882,6 +882,57 @@ func TestIntegrationPurgeAndDest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationDateBulkDelete(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
defer db.Close()
|
||||
|
||||
seed := func(table string) {
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS "+table)
|
||||
execSQL(t, db, `CREATE TABLE `+table+` (
|
||||
id BIGINT NOT NULL AUTO_INCREMENT,
|
||||
v VARCHAR(32),
|
||||
created_at DATETIME NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
KEY d (created_at, id))`)
|
||||
execSQL(t, db, `INSERT INTO `+table+` (id, v, created_at) VALUES
|
||||
(1,'a','2025-01-01 00:00:00'),
|
||||
(2,'b','2025-01-02 00:00:00'),
|
||||
(3,'c','2025-01-03 00:00:00'),
|
||||
(4,'d','2025-01-04 00:00:00')`)
|
||||
}
|
||||
|
||||
seed("it_date_bulk")
|
||||
out := t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_bulk", out,
|
||||
"--shard-by", "date", "--shard-column", "created_at",
|
||||
"--purge", "--bulk-delete", "--safe-auto-increment=false", "--chunk-size", "2")...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var count int
|
||||
db.QueryRow("SELECT COUNT(*) FROM it_date_bulk").Scan(&count)
|
||||
if count != 0 {
|
||||
t.Fatalf("safe off: %d rows left", count)
|
||||
}
|
||||
if got := len(readTree(t, out)); got != 4 {
|
||||
t.Fatalf("expected 4 shards, got %d", got)
|
||||
}
|
||||
|
||||
seed("it_date_bulk_safe")
|
||||
out = t.TempDir()
|
||||
if _, err := runCLI(t, context.Background(), baseCLI(dsn, "it_date_bulk_safe", out,
|
||||
"--shard-by", "date", "--shard-column", "created_at",
|
||||
"--purge", "--bulk-delete", "--chunk-size", "2")...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.QueryRow("SELECT COUNT(*) FROM it_date_bulk_safe").Scan(&count)
|
||||
var maxID int64
|
||||
db.QueryRow("SELECT MAX(id) FROM it_date_bulk_safe").Scan(&maxID)
|
||||
if count != 1 || maxID != 4 {
|
||||
t.Fatalf("safe on: count=%d max=%d", count, maxID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationCheckpointConfigMismatch(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openRoot(t, dsn)
|
||||
|
||||
@@ -87,8 +87,8 @@ func TestValidateShardRules(t *testing.T) {
|
||||
|
||||
args = append(baseArgs(), "--shard-by", "date", "--shard-column", "c", "--bulk-delete", "--purge")
|
||||
c, _ = Parse(args, &bytes.Buffer{})
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "bulk-delete") {
|
||||
t.Fatalf("got %v", err)
|
||||
if err := c.Validate(); err != nil {
|
||||
t.Fatalf("date mode bulk delete should be allowed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
|
||||
"archer/internal/shard"
|
||||
)
|
||||
|
||||
// ImportConfig is the resolved configuration of `archer import`. It is kept
|
||||
// separate from Config so archive (export) validation rules never leak into
|
||||
// replay and vice versa.
|
||||
type ImportConfig struct {
|
||||
FileDir string
|
||||
DestDSN string
|
||||
DestTable string
|
||||
Columns []string
|
||||
AssumeAll bool
|
||||
Selects []string
|
||||
Selectors []shard.Selector
|
||||
Mode string // ignore|insert|replace
|
||||
Engine string // load|insert
|
||||
Verify string // pre|off
|
||||
Workers int
|
||||
Charset string
|
||||
TimeZone string
|
||||
AllowPartial bool
|
||||
AllowWarnings bool
|
||||
DryRun bool
|
||||
Progress ProgressValue
|
||||
Statistics bool
|
||||
Version bool
|
||||
}
|
||||
|
||||
// ImportDefaults returns the documented defaults for `archer import`.
|
||||
func ImportDefaults() *ImportConfig {
|
||||
return &ImportConfig{
|
||||
Mode: "ignore",
|
||||
Engine: "load",
|
||||
Verify: "pre",
|
||||
Workers: 1,
|
||||
Charset: "utf8mb4",
|
||||
Progress: ProgressValue{Mode: ProgressAuto},
|
||||
}
|
||||
}
|
||||
|
||||
// exportOnlyFlags lists archive-mode flags. They are intentionally absent from
|
||||
// the import flag set, but a dedicated message is friendlier than the flag
|
||||
// package's generic "flag provided but not defined".
|
||||
var exportOnlyFlags = []string{
|
||||
"source", "table", "where", "purge", "bulk-delete", "insert-ignore",
|
||||
"replace", "chunk-size", "limit", "shard-by", "shard-rows", "shard-column",
|
||||
"shard-column-type", "shard-time-zone", "zero-date", "format", "compress",
|
||||
"compress-level", "csv-null", "max-lag", "check-interval",
|
||||
"check-replica-lag", "retries", "safe-auto-increment", "progress-interval",
|
||||
"allow-full-scan", "force-resume", "index",
|
||||
}
|
||||
|
||||
var charsetRE = regexp.MustCompile(`^[A-Za-z0-9_]+$`)
|
||||
|
||||
type stringList []string
|
||||
|
||||
func (s *stringList) String() string { return strings.Join(*s, ",") }
|
||||
func (s *stringList) Set(v string) error {
|
||||
*s = append(*s, v)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ParseImport parses args (without the "import" subcommand) into an
|
||||
// ImportConfig. Validation is separate.
|
||||
func ParseImport(args []string, errOut io.Writer) (*ImportConfig, error) {
|
||||
if err := rejectExportFlags(args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := ImportDefaults()
|
||||
var (
|
||||
columnsArg string
|
||||
selects stringList
|
||||
)
|
||||
fs := flag.NewFlagSet("archer import", flag.ContinueOnError)
|
||||
fs.SetOutput(errOut)
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintln(errOut, "usage: archer import --file <archive-dir> --dest <dsn> [flags]")
|
||||
fmt.Fprintln(errOut, "")
|
||||
fmt.Fprintln(errOut, "Replay a dump archive into an existing MySQL table.")
|
||||
fmt.Fprintln(errOut, "")
|
||||
fmt.Fprintln(errOut, "flags:")
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
|
||||
fs.StringVar(&c.FileDir, "file", "", "archive directory containing manifest.jsonl (required)")
|
||||
fs.StringVar(&c.DestDSN, "dest", "", "target MySQL DSN (required, must include a database)")
|
||||
fs.StringVar(&c.DestTable, "dest-table", "", "target table (defaults to the archived source table)")
|
||||
fs.StringVar(&columnsArg, "columns", "", "comma separated target columns in file field order")
|
||||
fs.BoolVar(&c.AssumeAll, "assume-all-columns", false, "accept positional mapping to all target columns, in ordinal order")
|
||||
fs.Var(&selects, "select", "shard selector (repeatable): all, seq:A-B, pk:LO-HI, date:D1..D2")
|
||||
fs.StringVar(&c.Mode, "mode", c.Mode, "insert mode: ignore|insert|replace")
|
||||
fs.StringVar(&c.Engine, "engine", c.Engine, "load engine: load|insert")
|
||||
fs.StringVar(&c.Verify, "verify", c.Verify, "integrity check: pre|off")
|
||||
fs.IntVar(&c.Workers, "workers", c.Workers, "parallel shard workers (must be greater than 0)")
|
||||
fs.StringVar(&c.Charset, "charset", c.Charset, "character set used for loading")
|
||||
fs.StringVar(&c.TimeZone, "time-zone", "", "target session time zone (empty: server default)")
|
||||
fs.BoolVar(&c.AllowPartial, "allow-partial", false, "import sealed shards from an archive whose state.json is incomplete")
|
||||
fs.BoolVar(&c.AllowWarnings, "allow-warnings", false, "do not treat server warnings as failures")
|
||||
fs.BoolVar(&c.DryRun, "dry-run", false, "print selected shards and configuration without connecting or writing")
|
||||
fs.Var(&c.Progress, "progress", "progress output: auto|on|off|N")
|
||||
fs.BoolVar(&c.Statistics, "statistics", false, "print a summary when finished")
|
||||
fs.BoolVar(&c.Version, "version", false, "print version and exit")
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if fs.NArg() > 0 {
|
||||
return nil, fmt.Errorf("unexpected argument %q", fs.Arg(0))
|
||||
}
|
||||
if c.Version {
|
||||
return c, nil
|
||||
}
|
||||
if columnsArg != "" {
|
||||
for _, part := range strings.Split(columnsArg, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
c.Columns = append(c.Columns, part)
|
||||
}
|
||||
}
|
||||
}
|
||||
c.Selects = append([]string(nil), selects...)
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func rejectExportFlags(args []string) error {
|
||||
exportOnly := make(map[string]bool, len(exportOnlyFlags))
|
||||
for _, name := range exportOnlyFlags {
|
||||
exportOnly[name] = true
|
||||
}
|
||||
for _, a := range args {
|
||||
if a == "--" {
|
||||
break
|
||||
}
|
||||
if !strings.HasPrefix(a, "--") {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimPrefix(a, "--")
|
||||
if i := strings.IndexByte(name, '='); i >= 0 {
|
||||
name = name[:i]
|
||||
}
|
||||
if exportOnly[name] {
|
||||
return cfgErr("--%s is only valid in archive (export) mode", name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate checks required flags and the import-only dependency rules. Archive
|
||||
// contents (completion, recorded columns, selectors against entries) are
|
||||
// checked by the importer before any database connection is opened.
|
||||
func (c *ImportConfig) Validate() error {
|
||||
if c.FileDir == "" {
|
||||
return cfgErr("--file is required")
|
||||
}
|
||||
if c.DestDSN == "" {
|
||||
return cfgErr("--dest is required")
|
||||
}
|
||||
dest, err := normalize(c.DestDSN)
|
||||
if err != nil {
|
||||
return cfgErr("--dest: %v", err)
|
||||
}
|
||||
if dest.DBName == "" {
|
||||
return cfgErr("--dest must include a database name")
|
||||
}
|
||||
switch c.Mode {
|
||||
case "ignore", "insert", "replace":
|
||||
default:
|
||||
return cfgErr("--mode must be one of ignore, insert, replace")
|
||||
}
|
||||
switch c.Engine {
|
||||
case "load", "insert":
|
||||
default:
|
||||
return cfgErr("--engine must be one of load, insert")
|
||||
}
|
||||
switch c.Verify {
|
||||
case "pre", "off":
|
||||
default:
|
||||
return cfgErr("--verify must be pre or off")
|
||||
}
|
||||
if c.Workers <= 0 {
|
||||
return cfgErr("--workers must be greater than 0")
|
||||
}
|
||||
if !charsetRE.MatchString(c.Charset) {
|
||||
return cfgErr("--charset %q is not a valid character set name", c.Charset)
|
||||
}
|
||||
if len(c.Columns) > 0 && c.AssumeAll {
|
||||
return cfgErr("--columns and --assume-all-columns are mutually exclusive")
|
||||
}
|
||||
if err := c.Progress.resolve(); err != nil {
|
||||
return cfgErr("%v", err)
|
||||
}
|
||||
sels, err := shard.ParseSelection(c.Selects)
|
||||
if err != nil {
|
||||
return cfgErr("%v", err)
|
||||
}
|
||||
for _, s := range sels {
|
||||
if s.Kind == "all" && len(sels) > 1 {
|
||||
return cfgErr("--select all cannot be combined with other selectors")
|
||||
}
|
||||
}
|
||||
c.Selectors = sels
|
||||
return nil
|
||||
}
|
||||
|
||||
// DestConfig parses the target DSN, forcing parseTime off like the export path.
|
||||
func (c *ImportConfig) DestConfig() (*mysql.Config, error) {
|
||||
return normalize(c.DestDSN)
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func importArgs(extra ...string) []string {
|
||||
args := []string{"--file", "archive", "--dest", "u:p@tcp(127.0.0.1:3306)/db"}
|
||||
return append(args, extra...)
|
||||
}
|
||||
|
||||
func parseImport(t *testing.T, args ...string) *ImportConfig {
|
||||
t.Helper()
|
||||
c, err := ParseImport(args, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatalf("parse %v: %v", args, err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func TestImportDefaults(t *testing.T) {
|
||||
c := parseImport(t, importArgs()...)
|
||||
if err := c.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Mode != "ignore" || c.Engine != "load" || c.Verify != "pre" || c.Workers != 1 {
|
||||
t.Fatalf("bad defaults: %+v", c)
|
||||
}
|
||||
if c.Charset != "utf8mb4" || c.Progress.Mode != ProgressAuto {
|
||||
t.Fatalf("bad defaults: %+v", c)
|
||||
}
|
||||
if len(c.Selectors) != 1 || c.Selectors[0].Kind != "all" {
|
||||
t.Fatalf("default selector = %+v", c.Selectors)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportRequired(t *testing.T) {
|
||||
c := parseImport(t, "--dest", "u:p@tcp(h:3306)/db")
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--file") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
c = parseImport(t, "--file", "archive")
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--dest") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
c = parseImport(t, "--file", "archive", "--dest", "u:p@tcp(h:3306)/")
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "database name") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportRejectsExportFlags(t *testing.T) {
|
||||
for _, arg := range []string{"--source", "--where", "--purge", "--table", "--format", "--replace"} {
|
||||
_, err := ParseImport(append(importArgs(), arg, "x"), &bytes.Buffer{})
|
||||
if err == nil {
|
||||
t.Fatalf("%s: expected an error", arg)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "archive (export) mode") {
|
||||
t.Fatalf("%s: got %v", arg, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportRejectsUnknownFlag(t *testing.T) {
|
||||
if _, err := ParseImport(append(importArgs(), "--bogus", "1"), &bytes.Buffer{}); err == nil {
|
||||
t.Fatal("expected unknown flag error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportInvalidValues(t *testing.T) {
|
||||
cases := []struct {
|
||||
extra []string
|
||||
want string
|
||||
}{
|
||||
{[]string{"--mode", "merge"}, "--mode"},
|
||||
{[]string{"--engine", "bulk"}, "--engine"},
|
||||
{[]string{"--verify", "after"}, "--verify"},
|
||||
{[]string{"--workers", "0"}, "--workers"},
|
||||
{[]string{"--charset", "utf8; drop"}, "--charset"},
|
||||
{[]string{"--select", "seq:5-2"}, "--select"},
|
||||
{[]string{"--select", "date:2025-01-31..2025-01-01"}, "--select"},
|
||||
{[]string{"--select", "bogus:1"}, "--select"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
c := parseImport(t, importArgs(tc.extra...)...)
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("%v: got %v", tc.extra, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportMappingConflict(t *testing.T) {
|
||||
c := parseImport(t, importArgs("--columns", "id,title", "--assume-all-columns")...)
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "mutually exclusive") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportSelectAllCannotCombine(t *testing.T) {
|
||||
c := parseImport(t, importArgs("--select", "all", "--select", "seq:1-2")...)
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "cannot be combined") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportRepeatedSelect(t *testing.T) {
|
||||
c := parseImport(t, importArgs("--select", "seq:1-2", "--select", "pk:10-20", "--select", "date:2025-01-01..2025-01-02")...)
|
||||
if err := c.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(c.Selectors) != 3 || c.Selectors[0].Kind != "seq" || c.Selectors[1].Kind != "pk" || c.Selectors[2].Kind != "date" {
|
||||
t.Fatalf("selectors = %+v", c.Selectors)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportUnexpectedPositional(t *testing.T) {
|
||||
if _, err := ParseImport(append(importArgs(), "stray"), &bytes.Buffer{}); err == nil {
|
||||
t.Fatal("expected an error for a positional argument")
|
||||
}
|
||||
}
|
||||
@@ -67,9 +67,6 @@ func (c *Config) Validate() error {
|
||||
if c.BulkDelete && !c.Purge {
|
||||
return cfgErr("--bulk-delete requires --purge")
|
||||
}
|
||||
if c.BulkDelete && c.ShardBy == "date" {
|
||||
return cfgErr("--bulk-delete cannot be used with --shard-by date; delete by IN list instead")
|
||||
}
|
||||
|
||||
switch c.Format {
|
||||
case "dump", "csv", "jsonl":
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
// Package importer replays a dump archive into an existing MySQL table.
|
||||
//
|
||||
// The archive's manifest.jsonl is the only authority: files on disk that are
|
||||
// not registered are never imported, and unparseable manifest lines abort the
|
||||
// run before a database connection is opened. Each shard is a complete,
|
||||
// independently loadable file, so shards are the unit of selection, atomicity
|
||||
// and failure reporting.
|
||||
package importer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/checkpoint"
|
||||
"archer/internal/shard"
|
||||
)
|
||||
|
||||
// Archive is a loaded, validated-by-reading view of an export directory.
|
||||
type Archive struct {
|
||||
Root string
|
||||
Entries []shard.Entry
|
||||
State *checkpoint.State
|
||||
Complete bool
|
||||
SourceTable string
|
||||
ArchivedColumns []string
|
||||
Format string
|
||||
Compress string
|
||||
Compressor *shard.Compressor
|
||||
}
|
||||
|
||||
// LoadArchive reads manifest.jsonl strictly and state.json when present. It
|
||||
// never writes to the archive directory.
|
||||
func LoadArchive(root string) (*Archive, error) {
|
||||
st, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return nil, archererr.Newf(archererr.KindConfig, "--file %s: %v", root, err)
|
||||
}
|
||||
if !st.IsDir() {
|
||||
return nil, archererr.Newf(archererr.KindConfig, "--file %s is not a directory", root)
|
||||
}
|
||||
entries, err := shard.LoadManifest(root)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
state, err := checkpoint.Load(root)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
a := &Archive{Root: root, Entries: entries, State: state}
|
||||
if state != nil {
|
||||
a.Complete = state.Complete
|
||||
a.SourceTable = strings.TrimSpace(state.Config["table"])
|
||||
if cols := strings.TrimSpace(state.Config["columns"]); cols != "" {
|
||||
for _, part := range strings.Split(cols, ",") {
|
||||
if part = strings.TrimSpace(part); part != "" {
|
||||
a.ArchivedColumns = append(a.ArchivedColumns, part)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// CheckComplete enforces the completion gate. A manifest only proves which
|
||||
// shards are sealed, not that the export covered the whole table, so an
|
||||
// incomplete archive is refused unless the operator opts in explicitly.
|
||||
func (a *Archive) CheckComplete(allowPartial bool) error {
|
||||
if allowPartial {
|
||||
return nil
|
||||
}
|
||||
if a.State == nil {
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"archive %s has no state.json; cannot confirm the export finished (pass --allow-partial to import its sealed shards)", a.Root)
|
||||
}
|
||||
if !a.State.Complete {
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"archive %s is incomplete; it may contain only part of the table (pass --allow-partial to import its sealed shards)", a.Root)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CheckMappingChoice rejects an explicit mapping when the archive already
|
||||
// recorded its own column list: two sources of truth would silently disagree.
|
||||
func (a *Archive) CheckMappingChoice(columns []string, assumeAll bool) error {
|
||||
if len(a.ArchivedColumns) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(columns) > 0 {
|
||||
return archererr.Newf(archererr.KindConfig,
|
||||
"archive records its own column list (%s); --columns cannot be combined with it", strings.Join(a.ArchivedColumns, ", "))
|
||||
}
|
||||
if assumeAll {
|
||||
return archererr.Newf(archererr.KindConfig,
|
||||
"archive records its own column list (%s); --assume-all-columns cannot be combined with it", strings.Join(a.ArchivedColumns, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var shardFileRE = regexp.MustCompile(`^\d+\..*\.(dump|csv|jsonl)(\.gz|\.zst|\.xz)?$`)
|
||||
|
||||
// UnregisteredFiles counts shard-looking files that the manifest does not
|
||||
// list. They are ignored by design; the count is surfaced in --statistics so
|
||||
// they are not mistaken for imported data.
|
||||
func UnregisteredFiles(root string, entries []shard.Entry) int {
|
||||
registered := make(map[string]bool, len(entries))
|
||||
for _, e := range entries {
|
||||
registered[filepath.ToSlash(e.Path)] = true
|
||||
}
|
||||
n := 0
|
||||
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
if name == "manifest.jsonl" || name == "state.json" || strings.HasSuffix(name, ".partial") ||
|
||||
strings.HasSuffix(name, ".partial.meta") || !shardFileRE.MatchString(name) {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if !registered[filepath.ToSlash(rel)] {
|
||||
n++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return n
|
||||
}
|
||||
|
||||
// ShardFormat derives the output format and compression from a manifest path
|
||||
// such as 000003.100-200.dump.gz.
|
||||
func ShardFormat(p string) (format, compress string, err error) {
|
||||
name := filepath.Base(p)
|
||||
switch {
|
||||
case strings.HasSuffix(name, ".gz"):
|
||||
compress, name = "gzip", strings.TrimSuffix(name, ".gz")
|
||||
case strings.HasSuffix(name, ".zst"):
|
||||
compress, name = "zstd", strings.TrimSuffix(name, ".zst")
|
||||
case strings.HasSuffix(name, ".xz"):
|
||||
compress, name = "xz", strings.TrimSuffix(name, ".xz")
|
||||
default:
|
||||
compress = "none"
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(name))
|
||||
if ext == "" {
|
||||
return "", "", fmt.Errorf("shard %s has no format extension", p)
|
||||
}
|
||||
format = strings.TrimPrefix(ext, ".")
|
||||
switch format {
|
||||
case "dump", "csv", "jsonl":
|
||||
return format, compress, nil
|
||||
default:
|
||||
return "", "", fmt.Errorf("shard %s has unknown format %q", p, format)
|
||||
}
|
||||
}
|
||||
|
||||
// ResolveFormat checks that every selected shard agrees on format and
|
||||
// compression, and that the format is dump (the only one v1 can replay).
|
||||
func ResolveFormat(entries []shard.Entry) (format, compress string, comp *shard.Compressor, err error) {
|
||||
for i, e := range entries {
|
||||
f, c, ferr := ShardFormat(e.Path)
|
||||
if ferr != nil {
|
||||
return "", "", nil, archererr.Wrap(archererr.KindValidation, ferr)
|
||||
}
|
||||
if i == 0 {
|
||||
format, compress = f, c
|
||||
continue
|
||||
}
|
||||
if f != format {
|
||||
return "", "", nil, archererr.Newf(archererr.KindValidation,
|
||||
"selected shards mix formats: %s is %s but %s is %s", entries[0].Path, format, e.Path, f)
|
||||
}
|
||||
if c != compress {
|
||||
return "", "", nil, archererr.Newf(archererr.KindValidation,
|
||||
"selected shards mix compression: %s is %s but %s is %s", entries[0].Path, compress, e.Path, c)
|
||||
}
|
||||
}
|
||||
if format != "dump" {
|
||||
return "", "", nil, archererr.Newf(archererr.KindValidation,
|
||||
"import supports only dump archives; the selected shards are %s", format)
|
||||
}
|
||||
comp, err = shard.NewCompressor(compress, shard.DefaultLevel)
|
||||
if err != nil {
|
||||
return "", "", nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
return format, compress, comp, nil
|
||||
}
|
||||
|
||||
// CheckShardFiles verifies every selected manifest entry has a readable file
|
||||
// before any connection is opened.
|
||||
func CheckShardFiles(root string, entries []shard.Entry) error {
|
||||
for _, e := range entries {
|
||||
path := filepath.Join(root, filepath.FromSlash(e.Path))
|
||||
st, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"manifest entry seq %d points to missing shard %s: %v", e.Seq, e.Path, err)
|
||||
}
|
||||
if st.IsDir() {
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"manifest entry seq %d points to a directory, not a shard: %s", e.Seq, e.Path)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func shardPath(root string, e shard.Entry) string {
|
||||
return filepath.Join(root, filepath.FromSlash(e.Path))
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"archer/internal/checkpoint"
|
||||
"archer/internal/shard"
|
||||
)
|
||||
|
||||
func writeManifest(t *testing.T, root string, entries []shard.Entry) {
|
||||
t.Helper()
|
||||
var b strings.Builder
|
||||
for _, e := range entries {
|
||||
j, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.Write(j)
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "manifest.jsonl"), []byte(b.String()), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeShard(t *testing.T, root, rel, body string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(root, filepath.FromSlash(rel))
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func saveState(t *testing.T, root string, complete bool, config map[string]string) {
|
||||
t.Helper()
|
||||
if err := checkpoint.Save(root, &checkpoint.State{Complete: complete, Config: config}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadArchiveReadsStateAndRecords(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
entries := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump", PKLo: "1", PKHi: "1", Rows: 1, Bytes: 3, SHA256: "aa"}}
|
||||
writeManifest(t, root, entries)
|
||||
writeShard(t, root, entries[0].Path, "1\tx\n")
|
||||
saveState(t, root, true, map[string]string{"table": "orders", "columns": "id,title"})
|
||||
|
||||
a, err := LoadArchive(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !a.Complete || a.SourceTable != "orders" {
|
||||
t.Fatalf("archive = %+v", a)
|
||||
}
|
||||
if strings.Join(a.ArchivedColumns, ",") != "id,title" {
|
||||
t.Fatalf("columns = %v", a.ArchivedColumns)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadArchiveRejectsCorruptManifest(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "manifest.jsonl"), []byte("{not json\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := LoadArchive(root)
|
||||
if err == nil || !strings.Contains(err.Error(), "line 1") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadArchiveMissingManifest(t *testing.T) {
|
||||
_, err := LoadArchive(t.TempDir())
|
||||
if err == nil || !strings.Contains(err.Error(), "manifest") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadArchiveNotADirectory(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "file")
|
||||
if err := os.WriteFile(path, []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := LoadArchive(path); err == nil {
|
||||
t.Fatal("expected an error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompletionGate(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeManifest(t, root, []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}})
|
||||
|
||||
a, err := LoadArchive(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := a.CheckComplete(false); err == nil || !strings.Contains(err.Error(), "state.json") {
|
||||
t.Fatalf("missing state: %v", err)
|
||||
}
|
||||
if err := a.CheckComplete(true); err != nil {
|
||||
t.Fatalf("--allow-partial: %v", err)
|
||||
}
|
||||
|
||||
saveState(t, root, false, nil)
|
||||
a, _ = LoadArchive(root)
|
||||
if err := a.CheckComplete(false); err == nil || !strings.Contains(err.Error(), "incomplete") {
|
||||
t.Fatalf("incomplete: %v", err)
|
||||
}
|
||||
if err := a.CheckComplete(true); err != nil {
|
||||
t.Fatalf("--allow-partial: %v", err)
|
||||
}
|
||||
|
||||
saveState(t, root, true, nil)
|
||||
a, _ = LoadArchive(root)
|
||||
if err := a.CheckComplete(false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingChoiceConflict(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeManifest(t, root, []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}})
|
||||
saveState(t, root, true, map[string]string{"columns": "id,title"})
|
||||
a, _ := LoadArchive(root)
|
||||
if err := a.CheckMappingChoice([]string{"id"}, false); err == nil || !strings.Contains(err.Error(), "--columns") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
if err := a.CheckMappingChoice(nil, true); err == nil || !strings.Contains(err.Error(), "--assume-all-columns") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
if err := a.CheckMappingChoice(nil, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckShardFiles(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeShard(t, root, "a/000001.1-1.dump", "1\n")
|
||||
ok := []shard.Entry{{Seq: 1, Path: "a/000001.1-1.dump"}}
|
||||
if err := CheckShardFiles(root, ok); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
missing := []shard.Entry{{Seq: 7, Path: "a/000007.1-1.dump"}}
|
||||
if err := CheckShardFiles(root, missing); err == nil || !strings.Contains(err.Error(), "seq 7") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveFormat(t *testing.T) {
|
||||
dump := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}, {Seq: 2, Path: "000002.2-2.dump.gz"}}
|
||||
if _, _, _, err := ResolveFormat(dump); err == nil || !strings.Contains(err.Error(), "mix compression") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
|
||||
gz := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump.gz"}, {Seq: 2, Path: "000002.2-2.dump.gz"}}
|
||||
format, compress, comp, err := ResolveFormat(gz)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if format != "dump" || compress != "gzip" || comp.Name != "gzip" {
|
||||
t.Fatalf("format=%s compress=%s comp=%+v", format, compress, comp)
|
||||
}
|
||||
|
||||
for _, bad := range []string{"000001.1-1.csv", "000001.1-1.jsonl", "000001.1-1.csv.gz"} {
|
||||
_, _, _, err := ResolveFormat([]shard.Entry{{Seq: 1, Path: bad}})
|
||||
if err == nil || !strings.Contains(err.Error(), "only dump") {
|
||||
t.Fatalf("%s: got %v", bad, err)
|
||||
}
|
||||
}
|
||||
|
||||
mixed := []shard.Entry{{Seq: 1, Path: "a/000001.1-1.dump"}, {Seq: 2, Path: "b/000002.2-2.csv"}}
|
||||
if _, _, _, err := ResolveFormat(mixed); err == nil || !strings.Contains(err.Error(), "mix formats") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShardFormat(t *testing.T) {
|
||||
for _, tc := range []struct{ path, format, compress string }{
|
||||
{"000001.1-2.dump", "dump", "none"},
|
||||
{"000001.1-2.dump.gz", "dump", "gzip"},
|
||||
{"000001.1-2.dump.zst", "dump", "zstd"},
|
||||
{"000001.1-2.dump.xz", "dump", "xz"},
|
||||
{"date=2025-01-01/000001.1-2.jsonl", "jsonl", "none"},
|
||||
} {
|
||||
f, c, err := ShardFormat(tc.path)
|
||||
if err != nil || f != tc.format || c != tc.compress {
|
||||
t.Fatalf("%s: f=%s c=%s err=%v", tc.path, f, c, err)
|
||||
}
|
||||
}
|
||||
if _, _, err := ShardFormat("000001.1-2"); err == nil {
|
||||
t.Fatal("expected an error for a missing extension")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnregisteredFiles(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
entries := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump"}}
|
||||
writeManifest(t, root, entries)
|
||||
writeShard(t, root, "000001.1-1.dump", "1\n")
|
||||
writeShard(t, root, "000002.2-2.dump", "2\n")
|
||||
writeShard(t, root, "notes.txt", "ignored")
|
||||
if got := UnregisteredFiles(root, entries); got != 1 {
|
||||
t.Fatalf("got %d", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/schema"
|
||||
"archer/internal/shard"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
// Mapping translates file fields to target columns. It is the single place
|
||||
// that decides whether a field is loaded into a named column or discarded into
|
||||
// a user variable (generated columns under positional mapping).
|
||||
type Mapping struct {
|
||||
// FileColumns has one entry per file field: the canonical target column
|
||||
// name, or "" when the field is intentionally discarded.
|
||||
FileColumns []string
|
||||
// InsertColumns and InsertFieldIdx describe the parameterized INSERT used
|
||||
// by the insert engine. InsertFieldIdx points into FileColumns.
|
||||
InsertColumns []string
|
||||
InsertFieldIdx []int
|
||||
// LoadList is the LOAD DATA column list: quoted identifiers and
|
||||
// @_archer_skip_N variables.
|
||||
LoadList []string
|
||||
// Positional is true for --assume-all-columns; callers must warn.
|
||||
Positional bool
|
||||
}
|
||||
|
||||
// BuildMapping resolves the three permitted mapping sources. Columns recorded
|
||||
// in the archive win over an explicit --columns by construction (the conflict
|
||||
// is rejected earlier); assumeAll is only consulted when neither list exists.
|
||||
func BuildMapping(table *schema.Table, archived, explicit []string, assumeAll bool) (*Mapping, error) {
|
||||
return buildMapping(table.Name, table.Columns, archived, explicit, assumeAll)
|
||||
}
|
||||
|
||||
func buildMapping(tableName string, columns []schema.Column, archived, explicit []string, assumeAll bool) (*Mapping, error) {
|
||||
names := archived
|
||||
if len(names) == 0 {
|
||||
names = explicit
|
||||
}
|
||||
switch {
|
||||
case len(names) > 0:
|
||||
return mappingByName(tableName, columns, names)
|
||||
case assumeAll:
|
||||
return mappingPositional(tableName, columns)
|
||||
default:
|
||||
return nil, archererr.Newf(archererr.KindConfig,
|
||||
"the archive did not record a column list; provide --columns in file field order or accept positional mapping with --assume-all-columns")
|
||||
}
|
||||
}
|
||||
|
||||
func mappingByName(tableName string, columns []schema.Column, names []string) (*Mapping, error) {
|
||||
byName := make(map[string]schema.Column, len(columns))
|
||||
for _, c := range columns {
|
||||
byName[strings.ToLower(c.Name)] = c
|
||||
}
|
||||
m := &Mapping{}
|
||||
seen := map[string]bool{}
|
||||
for _, requested := range names {
|
||||
col, ok := byName[strings.ToLower(requested)]
|
||||
if !ok {
|
||||
return nil, archererr.Newf(archererr.KindValidation,
|
||||
"column %q from the file field mapping does not exist in target table %s", requested, tableName)
|
||||
}
|
||||
if isGenerated(col) {
|
||||
return nil, archererr.Newf(archererr.KindValidation,
|
||||
"column %q is a generated column and cannot be written from an archive", col.Name)
|
||||
}
|
||||
if seen[col.Name] {
|
||||
return nil, archererr.Newf(archererr.KindValidation,
|
||||
"column %q appears more than once in the file field mapping", col.Name)
|
||||
}
|
||||
seen[col.Name] = true
|
||||
idx := len(m.FileColumns)
|
||||
m.FileColumns = append(m.FileColumns, col.Name)
|
||||
m.InsertColumns = append(m.InsertColumns, col.Name)
|
||||
m.InsertFieldIdx = append(m.InsertFieldIdx, idx)
|
||||
m.LoadList = append(m.LoadList, sqlutil.Ident(col.Name))
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func mappingPositional(tableName string, columns []schema.Column) (*Mapping, error) {
|
||||
m := &Mapping{Positional: true}
|
||||
for i, col := range columns {
|
||||
if isGenerated(col) {
|
||||
// The file still contains this field, so keep a placeholder and
|
||||
// discard the value; the target table recomputes it.
|
||||
m.FileColumns = append(m.FileColumns, "")
|
||||
m.LoadList = append(m.LoadList, fmt.Sprintf("@_archer_skip_%d", col.Ordinal))
|
||||
continue
|
||||
}
|
||||
m.FileColumns = append(m.FileColumns, col.Name)
|
||||
m.InsertColumns = append(m.InsertColumns, col.Name)
|
||||
m.InsertFieldIdx = append(m.InsertFieldIdx, i)
|
||||
m.LoadList = append(m.LoadList, sqlutil.Ident(col.Name))
|
||||
}
|
||||
if len(m.InsertColumns) == 0 {
|
||||
return nil, archererr.Newf(archererr.KindValidation,
|
||||
"target table %s has no writable columns", tableName)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func isGenerated(c schema.Column) bool {
|
||||
return strings.Contains(strings.ToLower(c.Extra), "generated")
|
||||
}
|
||||
|
||||
// CountFields counts dump fields in one raw line. Literal tabs are escaped by
|
||||
// the dump encoder, so every 0x09 byte is a separator. An empty line is a
|
||||
// single empty, non-NULL field.
|
||||
func CountFields(line []byte) int {
|
||||
return strings.Count(string(line), "\t") + 1
|
||||
}
|
||||
|
||||
// FirstLineFieldCount reads the first decompressed line of a shard.
|
||||
func FirstLineFieldCount(path string, comp *shard.Compressor) (fields int, hasRow bool, err error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
defer f.Close()
|
||||
r, err := comp.Reader(f)
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
defer r.Close()
|
||||
br := bufio.NewReaderSize(r, 64*1024)
|
||||
line, err := br.ReadBytes('\n')
|
||||
if err == io.EOF {
|
||||
if len(line) == 0 {
|
||||
return 0, false, nil
|
||||
}
|
||||
} else if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
line = bytes.TrimSuffix(line, []byte{'\n'})
|
||||
return CountFields(line), true, nil
|
||||
}
|
||||
|
||||
// CheckFieldCount compares the first row's field count with the load list
|
||||
// length. It is a sanity check only: it cannot prove that same-width columns
|
||||
// are in the right order.
|
||||
func CheckFieldCount(path string, comp *shard.Compressor, want int) error {
|
||||
got, hasRow, err := FirstLineFieldCount(path, comp)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, fmt.Errorf("read first row of %s: %w", path, err))
|
||||
}
|
||||
if !hasRow {
|
||||
return nil
|
||||
}
|
||||
if got != want {
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"first row of %s has %d fields, but the load column list has %d; refusing to load", path, got, want)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"archer/internal/schema"
|
||||
"archer/internal/shard"
|
||||
)
|
||||
|
||||
func targetColumns() []schema.Column {
|
||||
return []schema.Column{
|
||||
{Name: "id", Ordinal: 1},
|
||||
{Name: "title", Ordinal: 2},
|
||||
{Name: "full_name", Ordinal: 3, Extra: "STORED GENERATED"},
|
||||
{Name: "created_at", Ordinal: 4},
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingByNamePreservesOrder(t *testing.T) {
|
||||
m, err := buildMapping("orders", targetColumns(), []string{"title", "ID"}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(m.FileColumns, ",") != "title,id" {
|
||||
t.Fatalf("file columns = %v", m.FileColumns)
|
||||
}
|
||||
if strings.Join(m.InsertColumns, ",") != "title,id" {
|
||||
t.Fatalf("insert columns = %v", m.InsertColumns)
|
||||
}
|
||||
if strings.Join(m.LoadList, ",") != "`title`,`id`" {
|
||||
t.Fatalf("load list = %v", m.LoadList)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingByNameMissingColumn(t *testing.T) {
|
||||
_, err := buildMapping("orders", targetColumns(), []string{"id", "missing"}, nil, false)
|
||||
if err == nil || !strings.Contains(err.Error(), "missing") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingByNameRejectsGenerated(t *testing.T) {
|
||||
_, err := buildMapping("orders", targetColumns(), []string{"id", "full_name"}, nil, false)
|
||||
if err == nil || !strings.Contains(err.Error(), "generated") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingByNameRejectsDuplicate(t *testing.T) {
|
||||
_, err := buildMapping("orders", targetColumns(), []string{"id", "Id"}, nil, false)
|
||||
if err == nil || !strings.Contains(err.Error(), "more than once") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingPositionalDropsGenerated(t *testing.T) {
|
||||
m, err := buildMapping("orders", targetColumns(), nil, nil, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !m.Positional {
|
||||
t.Fatal("expected positional mapping")
|
||||
}
|
||||
if len(m.FileColumns) != 4 {
|
||||
t.Fatalf("file fields = %v", m.FileColumns)
|
||||
}
|
||||
if m.FileColumns[2] != "" {
|
||||
t.Fatalf("generated field should be empty, got %q", m.FileColumns[2])
|
||||
}
|
||||
if strings.Join(m.InsertColumns, ",") != "id,title,created_at" {
|
||||
t.Fatalf("insert columns = %v", m.InsertColumns)
|
||||
}
|
||||
if len(m.InsertFieldIdx) != 3 || m.InsertFieldIdx[2] != 3 {
|
||||
t.Fatalf("insert indices = %v", m.InsertFieldIdx)
|
||||
}
|
||||
if m.LoadList[2] != "@_archer_skip_3" {
|
||||
t.Fatalf("skip variable = %q", m.LoadList[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingRequiresSource(t *testing.T) {
|
||||
_, err := buildMapping("orders", targetColumns(), nil, nil, false)
|
||||
if err == nil || !strings.Contains(err.Error(), "--assume-all-columns") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingExplicitColumns(t *testing.T) {
|
||||
m, err := buildMapping("orders", targetColumns(), nil, []string{"id", "title"}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(m.FileColumns) != 2 || m.FileColumns[1] != "title" {
|
||||
t.Fatalf("file columns = %v", m.FileColumns)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountFields(t *testing.T) {
|
||||
cases := []struct {
|
||||
line string
|
||||
want int
|
||||
}{
|
||||
{"", 1},
|
||||
{"a", 1},
|
||||
{"a\tb", 2},
|
||||
{"a\tb\tc\t", 4},
|
||||
{`a\tb`, 1}, // escaped tab is not a separator
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := CountFields([]byte(tc.line)); got != tc.want {
|
||||
t.Fatalf("%q: got %d want %d", tc.line, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirstLineFieldCount(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, "x.dump")
|
||||
if err := os.WriteFile(path, []byte("a\tb\tc\n1\t2\t3\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
comp, _ := shard.NewCompressor("none", shard.DefaultLevel)
|
||||
n, ok, err := FirstLineFieldCount(path, comp)
|
||||
if err != nil || !ok || n != 3 {
|
||||
t.Fatalf("n=%d ok=%t err=%v", n, ok, err)
|
||||
}
|
||||
if err := CheckFieldCount(path, comp, 3); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := CheckFieldCount(path, comp, 4); err == nil || !strings.Contains(err.Error(), "3 fields") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirstLineFieldCountEmptyFile(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "empty.dump")
|
||||
if err := os.WriteFile(path, nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
comp, _ := shard.NewCompressor("none", shard.DefaultLevel)
|
||||
_, ok, err := FirstLineFieldCount(path, comp)
|
||||
if err != nil || ok {
|
||||
t.Fatalf("ok=%t err=%v", ok, err)
|
||||
}
|
||||
if err := CheckFieldCount(path, comp, 9); err != nil {
|
||||
t.Fatalf("empty shard must not fail the field check: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"io"
|
||||
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
// DumpReader decodes the dump format produced by internal/row.dumpEncoder:
|
||||
// TAB-separated fields, LF-terminated rows, \N for NULL and the MySQL escape
|
||||
// table. It exists for the insert fallback engine; the load engine hands the
|
||||
// bytes to the server unchanged.
|
||||
type DumpReader struct {
|
||||
br *bufio.Reader
|
||||
}
|
||||
|
||||
func NewDumpReader(r io.Reader) *DumpReader {
|
||||
return &DumpReader{br: bufio.NewReaderSize(r, 256*1024)}
|
||||
}
|
||||
|
||||
// ReadRow returns the next row. It returns io.EOF exactly when the stream ends.
|
||||
func (d *DumpReader) ReadRow() (row.Row, error) {
|
||||
line, err := d.br.ReadBytes('\n')
|
||||
if len(line) == 0 && err == io.EOF {
|
||||
return row.Row{}, io.EOF
|
||||
}
|
||||
if err != nil && err != io.EOF {
|
||||
return row.Row{}, err
|
||||
}
|
||||
line = bytes.TrimSuffix(line, []byte{'\n'})
|
||||
fields := bytes.Split(line, []byte{'\t'})
|
||||
out := row.Row{Values: make([]row.Value, len(fields))}
|
||||
for i, f := range fields {
|
||||
out.Values[i] = unescapeField(f)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// unescapeField mirrors MySQL's LOAD DATA escape handling. A field is NULL only
|
||||
// when it is exactly \N; unknown escapes yield the escaped character, matching
|
||||
// the server's default FIELDS ESCAPED BY '\\' behavior.
|
||||
func unescapeField(f []byte) row.Value {
|
||||
if len(f) == 2 && f[0] == '\\' && f[1] == 'N' {
|
||||
return row.Value{}
|
||||
}
|
||||
if bytes.IndexByte(f, '\\') < 0 {
|
||||
return row.Value{Raw: append([]byte{}, f...)}
|
||||
}
|
||||
out := make([]byte, 0, len(f))
|
||||
for i := 0; i < len(f); i++ {
|
||||
if f[i] != '\\' {
|
||||
out = append(out, f[i])
|
||||
continue
|
||||
}
|
||||
if i+1 >= len(f) {
|
||||
out = append(out, '\\')
|
||||
break
|
||||
}
|
||||
i++
|
||||
switch f[i] {
|
||||
case '\\':
|
||||
out = append(out, '\\')
|
||||
case 't':
|
||||
out = append(out, '\t')
|
||||
case 'n':
|
||||
out = append(out, '\n')
|
||||
case 'r':
|
||||
out = append(out, '\r')
|
||||
case '0':
|
||||
out = append(out, 0)
|
||||
case 'b':
|
||||
out = append(out, '\b')
|
||||
case 'Z':
|
||||
out = append(out, 0x1a)
|
||||
default:
|
||||
// Includes \N in a mixed field: only a whole-field \N is NULL.
|
||||
out = append(out, f[i])
|
||||
}
|
||||
}
|
||||
return row.Value{Raw: out}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
func TestDumpReaderEmptyFieldAndNull(t *testing.T) {
|
||||
input := "a\t\\N\t\tx\\Ny\n"
|
||||
dr := NewDumpReader(strings.NewReader(input))
|
||||
got, err := dr.ReadRow()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Values) != 4 {
|
||||
t.Fatalf("got %d fields", len(got.Values))
|
||||
}
|
||||
if got.Values[0].IsNull() || got.Values[0].String() != "a" {
|
||||
t.Fatalf("field 0 = %+v", got.Values[0])
|
||||
}
|
||||
if !got.Values[1].IsNull() {
|
||||
t.Fatalf("field 1 should be NULL")
|
||||
}
|
||||
if got.Values[2].IsNull() || len(got.Values[2].Raw) != 0 {
|
||||
t.Fatalf("field 2 should be an empty string, got %+v", got.Values[2])
|
||||
}
|
||||
if got.Values[3].String() != "xNy" {
|
||||
t.Fatalf("\\N inside a field should yield N, got %q", got.Values[3].String())
|
||||
}
|
||||
if _, err := dr.ReadRow(); err != io.EOF {
|
||||
t.Fatalf("expected EOF, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDumpReaderEscapeTable(t *testing.T) {
|
||||
input := "a\t\\N\t\t\\t\\n\\r\\0\\b\\Z\\\\\t中文😀\n"
|
||||
dr := NewDumpReader(strings.NewReader(input))
|
||||
got, err := dr.ReadRow()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Values) != 5 {
|
||||
t.Fatalf("got %d fields: %+v", len(got.Values), got.Values)
|
||||
}
|
||||
want := []byte{'\t', '\n', '\r', 0, '\b', 0x1a, '\\'}
|
||||
if !bytes.Equal(got.Values[3].Raw, want) {
|
||||
t.Fatalf("escapes = %q, want %q", got.Values[3].Raw, want)
|
||||
}
|
||||
if got.Values[4].String() != "中文😀" {
|
||||
t.Fatalf("unicode = %q", got.Values[4].String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDumpReaderRoundTripWithEncoder(t *testing.T) {
|
||||
cols := []row.Column{{Name: "a"}, {Name: "b"}, {Name: "c"}}
|
||||
enc, err := row.NewEncoder("dump", cols, []int{0, 1, 2}, row.Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows := []row.Row{
|
||||
{Values: []row.Value{{Raw: []byte("")}, {}, {Raw: []byte{0, 1, 2, 255}}}},
|
||||
{Values: []row.Value{{Raw: []byte("tab\there")}, {Raw: []byte("line\nbreak")}, {Raw: []byte("back\\slash")}}},
|
||||
{Values: []row.Value{{Raw: []byte("中文")}, {Raw: []byte("😀")}, {Raw: []byte("")}}},
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
for _, r := range rows {
|
||||
b, err := enc.Encode(nil, r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buf.Write(b)
|
||||
}
|
||||
dr := NewDumpReader(&buf)
|
||||
for i, want := range rows {
|
||||
got, err := dr.ReadRow()
|
||||
if err != nil {
|
||||
t.Fatalf("row %d: %v", i, err)
|
||||
}
|
||||
if len(got.Values) != len(want.Values) {
|
||||
t.Fatalf("row %d: %d fields", i, len(got.Values))
|
||||
}
|
||||
for j := range want.Values {
|
||||
w, g := want.Values[j], got.Values[j]
|
||||
if w.IsNull() != g.IsNull() || !bytes.Equal(w.Raw, g.Raw) {
|
||||
t.Fatalf("row %d field %d: got %+v want %+v", i, j, g, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := dr.ReadRow(); err != io.EOF {
|
||||
t.Fatalf("expected EOF, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/cli"
|
||||
"archer/internal/row"
|
||||
"archer/internal/shard"
|
||||
"archer/internal/sink"
|
||||
)
|
||||
|
||||
// insertBatchRows bounds the decoder's memory independently of
|
||||
// max_allowed_packet; sink.DB still splits each batch by packet size.
|
||||
const insertBatchRows = 1000
|
||||
|
||||
type worker struct {
|
||||
conn *sql.Conn
|
||||
sink *sink.DB
|
||||
cfg *cli.ImportConfig
|
||||
archive *Archive
|
||||
mapping *Mapping
|
||||
dbName string
|
||||
}
|
||||
|
||||
func newWorker(ctx context.Context, db *sql.DB, cfg *cli.ImportConfig, archive *Archive, mapping *Mapping, dbName string) (*worker, error) {
|
||||
conn, err := db.Conn(ctx)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("acquire connection: %w", err))
|
||||
}
|
||||
if err := SetSession(ctx, conn, cfg.Charset, cfg.TimeZone); err != nil {
|
||||
conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
w := &worker{conn: conn, cfg: cfg, archive: archive, mapping: mapping, dbName: dbName}
|
||||
if cfg.Engine == "insert" {
|
||||
s, err := sink.NewDB(ctx, db, dbName, cfg.DestTable, mapping.InsertColumns, cfg.Mode, mapping.InsertFieldIdx)
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
w.sink = s
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
func (w *worker) close() {
|
||||
if w.conn != nil {
|
||||
_ = w.conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// importShard loads one shard and returns affected rows plus, for ignore mode,
|
||||
// the number of rows the server skipped.
|
||||
func (w *worker) importShard(ctx context.Context, e shard.Entry) (affected, skipped int64, err error) {
|
||||
path := shardPath(w.archive.Root, e)
|
||||
if w.cfg.Verify == "pre" {
|
||||
if err := VerifyShard(path, e); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
}
|
||||
if w.cfg.Engine == "insert" {
|
||||
return w.insertShard(ctx, e, path)
|
||||
}
|
||||
return w.loadShard(ctx, e, path)
|
||||
}
|
||||
|
||||
func (w *worker) loadShard(ctx context.Context, e shard.Entry, path string) (int64, int64, error) {
|
||||
name := newReaderName(e.Seq)
|
||||
mysql.RegisterReaderHandler(name, func() io.Reader {
|
||||
return openShardReader(path, w.archive.Compressor)
|
||||
})
|
||||
defer mysql.DeregisterReaderHandler(name)
|
||||
|
||||
q := BuildLoadSQL(w.dbName, w.cfg.DestTable, w.cfg.Charset, w.mapping, w.cfg.Mode, name)
|
||||
res, err := w.conn.ExecContext(ctx, q)
|
||||
if err != nil {
|
||||
return 0, 0, archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: LOAD DATA: %w", e.Seq, err))
|
||||
}
|
||||
affected, _ := res.RowsAffected()
|
||||
if err := w.checkWarnings(ctx, e.Seq); err != nil {
|
||||
return affected, 0, err
|
||||
}
|
||||
skipped, err := CheckRowRule(w.cfg.Mode, e.Rows, affected)
|
||||
if err != nil {
|
||||
return affected, 0, fmt.Errorf("shard %d: %w", e.Seq, err)
|
||||
}
|
||||
return affected, skipped, nil
|
||||
}
|
||||
|
||||
func (w *worker) insertShard(ctx context.Context, e shard.Entry, path string) (int64, int64, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, 0, archererr.Wrap(archererr.KindValidation, fmt.Errorf("shard %d: open: %w", e.Seq, err))
|
||||
}
|
||||
defer f.Close()
|
||||
r, err := w.archive.Compressor.Reader(f)
|
||||
if err != nil {
|
||||
return 0, 0, archererr.Wrap(archererr.KindValidation, fmt.Errorf("shard %d: decompress: %w", e.Seq, err))
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
tx, err := w.conn.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, 0, archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: begin transaction: %w", e.Seq, err))
|
||||
}
|
||||
committed := false
|
||||
defer func() {
|
||||
if !committed {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
|
||||
dr := NewDumpReader(r)
|
||||
chunk := &row.Chunk{}
|
||||
var affected int64
|
||||
flush := func() error {
|
||||
if len(chunk.Rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
n, err := w.sink.InsertChunk(ctx, tx, chunk)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: %w", e.Seq, err))
|
||||
}
|
||||
affected += n
|
||||
chunk = &row.Chunk{}
|
||||
return w.checkWarnings(ctx, e.Seq)
|
||||
}
|
||||
|
||||
for {
|
||||
rw, err := dr.ReadRow()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return affected, 0, archererr.Wrap(archererr.KindValidation, fmt.Errorf("shard %d: decode: %w", e.Seq, err))
|
||||
}
|
||||
if len(rw.Values) != len(w.mapping.FileColumns) {
|
||||
return affected, 0, archererr.Newf(archererr.KindValidation,
|
||||
"shard %d: row has %d fields but the mapping expects %d", e.Seq, len(rw.Values), len(w.mapping.FileColumns))
|
||||
}
|
||||
chunk.Rows = append(chunk.Rows, rw)
|
||||
if len(chunk.Rows) >= insertBatchRows {
|
||||
if err := flush(); err != nil {
|
||||
return affected, 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := flush(); err != nil {
|
||||
return affected, 0, err
|
||||
}
|
||||
skipped, err := CheckRowRule(w.cfg.Mode, e.Rows, affected)
|
||||
if err != nil {
|
||||
return affected, 0, fmt.Errorf("shard %d: %w", e.Seq, err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return affected, 0, archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: commit: %w", e.Seq, err))
|
||||
}
|
||||
committed = true
|
||||
return affected, skipped, nil
|
||||
}
|
||||
|
||||
func (w *worker) checkWarnings(ctx context.Context, seq int64) error {
|
||||
warnings, err := ShowWarnings(ctx, w.conn)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: SHOW WARNINGS: %w", seq, err))
|
||||
}
|
||||
return CheckWarnings(warnings, w.cfg.AllowWarnings, w.cfg.Mode, seq)
|
||||
}
|
||||
|
||||
var readerCounter atomic.Uint64
|
||||
|
||||
func newReaderName(seq int64) string {
|
||||
return fmt.Sprintf("archer-import-%d-%d-%d", os.Getpid(), seq, readerCounter.Add(1))
|
||||
}
|
||||
|
||||
type errReader struct{ err error }
|
||||
|
||||
func (e errReader) Read([]byte) (int, error) { return 0, e.err }
|
||||
|
||||
type readCloserPair struct {
|
||||
r io.ReadCloser
|
||||
f *os.File
|
||||
}
|
||||
|
||||
func (p *readCloserPair) Read(b []byte) (int, error) { return p.r.Read(b) }
|
||||
|
||||
func (p *readCloserPair) Close() error {
|
||||
err := p.r.Close()
|
||||
if ferr := p.f.Close(); err == nil {
|
||||
err = ferr
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func openShardReader(path string, comp *shard.Compressor) io.Reader {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return errReader{err}
|
||||
}
|
||||
r, err := comp.Reader(f)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return errReader{err}
|
||||
}
|
||||
return &readCloserPair{r: r, f: f}
|
||||
}
|
||||
@@ -0,0 +1,429 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
|
||||
"archer/internal/app"
|
||||
"archer/internal/cli"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
if dsn := os.Getenv("ARCHER_TEST_DSN"); dsn != "" {
|
||||
if cfg, err := mysql.ParseDSN(dsn); err == nil && cfg.DBName != "" {
|
||||
dbName := cfg.DBName
|
||||
cfg.DBName = ""
|
||||
if db, err := sql.Open("mysql", cfg.FormatDSN()); err == nil {
|
||||
_, _ = db.Exec("CREATE DATABASE IF NOT EXISTS `" + dbName + "`")
|
||||
_, _ = db.Exec("SET GLOBAL local_infile=ON")
|
||||
_, _ = db.Exec("SET GLOBAL sql_mode=''")
|
||||
db.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func testDSN(t *testing.T) string {
|
||||
t.Helper()
|
||||
d := os.Getenv("ARCHER_TEST_DSN")
|
||||
if d == "" {
|
||||
t.Skip("ARCHER_TEST_DSN is not set; skipping MySQL integration test")
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func charsetDSN(t *testing.T, dsn string) string {
|
||||
t.Helper()
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Params == nil {
|
||||
cfg.Params = map[string]string{}
|
||||
}
|
||||
cfg.Params["charset"] = "utf8mb4"
|
||||
return cfg.FormatDSN()
|
||||
}
|
||||
|
||||
func openDB(t *testing.T, dsn string) *sql.DB {
|
||||
t.Helper()
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Ping(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
func execSQL(t *testing.T, db *sql.DB, q string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := db.Exec(q, args...); err != nil {
|
||||
t.Fatalf("exec %q: %v", q, err)
|
||||
}
|
||||
}
|
||||
|
||||
func exportTable(t *testing.T, dsn, table, dir string, extra ...string) {
|
||||
t.Helper()
|
||||
args := []string{"--source", dsn, "--table", table, "--where", "1=1", "--file", dir, "--progress", "off"}
|
||||
args = append(args, extra...)
|
||||
cfg, err := cli.Parse(args, io.Discard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := app.Run(context.Background(), cfg); err != nil {
|
||||
t.Fatalf("export %s: %v", table, err)
|
||||
}
|
||||
}
|
||||
|
||||
func importArchive(t *testing.T, dsn, dir string, extra ...string) error {
|
||||
t.Helper()
|
||||
args := []string{"--file", dir, "--dest", dsn, "--progress", "off"}
|
||||
args = append(args, extra...)
|
||||
cfg, err := cli.ParseImport(args, io.Discard)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return Run(context.Background(), cfg)
|
||||
}
|
||||
|
||||
func countRows(t *testing.T, db *sql.DB, table string) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := db.QueryRow("SELECT COUNT(*) FROM `" + table + "`").Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// fingerprint renders an ordered result set byte-for-byte, using hex so NULL,
|
||||
// tabs, newlines and binary values are all distinguishable.
|
||||
func fingerprint(t *testing.T, db *sql.DB, q string) string {
|
||||
t.Helper()
|
||||
rows, err := db.Query(q)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
cols, err := rows.Columns()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var b strings.Builder
|
||||
for rows.Next() {
|
||||
raw := make([]sql.RawBytes, len(cols))
|
||||
ptrs := make([]any, len(cols))
|
||||
for i := range raw {
|
||||
ptrs[i] = &raw[i]
|
||||
}
|
||||
if err := rows.Scan(ptrs...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, v := range raw {
|
||||
if i > 0 {
|
||||
b.WriteByte('|')
|
||||
}
|
||||
if v == nil {
|
||||
b.WriteString("<NULL>")
|
||||
} else {
|
||||
fmt.Fprintf(&b, "%x", []byte(v))
|
||||
}
|
||||
}
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func seedSimple(t *testing.T, db *sql.DB, table string) {
|
||||
t.Helper()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS `"+table+"`")
|
||||
execSQL(t, db, "CREATE TABLE `"+table+"` (id BIGINT NOT NULL, v VARCHAR(64) NULL, PRIMARY KEY (id)) DEFAULT CHARSET=utf8mb4")
|
||||
execSQL(t, db, "INSERT INTO `"+table+"` (id, v) VALUES (1,'a'),(2,'b'),(3,'c'),(4,'tab\\there'),(5,NULL)")
|
||||
}
|
||||
|
||||
func freshTable(t *testing.T, db *sql.DB, table string) {
|
||||
t.Helper()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS `"+table+"`")
|
||||
execSQL(t, db, "CREATE TABLE `"+table+"` (id BIGINT NOT NULL, v VARCHAR(64) NULL, PRIMARY KEY (id)) DEFAULT CHARSET=utf8mb4")
|
||||
}
|
||||
|
||||
func TestIntegrationExportImport(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openDB(t, dsn)
|
||||
defer db.Close()
|
||||
seedSimple(t, db, "it_imp_src")
|
||||
dir := t.TempDir()
|
||||
exportTable(t, dsn, "it_imp_src", dir)
|
||||
|
||||
freshTable(t, db, "it_imp_dst")
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_dst", "--assume-all-columns"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := countRows(t, db, "it_imp_dst"); n != 5 {
|
||||
t.Fatalf("imported %d rows", n)
|
||||
}
|
||||
if got, want := fingerprint(t, db, "SELECT id, v FROM it_imp_dst ORDER BY id"),
|
||||
fingerprint(t, db, "SELECT id, v FROM it_imp_src ORDER BY id"); got != want {
|
||||
t.Fatalf("rows differ:\n%s\nwant:\n%s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationCompression(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openDB(t, dsn)
|
||||
defer db.Close()
|
||||
seedSimple(t, db, "it_imp_comp_src")
|
||||
for _, comp := range []string{"none", "gzip", "zstd", "xz"} {
|
||||
dir := t.TempDir()
|
||||
exportTable(t, dsn, "it_imp_comp_src", dir, "--compress", comp)
|
||||
if files, _ := filepath.Glob(filepath.Join(dir, "*.dump*")); len(files) != 1 {
|
||||
t.Fatalf("%s: shard files %v", comp, files)
|
||||
}
|
||||
dst := "it_imp_comp_" + comp
|
||||
freshTable(t, db, dst)
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", dst, "--assume-all-columns"); err != nil {
|
||||
t.Fatalf("%s: %v", comp, err)
|
||||
}
|
||||
if n := countRows(t, db, dst); n != 5 {
|
||||
t.Fatalf("%s: %d rows", comp, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationModes(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openDB(t, dsn)
|
||||
defer db.Close()
|
||||
seedSimple(t, db, "it_imp_modes_src")
|
||||
dir := t.TempDir()
|
||||
exportTable(t, dsn, "it_imp_modes_src", dir)
|
||||
|
||||
freshTable(t, db, "it_imp_modes_dst")
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns"); err != nil {
|
||||
t.Fatalf("ignore re-import must be idempotent: %v", err)
|
||||
}
|
||||
if n := countRows(t, db, "it_imp_modes_dst"); n != 5 {
|
||||
t.Fatalf("ignore re-import changed row count: %d", n)
|
||||
}
|
||||
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns", "--mode", "insert"); err == nil {
|
||||
t.Fatal("insert mode must fail on duplicate keys")
|
||||
}
|
||||
|
||||
execSQL(t, db, "UPDATE it_imp_modes_dst SET v='changed' WHERE id=2")
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_modes_dst", "--assume-all-columns", "--mode", "replace"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var v string
|
||||
if err := db.QueryRow("SELECT v FROM it_imp_modes_dst WHERE id=2").Scan(&v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v != "b" {
|
||||
t.Fatalf("replace did not restore value: %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationVerifyAndPartial(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openDB(t, dsn)
|
||||
defer db.Close()
|
||||
seedSimple(t, db, "it_imp_verify_src")
|
||||
dir := t.TempDir()
|
||||
exportTable(t, dsn, "it_imp_verify_src", dir)
|
||||
|
||||
files, err := filepath.Glob(filepath.Join(dir, "*.dump"))
|
||||
if err != nil || len(files) != 1 {
|
||||
t.Fatalf("files %v err %v", files, err)
|
||||
}
|
||||
f, err := os.OpenFile(files[0], os.O_APPEND|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.WriteString("tampered\n"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Close()
|
||||
freshTable(t, db, "it_imp_verify_dst")
|
||||
err = importArchive(t, dsn, dir, "--dest-table", "it_imp_verify_dst", "--assume-all-columns")
|
||||
if err == nil || !strings.Contains(err.Error(), "failed verification") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
|
||||
// Re-export a pristine archive for the completion-gate half.
|
||||
dir = t.TempDir()
|
||||
exportTable(t, dsn, "it_imp_verify_src", dir)
|
||||
statePath := filepath.Join(dir, "state.json")
|
||||
body, err := os.ReadFile(statePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body = []byte(strings.Replace(string(body), `"complete": true`, `"complete": false`, 1))
|
||||
if err := os.WriteFile(statePath, body, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
freshTable(t, db, "it_imp_partial_dst")
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_partial_dst", "--assume-all-columns"); err == nil || !strings.Contains(err.Error(), "incomplete") {
|
||||
t.Fatalf("incomplete archive must be refused, got %v", err)
|
||||
}
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_partial_dst", "--assume-all-columns", "--allow-partial"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := countRows(t, db, "it_imp_partial_dst"); n != 5 {
|
||||
t.Fatalf("partial import: %d rows", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationWorkersAndEnginesAgree(t *testing.T) {
|
||||
dsn := testDSN(t)
|
||||
db := openDB(t, dsn)
|
||||
defer db.Close()
|
||||
seedSimple(t, db, "it_imp_par_src")
|
||||
for i := 6; i <= 20; i++ {
|
||||
execSQL(t, db, "INSERT INTO it_imp_par_src (id, v) VALUES (?, ?)", i, fmt.Sprintf("v%d", i))
|
||||
}
|
||||
dir := t.TempDir()
|
||||
exportTable(t, dsn, "it_imp_par_src", dir, "--shard-by", "count", "--shard-rows", "4", "--chunk-size", "3")
|
||||
|
||||
for _, dst := range []string{"it_imp_par_1", "it_imp_par_4", "it_imp_par_ins"} {
|
||||
freshTable(t, db, dst)
|
||||
}
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_par_1", "--workers", "1", "--assume-all-columns"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_par_4", "--workers", "4", "--assume-all-columns"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_par_ins", "--engine", "insert", "--assume-all-columns"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := fingerprint(t, db, "SELECT id, v FROM it_imp_par_1 ORDER BY id")
|
||||
for _, dst := range []string{"it_imp_par_4", "it_imp_par_ins"} {
|
||||
if got := fingerprint(t, db, "SELECT id, v FROM `"+dst+"` ORDER BY id"); got != want {
|
||||
t.Fatalf("%s differs from serial load", dst)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationFidelity(t *testing.T) {
|
||||
dsn := charsetDSN(t, testDSN(t))
|
||||
db := openDB(t, dsn)
|
||||
defer db.Close()
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_fid_src")
|
||||
execSQL(t, db, `CREATE TABLE it_imp_fid_src (
|
||||
id BIGINT NOT NULL PRIMARY KEY,
|
||||
s VARCHAR(128) NULL,
|
||||
b BLOB NULL,
|
||||
n INT NULL) DEFAULT CHARSET=utf8mb4`)
|
||||
execSQL(t, db, "INSERT INTO it_imp_fid_src (id, s, b, n) VALUES (?,?,?,?)", 1, nil, nil, nil)
|
||||
execSQL(t, db, "INSERT INTO it_imp_fid_src (id, s, b, n) VALUES (?,?,?,?)", 2, "tab\there\nnewline\\back", []byte{0x00, 0x01, 0x02, 0x03, 0xff}, 0)
|
||||
execSQL(t, db, "INSERT INTO it_imp_fid_src (id, s, b, n) VALUES (?,?,?,?)", 3, "中文😀", []byte("hi"), 42)
|
||||
|
||||
dir := t.TempDir()
|
||||
exportTable(t, dsn, "it_imp_fid_src", dir)
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_fid_dst")
|
||||
execSQL(t, db, `CREATE TABLE it_imp_fid_dst (
|
||||
id BIGINT NOT NULL PRIMARY KEY,
|
||||
s VARCHAR(128) NULL,
|
||||
b BLOB NULL,
|
||||
n INT NULL) DEFAULT CHARSET=utf8mb4`)
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_fid_dst", "--assume-all-columns", "--charset", "binary"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q := "SELECT id, s, HEX(b), n FROM %s ORDER BY id"
|
||||
got := fingerprint(t, db, fmt.Sprintf(q, "it_imp_fid_dst"))
|
||||
want := fingerprint(t, db, fmt.Sprintf(q, "it_imp_fid_src"))
|
||||
if got != want {
|
||||
t.Fatalf("fidelity mismatch:\n got: %s\nwant: %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationColumnMappings(t *testing.T) {
|
||||
dsn := charsetDSN(t, testDSN(t))
|
||||
db := openDB(t, dsn)
|
||||
defer db.Close()
|
||||
|
||||
// (a) archive-recorded column list
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_cols_src")
|
||||
execSQL(t, db, "CREATE TABLE it_imp_cols_src (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))")
|
||||
execSQL(t, db, "INSERT INTO it_imp_cols_src (id, a, b) VALUES (1,'a1','b1'),(2,'a2','b2')")
|
||||
dir := t.TempDir()
|
||||
exportTable(t, dsn, "it_imp_cols_src", dir, "--columns", "id,a")
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_cols_dst")
|
||||
execSQL(t, db, "CREATE TABLE it_imp_cols_dst (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))")
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_cols_dst"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fingerprint(t, db, "SELECT id, a, b FROM it_imp_cols_dst ORDER BY id"); got != "31|6131|<NULL>\n32|6132|<NULL>\n" {
|
||||
t.Fatalf("archive-recorded mapping: %q", got)
|
||||
}
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_cols_dst", "--columns", "id,a"); err == nil {
|
||||
t.Fatal("archive column list plus --columns must be rejected")
|
||||
}
|
||||
|
||||
// (b) explicit --columns on a default (all-columns) archive
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_all_src")
|
||||
execSQL(t, db, "CREATE TABLE it_imp_all_src (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))")
|
||||
execSQL(t, db, "INSERT INTO it_imp_all_src (id, a, b) VALUES (1,'a1','b1')")
|
||||
dir = t.TempDir()
|
||||
exportTable(t, dsn, "it_imp_all_src", dir)
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_all_dst")
|
||||
execSQL(t, db, "CREATE TABLE it_imp_all_dst (id BIGINT NOT NULL, a VARCHAR(32), b VARCHAR(32), PRIMARY KEY (id))")
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst"); err == nil {
|
||||
t.Fatal("default all-columns archive without a mapping must be refused")
|
||||
}
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst", "--columns", "id,a,b"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst", "--columns", "id,nope"); err == nil || !strings.Contains(err.Error(), "does not exist") {
|
||||
t.Fatalf("missing column: %v", err)
|
||||
}
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_all_dst", "--columns", "id"); err == nil || !strings.Contains(err.Error(), "fields") {
|
||||
t.Fatalf("field count mismatch: %v", err)
|
||||
}
|
||||
|
||||
// (c) positional mapping with a generated column
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_gen_src")
|
||||
execSQL(t, db, `CREATE TABLE it_imp_gen_src (
|
||||
id BIGINT NOT NULL PRIMARY KEY,
|
||||
a INT NOT NULL,
|
||||
b INT NOT NULL,
|
||||
total INT AS (a + b) STORED)`)
|
||||
execSQL(t, db, "INSERT INTO it_imp_gen_src (id, a, b) VALUES (1,2,3),(2,10,20)")
|
||||
dir = t.TempDir()
|
||||
exportTable(t, dsn, "it_imp_gen_src", dir)
|
||||
execSQL(t, db, "DROP TABLE IF EXISTS it_imp_gen_dst")
|
||||
execSQL(t, db, `CREATE TABLE it_imp_gen_dst (
|
||||
id BIGINT NOT NULL PRIMARY KEY,
|
||||
a INT NOT NULL,
|
||||
b INT NOT NULL,
|
||||
total INT AS (a + b) STORED)`)
|
||||
if err := importArchive(t, dsn, dir, "--dest-table", "it_imp_gen_dst", "--assume-all-columns"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []struct{ id, a, b, total int }{{1, 2, 3, 5}, {2, 10, 20, 30}} {
|
||||
var gotID, gotA, gotB, gotTotal int
|
||||
if err := db.QueryRow("SELECT id, a, b, total FROM it_imp_gen_dst WHERE id=?", want.id).
|
||||
Scan(&gotID, &gotA, &gotB, &gotTotal); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gotID != want.id || gotA != want.a || gotB != want.b || gotTotal != want.total {
|
||||
t.Fatalf("row %d = (%d,%d,%d,%d), want %+v", want.id, gotID, gotA, gotB, gotTotal, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/cli"
|
||||
"archer/internal/progress"
|
||||
"archer/internal/schema"
|
||||
"archer/internal/shard"
|
||||
)
|
||||
|
||||
// Stats collects the --statistics counters for one import run.
|
||||
type Stats struct {
|
||||
Started time.Time
|
||||
Elapsed time.Duration
|
||||
Selected int
|
||||
NotSelected int
|
||||
Succeeded int
|
||||
Failed int
|
||||
ExpectedRows int64
|
||||
ImportedRows int64
|
||||
SkippedRows int64
|
||||
Bytes int64
|
||||
IgnoredFiles int
|
||||
Partial bool
|
||||
}
|
||||
|
||||
// Run executes one import.
|
||||
func Run(ctx context.Context, cfg *cli.ImportConfig) error {
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
stats := &Stats{Started: time.Now()}
|
||||
|
||||
archive, err := LoadArchive(cfg.FileDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := archive.CheckComplete(cfg.AllowPartial); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := archive.CheckMappingChoice(cfg.Columns, cfg.AssumeAll); err != nil {
|
||||
return err
|
||||
}
|
||||
stats.Partial = cfg.AllowPartial && (archive.State == nil || !archive.State.Complete)
|
||||
|
||||
selected, warnings, err := shard.ApplySelection(archive.Entries, cfg.Selectors)
|
||||
if err != nil {
|
||||
return archererr.Newf(archererr.KindConfig, "%v", err)
|
||||
}
|
||||
if err := CheckShardFiles(archive.Root, selected); err != nil {
|
||||
return err
|
||||
}
|
||||
format, compress, comp, err := ResolveFormat(selected)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
archive.Format, archive.Compress, archive.Compressor = format, compress, comp
|
||||
stats.Selected = len(selected)
|
||||
stats.NotSelected = len(archive.Entries) - len(selected)
|
||||
for _, e := range selected {
|
||||
stats.ExpectedRows += e.Rows
|
||||
stats.Bytes += e.Bytes
|
||||
}
|
||||
|
||||
if cfg.DestTable == "" {
|
||||
cfg.DestTable = archive.SourceTable
|
||||
}
|
||||
if cfg.DestTable == "" {
|
||||
return archererr.Newf(archererr.KindConfig,
|
||||
"--dest-table is required: the archive does not record the source table name")
|
||||
}
|
||||
|
||||
if cfg.DryRun {
|
||||
fields, hasRow, ferr := FirstLineFieldCount(shardPath(archive.Root, selected[0]), comp)
|
||||
if ferr != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, ferr)
|
||||
}
|
||||
return PrintDryRun(os.Stdout, cfg, archive, selected, warnings, compress, fields, hasRow)
|
||||
}
|
||||
|
||||
for _, w := range warnings {
|
||||
fmt.Fprintln(os.Stderr, "warning:", w)
|
||||
}
|
||||
if len(warnings) > 0 {
|
||||
fmt.Fprintln(os.Stderr, "warning: shard selection is whole-shard; rows outside the requested key range may be imported")
|
||||
}
|
||||
|
||||
destCfg, err := cfg.DestConfig()
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConfig, err)
|
||||
}
|
||||
db, err := openDest(destCfg, cfg.Workers)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if err := checkTarget(ctx, db, cfg.Engine); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkTableExists(ctx, db, destCfg.DBName, cfg.DestTable); err != nil {
|
||||
return err
|
||||
}
|
||||
table, err := schema.Load(ctx, db, destCfg.DBName, cfg.DestTable)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
mapping, err := BuildMapping(table, archive.ArchivedColumns, cfg.Columns, cfg.AssumeAll)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if mapping.Positional {
|
||||
fmt.Fprintln(os.Stderr, "warning: --assume-all-columns accepts positional mapping; only the field count is checked, not column names or order")
|
||||
}
|
||||
if err := CheckFieldCount(shardPath(archive.Root, selected[0]), comp, len(mapping.FileColumns)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rep := startReporter(cfg, stats.ExpectedRows)
|
||||
if rep != nil {
|
||||
defer rep.Stop()
|
||||
}
|
||||
if err := runWorkers(ctx, db, destCfg.DBName, cfg, archive, mapping, selected, stats, rep); err != nil {
|
||||
if archererr.KindOf(err) == archererr.KindInterrupted && rep != nil {
|
||||
rep.Interrupted()
|
||||
}
|
||||
return err
|
||||
}
|
||||
stats.Elapsed = time.Since(stats.Started)
|
||||
stats.IgnoredFiles = UnregisteredFiles(archive.Root, archive.Entries)
|
||||
if rep != nil {
|
||||
rep.SetShards(int64(len(selected)))
|
||||
rep.SetBytes(stats.Bytes)
|
||||
rep.Done()
|
||||
}
|
||||
if cfg.Statistics {
|
||||
PrintStats(os.Stdout, archive, stats)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func openDest(cfg *mysql.Config, workers int) (*sql.DB, error) {
|
||||
db, err := sql.Open("mysql", cfg.FormatDSN())
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindConnection, err)
|
||||
}
|
||||
db.SetMaxOpenConns(workers + 2)
|
||||
db.SetMaxIdleConns(workers + 1)
|
||||
db.SetConnMaxLifetime(time.Hour)
|
||||
if err := db.Ping(); err != nil {
|
||||
db.Close()
|
||||
return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("connect %s: %w", cfg.Addr, err))
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func checkTarget(ctx context.Context, db *sql.DB, engine string) error {
|
||||
var localInfile int
|
||||
var version string
|
||||
if err := db.QueryRowContext(ctx, "SELECT @@local_infile, @@version").Scan(&localInfile, &version); err != nil {
|
||||
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("query local_infile: %w", err))
|
||||
}
|
||||
if engine == "load" && localInfile == 0 {
|
||||
return archererr.Newf(archererr.KindConfig,
|
||||
"target server (%s) has local_infile disabled; enable it on the server/proxy, or use --engine insert", version)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkTableExists(ctx context.Context, db *sql.DB, dbName, table string) error {
|
||||
var n int
|
||||
err := db.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*) FROM information_schema.tables
|
||||
WHERE table_schema = ? AND table_name = ?`, dbName, table).Scan(&n)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("check target table: %w", err))
|
||||
}
|
||||
if n == 0 {
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"target table %s.%s does not exist; create it before importing (archer does not create tables)", dbName, table)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func startReporter(cfg *cli.ImportConfig, total int64) *progress.Reporter {
|
||||
tty := progress.IsTerminal(os.Stderr)
|
||||
enabled := cfg.Progress.Enabled(tty)
|
||||
r := progress.New(progress.Options{
|
||||
Out: os.Stderr,
|
||||
Enabled: enabled,
|
||||
Live: enabled && tty,
|
||||
Interval: time.Second,
|
||||
RowEvery: cfg.Progress.Rows,
|
||||
})
|
||||
r.Start()
|
||||
r.SetEstimate(&progress.Estimate{Kind: "rows", Lo: 0, Hi: float64(total)})
|
||||
r.SetPosition(0)
|
||||
return r
|
||||
}
|
||||
|
||||
func runWorkers(ctx context.Context, db *sql.DB, dbName string, cfg *cli.ImportConfig, archive *Archive, mapping *Mapping, selected []shard.Entry, stats *Stats, rep *progress.Reporter) error {
|
||||
parent := ctx
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
firstErr error
|
||||
failedSeq int64
|
||||
processedRows int64
|
||||
rowsAtFailure int64
|
||||
importedRows int64
|
||||
skippedRows int64
|
||||
okShards int
|
||||
)
|
||||
recordFailure := func(e shard.Entry, err error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if firstErr != nil {
|
||||
return
|
||||
}
|
||||
if parent.Err() != nil {
|
||||
firstErr = archererr.Wrap(archererr.KindInterrupted, parent.Err())
|
||||
} else {
|
||||
firstErr = err
|
||||
}
|
||||
failedSeq = e.Seq
|
||||
rowsAtFailure = processedRows
|
||||
}
|
||||
|
||||
ranges := splitRanges(len(selected), cfg.Workers)
|
||||
for _, rg := range ranges {
|
||||
if rg[0] >= rg[1] {
|
||||
continue
|
||||
}
|
||||
part := selected[rg[0]:rg[1]]
|
||||
wg.Add(1)
|
||||
go func(part []shard.Entry) {
|
||||
defer wg.Done()
|
||||
if err := ctx.Err(); err != nil {
|
||||
recordFailure(part[0], err)
|
||||
return
|
||||
}
|
||||
w, err := newWorker(ctx, db, cfg, archive, mapping, dbName)
|
||||
if err != nil {
|
||||
recordFailure(part[0], err)
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
defer w.close()
|
||||
for _, e := range part {
|
||||
if err := ctx.Err(); err != nil {
|
||||
recordFailure(e, err)
|
||||
return
|
||||
}
|
||||
affected, skipped, err := w.importShard(ctx, e)
|
||||
if err != nil {
|
||||
recordFailure(e, err)
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
processedRows += e.Rows
|
||||
importedRows += affected
|
||||
skippedRows += skipped
|
||||
okShards++
|
||||
done := processedRows
|
||||
mu.Unlock()
|
||||
if rep != nil {
|
||||
rep.AddRows(e.Rows)
|
||||
rep.SetPosition(float64(done))
|
||||
}
|
||||
}
|
||||
}(part)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
mu.Lock()
|
||||
stats.Succeeded = okShards
|
||||
stats.ImportedRows = importedRows
|
||||
stats.SkippedRows = skippedRows
|
||||
err := firstErr
|
||||
seq := failedSeq
|
||||
atFailure := rowsAtFailure
|
||||
doneRows := processedRows
|
||||
mu.Unlock()
|
||||
if err != nil {
|
||||
if archererr.KindOf(err) == archererr.KindInterrupted {
|
||||
return fmt.Errorf("interrupted after %d rows: %w", doneRows, err)
|
||||
}
|
||||
stats.Failed = 1
|
||||
return fmt.Errorf("shard %d failed after %d rows: %w", seq, atFailure, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func splitRanges(n, workers int) [][2]int {
|
||||
if n <= 0 || workers <= 0 {
|
||||
return nil
|
||||
}
|
||||
if workers > n {
|
||||
workers = n
|
||||
}
|
||||
out := make([][2]int, 0, workers)
|
||||
base, rem := n/workers, n%workers
|
||||
start := 0
|
||||
for i := 0; i < workers; i++ {
|
||||
size := base
|
||||
if i < rem {
|
||||
size++
|
||||
}
|
||||
out = append(out, [2]int{start, start + size})
|
||||
start += size
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PrintDryRun prints the resolved configuration and selection without opening
|
||||
// a database connection.
|
||||
func PrintDryRun(w io.Writer, cfg *cli.ImportConfig, archive *Archive, selected []shard.Entry, warnings []string, compress string, fields int, hasRow bool) error {
|
||||
fmt.Fprintln(w, "== import configuration ==")
|
||||
fmt.Fprintf(w, "archive: %s\n", archive.Root)
|
||||
fmt.Fprintf(w, "dest: %s\n", redactDSN(cfg.DestDSN))
|
||||
fmt.Fprintf(w, "dest table: %s\n", cfg.DestTable)
|
||||
if len(archive.ArchivedColumns) > 0 {
|
||||
fmt.Fprintf(w, "archive columns: %s\n", join(archive.ArchivedColumns))
|
||||
} else if len(cfg.Columns) > 0 {
|
||||
fmt.Fprintf(w, "columns: %s\n", join(cfg.Columns))
|
||||
} else if cfg.AssumeAll {
|
||||
fmt.Fprintf(w, "columns: positional (--assume-all-columns)\n")
|
||||
} else {
|
||||
fmt.Fprintf(w, "columns: (unresolved; --columns or --assume-all-columns is required)\n")
|
||||
}
|
||||
fmt.Fprintf(w, "engine: %s\n", cfg.Engine)
|
||||
fmt.Fprintf(w, "mode: %s\n", cfg.Mode)
|
||||
fmt.Fprintf(w, "verify: %s\n", cfg.Verify)
|
||||
fmt.Fprintf(w, "workers: %d\n", cfg.Workers)
|
||||
fmt.Fprintf(w, "charset: %s\n", cfg.Charset)
|
||||
if cfg.TimeZone != "" {
|
||||
fmt.Fprintf(w, "time zone: %s\n", cfg.TimeZone)
|
||||
}
|
||||
fmt.Fprintf(w, "compression: %s\n", compress)
|
||||
switch {
|
||||
case archive.State == nil:
|
||||
fmt.Fprintln(w, "archive complete: unknown (state.json missing)")
|
||||
case archive.Complete:
|
||||
fmt.Fprintln(w, "archive complete: true")
|
||||
default:
|
||||
fmt.Fprintln(w, "archive complete: false (--allow-partial given)")
|
||||
}
|
||||
if hasRow {
|
||||
fmt.Fprintf(w, "first row fields: %d\n", fields)
|
||||
}
|
||||
|
||||
fmt.Fprintf(w, "\n== selected shards (%d) ==\n", len(selected))
|
||||
var rows, bytes int64
|
||||
for _, e := range selected {
|
||||
date := e.Date
|
||||
if date == "" {
|
||||
date = "-"
|
||||
}
|
||||
fmt.Fprintf(w, " seq=%-6d pk=%-24s date=%-12s rows=%-10d bytes=%-10d %s\n",
|
||||
e.Seq, e.PKLo+"-"+e.PKHi, date, e.Rows, e.Bytes, e.Path)
|
||||
rows += e.Rows
|
||||
bytes += e.Bytes
|
||||
}
|
||||
fmt.Fprintf(w, "total: %d shards, %d rows, %d bytes\n", len(selected), rows, bytes)
|
||||
fmt.Fprintln(w, "\nno database connection was made and no data was written")
|
||||
return nil
|
||||
}
|
||||
|
||||
// PrintStats prints the --statistics summary.
|
||||
func PrintStats(w io.Writer, archive *Archive, s *Stats) {
|
||||
fmt.Fprintln(w, "import summary:")
|
||||
fmt.Fprintf(w, " archive: %s\n", archive.Root)
|
||||
fmt.Fprintf(w, " partial archive: %t\n", s.Partial)
|
||||
fmt.Fprintf(w, " shards selected: %d\n", s.Selected)
|
||||
fmt.Fprintf(w, " shards succeeded: %d\n", s.Succeeded)
|
||||
fmt.Fprintf(w, " shards failed: %d\n", s.Failed)
|
||||
fmt.Fprintf(w, " shards not selected: %d\n", s.NotSelected)
|
||||
fmt.Fprintf(w, " rows expected: %d\n", s.ExpectedRows)
|
||||
fmt.Fprintf(w, " rows affected: %d\n", s.ImportedRows)
|
||||
fmt.Fprintf(w, " rows skipped: %d\n", s.SkippedRows)
|
||||
fmt.Fprintf(w, " bytes: %d\n", s.Bytes)
|
||||
fmt.Fprintf(w, " ignored files: %d (present on disk, not in manifest)\n", s.IgnoredFiles)
|
||||
fmt.Fprintf(w, " elapsed: %s\n", s.Elapsed.Round(time.Millisecond))
|
||||
}
|
||||
|
||||
func join(parts []string) string {
|
||||
out := ""
|
||||
for i, p := range parts {
|
||||
if i > 0 {
|
||||
out += ", "
|
||||
}
|
||||
out += p
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func redactDSN(dsn string) string {
|
||||
c, err := mysql.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
return "<invalid dsn>"
|
||||
}
|
||||
c.Passwd = ""
|
||||
return c.FormatDSN()
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/cli"
|
||||
"archer/internal/shard"
|
||||
)
|
||||
|
||||
func TestSplitRanges(t *testing.T) {
|
||||
got := splitRanges(10, 3)
|
||||
want := [][2]int{{0, 4}, {4, 7}, {7, 10}}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("got %v", got)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("got %v want %v", got, want)
|
||||
}
|
||||
}
|
||||
if got := splitRanges(2, 5); len(got) != 2 || got[1] != [2]int{1, 2} {
|
||||
t.Fatalf("workers beyond shards: %v", got)
|
||||
}
|
||||
if got := splitRanges(0, 3); got != nil {
|
||||
t.Fatalf("empty selection: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigErrorExitCode(t *testing.T) {
|
||||
err := Run(context.Background(), cli.ImportDefaults())
|
||||
if err == nil {
|
||||
t.Fatal("expected an error")
|
||||
}
|
||||
if got := archererr.ExitCode(err); got != 2 {
|
||||
t.Fatalf("exit code = %d, want 2 (%v)", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunEmptySelectionIsConfigError(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeManifest(t, root, []shard.Entry{{Seq: 1, Path: "000001.1-1.dump", Rows: 1}})
|
||||
writeShard(t, root, "000001.1-1.dump", "1\n")
|
||||
saveState(t, root, true, map[string]string{"table": "src"})
|
||||
|
||||
cfg := cli.ImportDefaults()
|
||||
cfg.FileDir = root
|
||||
cfg.DestDSN = "u:p@tcp(127.0.0.1:3306)/db"
|
||||
cfg.Selects = []string{"seq:9-9"}
|
||||
|
||||
err := Run(context.Background(), cfg)
|
||||
if err == nil || !strings.Contains(err.Error(), "no shards matched") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
if got := archererr.ExitCode(err); got != 2 {
|
||||
t.Fatalf("exit code = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDryRunDoesNotConnect(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeManifest(t, root, []shard.Entry{{Seq: 1, Path: "000001.1-1.dump", PKLo: "1", PKHi: "1", Rows: 1, Bytes: 3}})
|
||||
writeShard(t, root, "000001.1-1.dump", "1\tx\n")
|
||||
saveState(t, root, true, map[string]string{"table": "src"})
|
||||
|
||||
cfg := cli.ImportDefaults()
|
||||
cfg.FileDir = root
|
||||
cfg.DestDSN = "u:p@tcp(127.0.0.1:1)/db" // would fail if a connection were attempted
|
||||
cfg.DryRun = true
|
||||
|
||||
// PrintDryRun goes to stdout; Run reaching it without a connection error is
|
||||
// what matters here.
|
||||
if err := Run(context.Background(), cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrintDryRun(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
entries := []shard.Entry{{Seq: 1, Path: "000001.1-1.dump", PKLo: "1", PKHi: "1", Rows: 2, Bytes: 10}}
|
||||
writeManifest(t, root, entries)
|
||||
saveState(t, root, true, map[string]string{"table": "src", "columns": "id,title"})
|
||||
archive, err := LoadArchive(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := cli.ImportDefaults()
|
||||
cfg.FileDir = root
|
||||
cfg.DestDSN = "u:p@tcp(h:3306)/db"
|
||||
cfg.DestTable = "src"
|
||||
var buf bytes.Buffer
|
||||
if err := PrintDryRun(&buf, cfg, archive, entries, nil, "none", 2, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := buf.String()
|
||||
for _, want := range []string{"archive:", "dest table:", "selected shards (1)", "seq=1", "total: 1 shards, 2 rows, 10 bytes", "no database connection"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("missing %q in:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/shard"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
// BuildLoadSQL renders the LOAD DATA statement. Field and line rules match
|
||||
// internal/row.dumpEncoder exactly: TAB, LF, backslash escaping and \N.
|
||||
func BuildLoadSQL(dbName, table, charset string, m *Mapping, mode, readerName string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("LOAD DATA LOCAL INFILE '")
|
||||
b.WriteString(escapeSQLString("Reader::" + readerName))
|
||||
b.WriteString("'")
|
||||
switch mode {
|
||||
case "ignore":
|
||||
b.WriteString(" IGNORE")
|
||||
case "replace":
|
||||
b.WriteString(" REPLACE")
|
||||
}
|
||||
b.WriteString(" INTO TABLE ")
|
||||
b.WriteString(sqlutil.Qualified(dbName, table))
|
||||
b.WriteString(" CHARACTER SET ")
|
||||
b.WriteString(charset)
|
||||
b.WriteString(` FIELDS TERMINATED BY '\t' ESCAPED BY '\\' LINES TERMINATED BY '\n'`)
|
||||
if len(m.LoadList) > 0 {
|
||||
b.WriteString(" (")
|
||||
b.WriteString(strings.Join(m.LoadList, ", "))
|
||||
b.WriteString(")")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func escapeSQLString(s string) string {
|
||||
s = strings.ReplaceAll(s, `\`, `\\`)
|
||||
return strings.ReplaceAll(s, `'`, `''`)
|
||||
}
|
||||
|
||||
// CheckRowRule applies the per-mode affected-row rule and returns the number
|
||||
// of rows that were skipped (only meaningful for ignore mode).
|
||||
func CheckRowRule(mode string, expected, affected int64) (int64, error) {
|
||||
switch mode {
|
||||
case "insert":
|
||||
if affected != expected {
|
||||
return 0, archererr.Newf(archererr.KindValidation,
|
||||
"insert expected %d rows but wrote %d", expected, affected)
|
||||
}
|
||||
return 0, nil
|
||||
case "ignore":
|
||||
if affected > expected {
|
||||
return 0, archererr.Newf(archererr.KindValidation,
|
||||
"ignore wrote %d rows but the manifest records only %d", affected, expected)
|
||||
}
|
||||
return expected - affected, nil
|
||||
default: // replace may delete+insert, so affected can exceed rows
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
// VerifyShard streams the compressed shard through SHA-256 and compares it
|
||||
// with the manifest before any bytes are sent to the server.
|
||||
func VerifyShard(path string, e shard.Entry) error {
|
||||
if e.SHA256 == "" {
|
||||
return nil
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return archererr.Newf(archererr.KindValidation, "open shard %s: %v", e.Path, err)
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return archererr.Newf(archererr.KindValidation, "hash shard %s: %v", e.Path, err)
|
||||
}
|
||||
got := hex.EncodeToString(h.Sum(nil))
|
||||
if !strings.EqualFold(got, e.SHA256) {
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"shard %d (%s) failed verification: manifest sha256 %s, actual %s", e.Seq, e.Path, e.SHA256, got)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Warning is one SHOW WARNINGS row.
|
||||
type Warning struct {
|
||||
Level string
|
||||
Code int
|
||||
Message string
|
||||
}
|
||||
|
||||
func (w Warning) String() string { return fmt.Sprintf("%s %d: %s", w.Level, w.Code, w.Message) }
|
||||
|
||||
// ShowWarnings returns the current session warnings.
|
||||
func ShowWarnings(ctx context.Context, conn *sql.Conn) ([]Warning, error) {
|
||||
rows, err := conn.QueryContext(ctx, "SHOW WARNINGS")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Warning
|
||||
for rows.Next() {
|
||||
var w Warning
|
||||
if err := rows.Scan(&w.Level, &w.Code, &w.Message); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, w)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// CheckWarnings turns server warnings into a per-shard failure unless the
|
||||
// operator explicitly allowed them. Duplicate-key warnings under ignore mode
|
||||
// are expected skips (they are the mechanism that makes re-import idempotent),
|
||||
// so they are not failures; the skipped count comes from affected rows.
|
||||
func CheckWarnings(warnings []Warning, allow bool, mode string, seq int64) error {
|
||||
if allow || len(warnings) == 0 {
|
||||
return nil
|
||||
}
|
||||
fatal := warnings[:0:0]
|
||||
for _, w := range warnings {
|
||||
if mode == "ignore" && w.Code == 1062 {
|
||||
continue
|
||||
}
|
||||
fatal = append(fatal, w)
|
||||
}
|
||||
if len(fatal) == 0 {
|
||||
return nil
|
||||
}
|
||||
const maxShown = 5
|
||||
shown := fatal
|
||||
suffix := ""
|
||||
if len(shown) > maxShown {
|
||||
suffix = fmt.Sprintf(" (and %d more)", len(shown)-maxShown)
|
||||
shown = shown[:maxShown]
|
||||
}
|
||||
parts := make([]string, len(shown))
|
||||
for i, w := range shown {
|
||||
parts[i] = w.String()
|
||||
}
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"shard %d produced %d server warning(s); pass --allow-warnings to continue: %s%s",
|
||||
seq, len(fatal), strings.Join(parts, "; "), suffix)
|
||||
}
|
||||
|
||||
// SetSession applies the loader's character set and optional time zone to one
|
||||
// fixed connection.
|
||||
func SetSession(ctx context.Context, conn *sql.Conn, charset, timeZone string) error {
|
||||
if _, err := conn.ExecContext(ctx, "SET NAMES "+charset); err != nil {
|
||||
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("set character set %s: %w", charset, err))
|
||||
}
|
||||
if timeZone != "" {
|
||||
q := "SET time_zone = '" + escapeSQLString(timeZone) + "'"
|
||||
if _, err := conn.ExecContext(ctx, q); err != nil {
|
||||
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("set time_zone %q: %w", timeZone, err))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"archer/internal/shard"
|
||||
)
|
||||
|
||||
func TestBuildLoadSQLShape(t *testing.T) {
|
||||
m := &Mapping{LoadList: []string{"`id`", "`title`"}}
|
||||
q := BuildLoadSQL("db", "orders", "utf8mb4", m, "ignore", "r1")
|
||||
for _, want := range []string{
|
||||
"LOAD DATA LOCAL INFILE 'Reader::r1'",
|
||||
"IGNORE INTO TABLE `db`.`orders`",
|
||||
"CHARACTER SET utf8mb4",
|
||||
`FIELDS TERMINATED BY '\t' ESCAPED BY '\\' LINES TERMINATED BY '\n'`,
|
||||
"(`id`, `title`)",
|
||||
} {
|
||||
if !strings.Contains(q, want) {
|
||||
t.Fatalf("missing %q in %s", want, q)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildLoadSQLModes(t *testing.T) {
|
||||
m := &Mapping{}
|
||||
for _, tc := range []struct{ mode, want string }{
|
||||
{"insert", "INFILE 'Reader::r' INTO TABLE"},
|
||||
{"ignore", "INFILE 'Reader::r' IGNORE INTO TABLE"},
|
||||
{"replace", "INFILE 'Reader::r' REPLACE INTO TABLE"},
|
||||
} {
|
||||
q := BuildLoadSQL("db", "t", "binary", m, tc.mode, "r")
|
||||
if !strings.Contains(q, tc.want) {
|
||||
t.Fatalf("%s: %s", tc.mode, q)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildLoadSQLNoColumnList(t *testing.T) {
|
||||
q := BuildLoadSQL("db", "t", "utf8mb4", &Mapping{}, "insert", "r")
|
||||
if strings.Contains(q, "(") {
|
||||
t.Fatalf("unexpected column list: %s", q)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEscapeSQLString(t *testing.T) {
|
||||
if got := escapeSQLString(`a'b\c`); got != `a''b\\c` {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckRowRule(t *testing.T) {
|
||||
if _, err := CheckRowRule("insert", 10, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := CheckRowRule("insert", 10, 9); err == nil || !strings.Contains(err.Error(), "expected 10") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
skipped, err := CheckRowRule("ignore", 10, 7)
|
||||
if err != nil || skipped != 3 {
|
||||
t.Fatalf("skipped=%d err=%v", skipped, err)
|
||||
}
|
||||
if _, err := CheckRowRule("ignore", 10, 11); err == nil {
|
||||
t.Fatal("ignore must fail when affected exceeds rows")
|
||||
}
|
||||
if _, err := CheckRowRule("replace", 10, 15); err != nil {
|
||||
t.Fatalf("replace only reports: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyShard(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, "000001.1-1.dump")
|
||||
body := []byte("1\ta\n")
|
||||
if err := os.WriteFile(path, body, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
good := shard.Entry{Seq: 1, Path: "000001.1-1.dump", SHA256: hex.EncodeToString(sum[:])}
|
||||
if err := VerifyShard(path, good); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bad := good
|
||||
bad.SHA256 = strings.Repeat("0", 64)
|
||||
if err := VerifyShard(path, bad); err == nil || !strings.Contains(err.Error(), "failed verification") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
if err := VerifyShard(path, shard.Entry{Seq: 1, Path: path}); err != nil {
|
||||
t.Fatalf("missing hash should be skipped: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckWarnings(t *testing.T) {
|
||||
if err := CheckWarnings(nil, false, "insert", 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := []Warning{{Level: "Warning", Code: 1265, Message: "data truncated"}}
|
||||
if err := CheckWarnings(w, true, "insert", 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := CheckWarnings(w, false, "insert", 3); err == nil || !strings.Contains(err.Error(), "shard 3") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
dup := []Warning{{Level: "Warning", Code: 1062, Message: "Duplicate entry '1'"}}
|
||||
if err := CheckWarnings(dup, false, "ignore", 1); err != nil {
|
||||
t.Fatalf("duplicate-key warnings are expected skips under ignore: %v", err)
|
||||
}
|
||||
if err := CheckWarnings(dup, false, "insert", 1); err == nil {
|
||||
t.Fatal("duplicate warnings must fail outside ignore mode")
|
||||
}
|
||||
}
|
||||
+21
-7
@@ -63,8 +63,8 @@ func (p *Purger) DeleteChunk(ctx context.Context, exec Execer, c *row.Chunk) (in
|
||||
maxKey = v
|
||||
}
|
||||
|
||||
if p.Bulk && maxKey == nil {
|
||||
return p.deleteRange(ctx, exec, c)
|
||||
if p.Bulk {
|
||||
return p.deleteRange(ctx, exec, c, maxKey)
|
||||
}
|
||||
return p.deleteInList(ctx, exec, keys, maxKey)
|
||||
}
|
||||
@@ -85,8 +85,13 @@ func (p *Purger) currentMax(ctx context.Context) (*row.KeyPart, error) {
|
||||
return &row.KeyPart{Val: v.String}, nil
|
||||
}
|
||||
|
||||
func (p *Purger) deleteRange(ctx context.Context, exec Execer, c *row.Chunk) (int64, error) {
|
||||
lo, hi := c.FirstKey, c.LastKey
|
||||
func (p *Purger) deleteRange(ctx context.Context, exec Execer, c *row.Chunk, maxKey *row.KeyPart) (int64, error) {
|
||||
// Date scanners rewrite LastKey into a (shard column, key...) cursor for
|
||||
// resumption, so the bare key range must come from Keys.
|
||||
if len(c.Keys) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
lo, hi := c.Keys[0], c.Keys[len(c.Keys)-1]
|
||||
if lo.IsZero() || hi.IsZero() {
|
||||
return 0, nil
|
||||
}
|
||||
@@ -95,9 +100,18 @@ func (p *Purger) deleteRange(ctx context.Context, exec Execer, c *row.Chunk) (in
|
||||
args []any
|
||||
)
|
||||
if len(p.KeyCols) == 1 {
|
||||
q = fmt.Sprintf("DELETE FROM %s WHERE %s BETWEEN ? AND ? AND (%s)",
|
||||
p.table(), sqlutil.Ident(p.KeyCols[0]), p.Where)
|
||||
args = []any{lo[0].DriverValue(), hi[0].DriverValue()}
|
||||
col := sqlutil.Ident(p.KeyCols[0])
|
||||
if maxKey != nil {
|
||||
// Keep safe-auto-increment semantics: the current max row is
|
||||
// excluded even when it falls inside the range.
|
||||
q = fmt.Sprintf("DELETE FROM %s WHERE %s BETWEEN ? AND ? AND %s <> ? AND (%s)",
|
||||
p.table(), col, col, p.Where)
|
||||
args = []any{lo[0].DriverValue(), hi[0].DriverValue(), maxKey.DriverValue()}
|
||||
} else {
|
||||
q = fmt.Sprintf("DELETE FROM %s WHERE %s BETWEEN ? AND ? AND (%s)",
|
||||
p.table(), col, p.Where)
|
||||
args = []any{lo[0].DriverValue(), hi[0].DriverValue()}
|
||||
}
|
||||
} else {
|
||||
q = fmt.Sprintf("DELETE FROM %s WHERE %s >= %s AND %s <= %s AND (%s)",
|
||||
p.table(), sqlutil.RowTuple(p.KeyCols), sqlutil.ParamTuple(len(p.KeyCols)),
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package purge
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
type recordingExecer struct {
|
||||
query string
|
||||
args []any
|
||||
}
|
||||
|
||||
func (e *recordingExecer) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
|
||||
e.query = query
|
||||
e.args = args
|
||||
return fakeResult(2), nil
|
||||
}
|
||||
|
||||
type fakeResult int64
|
||||
|
||||
func (r fakeResult) LastInsertId() (int64, error) { return 0, nil }
|
||||
func (r fakeResult) RowsAffected() (int64, error) { return int64(r), nil }
|
||||
|
||||
func TestDeleteRangeUsesBareKeys(t *testing.T) {
|
||||
p := New(nil, "db", "t", []string{"id"}, "v > 0", true, false, true)
|
||||
c := &row.Chunk{
|
||||
Keys: []row.Key{row.KeyOf("1001"), row.KeyOf("3000")},
|
||||
// Date scanners set LastKey to a (shard column, key...) cursor.
|
||||
LastKey: row.KeyOf("2025-01-01 00:00:00", "3000"),
|
||||
}
|
||||
exec := &recordingExecer{}
|
||||
|
||||
n, err := p.DeleteChunk(context.Background(), exec, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("affected = %d", n)
|
||||
}
|
||||
want := "DELETE FROM `db`.`t` WHERE `id` BETWEEN ? AND ? AND (v > 0)"
|
||||
if exec.query != want {
|
||||
t.Fatalf("query:\n got %s\nwant %s", exec.query, want)
|
||||
}
|
||||
if !reflect.DeepEqual(exec.args, []any{"1001", "3000"}) {
|
||||
t.Fatalf("args = %#v", exec.args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteRangeExcludesMaxAutoIncrementKey(t *testing.T) {
|
||||
p := New(nil, "db", "t", []string{"id"}, "1=1", true, true, true)
|
||||
c := &row.Chunk{Keys: []row.Key{row.KeyOf("1001"), row.KeyOf("3000")}}
|
||||
exec := &recordingExecer{}
|
||||
|
||||
maxKey := &row.KeyPart{Val: "3000"}
|
||||
if _, err := p.deleteRange(context.Background(), exec, c, maxKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "DELETE FROM `db`.`t` WHERE `id` BETWEEN ? AND ? AND `id` <> ? AND (1=1)"
|
||||
if exec.query != want {
|
||||
t.Fatalf("query:\n got %s\nwant %s", exec.query, want)
|
||||
}
|
||||
if !reflect.DeepEqual(exec.args, []any{"1001", "3000", "3000"}) {
|
||||
t.Fatalf("args = %#v", exec.args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteRangeCompositeKeyUsesBareKeys(t *testing.T) {
|
||||
p := New(nil, "db", "t", []string{"created_at", "id"}, "1=1", true, false, false)
|
||||
c := &row.Chunk{
|
||||
Keys: []row.Key{
|
||||
row.KeyOf("2025-01-01 00:00:00", "1001"),
|
||||
row.KeyOf("2025-01-02 00:00:00", "3000"),
|
||||
},
|
||||
LastKey: row.KeyOf("2025-01-02 00:00:00", "2025-01-02 00:00:00", "3000"),
|
||||
}
|
||||
exec := &recordingExecer{}
|
||||
|
||||
if _, err := p.DeleteChunk(context.Background(), exec, c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "DELETE FROM `db`.`t` WHERE (`created_at`, `id`) >= (?, ?) AND (`created_at`, `id`) <= (?, ?) AND (1=1)"
|
||||
if exec.query != want {
|
||||
t.Fatalf("query:\n got %s\nwant %s", exec.query, want)
|
||||
}
|
||||
wantArgs := []any{"2025-01-01 00:00:00", "1001", "2025-01-02 00:00:00", "3000"}
|
||||
if !reflect.DeepEqual(exec.args, wantArgs) {
|
||||
t.Fatalf("args = %#v", exec.args)
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,43 @@ type Entry struct {
|
||||
ClosedAt string `json:"closed_at"`
|
||||
}
|
||||
|
||||
// LoadManifest reads manifest.jsonl strictly for replay. Unlike OpenManifest it
|
||||
// never creates or appends to the archive, and a malformed line is an error
|
||||
// rather than a line to skip: an importer must not mistake corruption for a
|
||||
// missing shard.
|
||||
func LoadManifest(root string) ([]Entry, error) {
|
||||
path := filepath.Join(root, manifestName)
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open manifest: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var entries []Entry
|
||||
sc := bufio.NewScanner(f)
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
|
||||
line := 0
|
||||
for sc.Scan() {
|
||||
line++
|
||||
text := strings.TrimSpace(sc.Text())
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
var e Entry
|
||||
if err := json.Unmarshal([]byte(text), &e); err != nil {
|
||||
return nil, fmt.Errorf("parse manifest.jsonl line %d: %w", line, err)
|
||||
}
|
||||
if e.Path == "" {
|
||||
return nil, fmt.Errorf("manifest.jsonl line %d: missing path", line)
|
||||
}
|
||||
entries = append(entries, e)
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return nil, fmt.Errorf("read manifest: %w", err)
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// Manifest appends sealed-shard records to manifest.jsonl and knows which
|
||||
// shards already exist.
|
||||
type Manifest struct {
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archer/internal/schema"
|
||||
)
|
||||
|
||||
// Selector is one parsed --select condition. The zero value, Kind "all", holds
|
||||
// only the raw spelling.
|
||||
type Selector struct {
|
||||
Kind string // all|seq|pk|date
|
||||
Raw string
|
||||
|
||||
SeqLo int64
|
||||
SeqHi int64
|
||||
|
||||
Lo string
|
||||
Hi string
|
||||
|
||||
DateLo string
|
||||
DateHi string
|
||||
}
|
||||
|
||||
const dateLayout = "2006-01-02"
|
||||
|
||||
// ParseSelector parses one --select value.
|
||||
func ParseSelector(s string) (Selector, error) {
|
||||
raw := strings.TrimSpace(s)
|
||||
if raw == "" {
|
||||
return Selector{}, fmt.Errorf("--select cannot be empty")
|
||||
}
|
||||
if raw == "all" {
|
||||
return Selector{Kind: "all", Raw: raw}, nil
|
||||
}
|
||||
kind, rest, ok := strings.Cut(raw, ":")
|
||||
if !ok {
|
||||
return Selector{}, fmt.Errorf("invalid --select %q: expected all, seq:A-B, pk:LO-HI or date:D1..D2", raw)
|
||||
}
|
||||
switch kind {
|
||||
case "seq":
|
||||
lo, hi, err := parseSeqRange(rest)
|
||||
if err != nil {
|
||||
return Selector{}, fmt.Errorf("invalid --select %q: %v", raw, err)
|
||||
}
|
||||
return Selector{Kind: "seq", Raw: raw, SeqLo: lo, SeqHi: hi}, nil
|
||||
case "pk":
|
||||
lo, hi, ok := strings.Cut(rest, "-")
|
||||
lo = strings.TrimSpace(lo)
|
||||
hi = strings.TrimSpace(hi)
|
||||
if !ok || lo == "" || hi == "" {
|
||||
return Selector{}, fmt.Errorf("invalid --select %q: expected pk:LO-HI", raw)
|
||||
}
|
||||
return Selector{Kind: "pk", Raw: raw, Lo: lo, Hi: hi}, nil
|
||||
case "date":
|
||||
lo, hi, ok := strings.Cut(rest, "..")
|
||||
lo, hi = strings.TrimSpace(lo), strings.TrimSpace(hi)
|
||||
if !ok || lo == "" || hi == "" {
|
||||
return Selector{}, fmt.Errorf("invalid --select %q: expected date:D1..D2", raw)
|
||||
}
|
||||
if _, err := time.Parse(dateLayout, lo); err != nil {
|
||||
return Selector{}, fmt.Errorf("invalid --select %q: %v", raw, err)
|
||||
}
|
||||
if _, err := time.Parse(dateLayout, hi); err != nil {
|
||||
return Selector{}, fmt.Errorf("invalid --select %q: %v", raw, err)
|
||||
}
|
||||
if lo > hi {
|
||||
return Selector{}, fmt.Errorf("invalid --select %q: date range is reversed", raw)
|
||||
}
|
||||
return Selector{Kind: "date", Raw: raw, DateLo: lo, DateHi: hi}, nil
|
||||
default:
|
||||
return Selector{}, fmt.Errorf("invalid --select %q: unknown selector %q", raw, kind)
|
||||
}
|
||||
}
|
||||
|
||||
func parseSeqRange(s string) (int64, int64, error) {
|
||||
lo, hi, ok := strings.Cut(s, "-")
|
||||
if !ok {
|
||||
return 0, 0, fmt.Errorf("expected A-B")
|
||||
}
|
||||
a, err := strconv.ParseInt(strings.TrimSpace(lo), 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("bad range start %q", lo)
|
||||
}
|
||||
b, err := strconv.ParseInt(strings.TrimSpace(hi), 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("bad range end %q", hi)
|
||||
}
|
||||
if a < 0 || b < 0 {
|
||||
return 0, 0, fmt.Errorf("shard sequence numbers cannot be negative")
|
||||
}
|
||||
if a > b {
|
||||
return 0, 0, fmt.Errorf("range is reversed")
|
||||
}
|
||||
return a, b, nil
|
||||
}
|
||||
|
||||
// ParseSelection parses every --select value. An empty list means "all".
|
||||
func ParseSelection(specs []string) ([]Selector, error) {
|
||||
if len(specs) == 0 {
|
||||
return []Selector{{Kind: "all", Raw: "all"}}, nil
|
||||
}
|
||||
out := make([]Selector, 0, len(specs))
|
||||
for _, s := range specs {
|
||||
sel, err := ParseSelector(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, sel)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ApplySelection returns the deduplicated, ordered entries matched by sels.
|
||||
// Order follows the archive: bucket order (NULL, zero dates, real dates) and
|
||||
// then ascending seq, which is also the order a serial import should use.
|
||||
// Warnings describe boundary shards whose whole contents will be imported even
|
||||
// though only part of their key range was requested.
|
||||
func ApplySelection(entries []Entry, sels []Selector) ([]Entry, []string, error) {
|
||||
if len(sels) == 0 {
|
||||
sels = []Selector{{Kind: "all", Raw: "all"}}
|
||||
}
|
||||
hasDate := false
|
||||
hasAll := false
|
||||
var pkSelectors []Selector
|
||||
for _, s := range sels {
|
||||
switch s.Kind {
|
||||
case "date":
|
||||
for _, e := range entries {
|
||||
if e.Date != "" {
|
||||
hasDate = true
|
||||
break
|
||||
}
|
||||
}
|
||||
case "all":
|
||||
hasAll = true
|
||||
case "pk":
|
||||
pkSelectors = append(pkSelectors, s)
|
||||
}
|
||||
}
|
||||
for _, s := range sels {
|
||||
if s.Kind == "date" && !hasDate {
|
||||
return nil, nil, fmt.Errorf("archive has no date buckets (it was not exported with --shard-by date); --select date:%s..%s cannot be used", s.DateLo, s.DateHi)
|
||||
}
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
var selected []Entry
|
||||
var warnings []string
|
||||
for _, e := range entries {
|
||||
matched, pkFunded := matchEntry(e, sels)
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
key := e.Date + "\x00" + strconv.FormatInt(e.Seq, 10)
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
selected = append(selected, e)
|
||||
if pkFunded && !hasAll {
|
||||
if w, ok := pkBoundaryWarning(e, pkSelectors); ok {
|
||||
warnings = append(warnings, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(selected) == 0 {
|
||||
return nil, nil, fmt.Errorf("no shards matched the given --select conditions")
|
||||
}
|
||||
sort.Slice(selected, func(i, j int) bool {
|
||||
if selected[i].Date != selected[j].Date {
|
||||
return schema.BucketLess(selected[i].Date, selected[j].Date)
|
||||
}
|
||||
if selected[i].Seq != selected[j].Seq {
|
||||
return selected[i].Seq < selected[j].Seq
|
||||
}
|
||||
return selected[i].Path < selected[j].Path
|
||||
})
|
||||
return selected, warnings, nil
|
||||
}
|
||||
|
||||
func matchEntry(e Entry, sels []Selector) (matched bool, pk bool) {
|
||||
for _, s := range sels {
|
||||
switch s.Kind {
|
||||
case "all":
|
||||
return true, false
|
||||
case "seq":
|
||||
if e.Seq >= s.SeqLo && e.Seq <= s.SeqHi {
|
||||
return true, false
|
||||
}
|
||||
case "date":
|
||||
if e.Date != "" && e.Date >= s.DateLo && e.Date <= s.DateHi {
|
||||
return true, false
|
||||
}
|
||||
case "pk":
|
||||
if pkOverlap(e, s) {
|
||||
return true, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, false
|
||||
}
|
||||
|
||||
func pkOverlap(e Entry, s Selector) bool {
|
||||
if e.PKLo == "" || e.PKHi == "" {
|
||||
return false
|
||||
}
|
||||
return CompareKeyText(e.PKHi, s.Lo) >= 0 && CompareKeyText(e.PKLo, s.Hi) <= 0
|
||||
}
|
||||
|
||||
func containedInAnyPK(e Entry, sels []Selector) bool {
|
||||
for _, s := range sels {
|
||||
if CompareKeyText(e.PKLo, s.Lo) >= 0 && CompareKeyText(e.PKHi, s.Hi) <= 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func pkBoundaryWarning(e Entry, sels []Selector) (string, bool) {
|
||||
if containedInAnyPK(e, sels) {
|
||||
return "", false
|
||||
}
|
||||
for _, s := range sels {
|
||||
if !pkOverlap(e, s) {
|
||||
continue
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"shard %d covers pk [%s, %s]; requested %s overlaps it, so rows outside the requested range will be imported too",
|
||||
e.Seq, emptyAs(e.PKLo, "?"), emptyAs(e.PKHi, "?"), s.Raw), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// CompareKeyText compares two key fragments, using numeric order when both are
|
||||
// integers and lexical order otherwise.
|
||||
func CompareKeyText(a, b string) int {
|
||||
ai, aErr := strconv.ParseInt(a, 10, 64)
|
||||
bi, bErr := strconv.ParseInt(b, 10, 64)
|
||||
if aErr == nil && bErr == nil {
|
||||
switch {
|
||||
case ai < bi:
|
||||
return -1
|
||||
case ai > bi:
|
||||
return 1
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
return strings.Compare(a, b)
|
||||
}
|
||||
|
||||
func emptyAs(s, fallback string) string {
|
||||
if s == "" {
|
||||
return fallback
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func sampleEntries() []Entry {
|
||||
return []Entry{
|
||||
{Seq: 1, Path: "000001.1-100.dump", PKLo: "1", PKHi: "100", Rows: 100},
|
||||
{Seq: 2, Path: "000002.101-200.dump", PKLo: "101", PKHi: "200", Rows: 100},
|
||||
{Seq: 3, Path: "000003.201-300.dump", PKLo: "201", PKHi: "300", Rows: 100},
|
||||
{Seq: 4, Path: "000004.301-400.dump", PKLo: "301", PKHi: "400", Rows: 100},
|
||||
}
|
||||
}
|
||||
|
||||
func mustSelect(t *testing.T, entries []Entry, specs ...string) ([]Entry, []string) {
|
||||
t.Helper()
|
||||
sels, err := ParseSelection(specs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, warnings, err := ApplySelection(entries, sels)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out, warnings
|
||||
}
|
||||
|
||||
func seqs(entries []Entry) []int64 {
|
||||
out := make([]int64, len(entries))
|
||||
for i, e := range entries {
|
||||
out[i] = e.Seq
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestSelectAllByDefault(t *testing.T) {
|
||||
got, _ := mustSelect(t, sampleEntries())
|
||||
if len(got) != 4 {
|
||||
t.Fatalf("got %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSeqRange(t *testing.T) {
|
||||
got, _ := mustSelect(t, sampleEntries(), "seq:2-3")
|
||||
if len(got) != 2 || got[0].Seq != 2 || got[1].Seq != 3 {
|
||||
t.Fatalf("got %v", seqs(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectPKIntersection(t *testing.T) {
|
||||
got, warnings := mustSelect(t, sampleEntries(), "pk:150-250")
|
||||
if len(got) != 2 || got[0].Seq != 2 || got[1].Seq != 3 {
|
||||
t.Fatalf("got %v", seqs(got))
|
||||
}
|
||||
if len(warnings) != 2 {
|
||||
t.Fatalf("expected boundary warnings, got %v", warnings)
|
||||
}
|
||||
if !strings.Contains(warnings[0], "150") {
|
||||
t.Fatalf("warning should name the requested bound: %v", warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectPKContainedNoWarning(t *testing.T) {
|
||||
got, warnings := mustSelect(t, sampleEntries(), "pk:101-200")
|
||||
if len(got) != 1 || got[0].Seq != 2 {
|
||||
t.Fatalf("got %v", seqs(got))
|
||||
}
|
||||
if len(warnings) != 0 {
|
||||
t.Fatalf("unexpected warnings: %v", warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectUnionDedupes(t *testing.T) {
|
||||
got, _ := mustSelect(t, sampleEntries(), "seq:1-2", "seq:2-3")
|
||||
if len(got) != 3 || got[0].Seq != 1 || got[1].Seq != 2 || got[2].Seq != 3 {
|
||||
t.Fatalf("got %v", seqs(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectEmptyIsError(t *testing.T) {
|
||||
sels, _ := ParseSelection([]string{"seq:9-10"})
|
||||
if _, _, err := ApplySelection(sampleEntries(), sels); err == nil || !strings.Contains(err.Error(), "no shards matched") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectDateOnNonDateArchive(t *testing.T) {
|
||||
sels, _ := ParseSelection([]string{"date:2025-01-01..2025-01-31"})
|
||||
if _, _, err := ApplySelection(sampleEntries(), sels); err == nil || !strings.Contains(err.Error(), "not exported with --shard-by date") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectDateBucket(t *testing.T) {
|
||||
entries := []Entry{
|
||||
{Seq: 1, Path: "date=2025-01-01/000001.1-1.dump", Date: "2025-01-01", PKLo: "1", PKHi: "1"},
|
||||
{Seq: 1, Path: "date=2025-01-02/000001.2-2.dump", Date: "2025-01-02", PKLo: "2", PKHi: "2"},
|
||||
{Seq: 2, Path: "date=2025-01-02/000002.3-3.dump", Date: "2025-01-02", PKLo: "3", PKHi: "3"},
|
||||
{Seq: 1, Path: "date=2025-01-03/000001.4-4.dump", Date: "2025-01-03", PKLo: "4", PKHi: "4"},
|
||||
}
|
||||
got, _ := mustSelect(t, entries, "date:2025-01-02..2025-01-02")
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d entries", len(got))
|
||||
}
|
||||
// Seq repeats across buckets; dedupe must not collapse them.
|
||||
if got[0].Path != entries[1].Path || got[1].Path != entries[2].Path {
|
||||
t.Fatalf("wrong entries: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSelectorErrors(t *testing.T) {
|
||||
for _, s := range []string{"", "seq", "seq:1", "seq:a-b", "seq:3-1", "pk:1", "date:2025-01-01", "date:2025-01-02..2025-01-01", "nope:1"} {
|
||||
if _, err := ParseSelector(s); err == nil {
|
||||
t.Fatalf("%q: expected an error", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareKeyText(t *testing.T) {
|
||||
if CompareKeyText("9", "10") >= 0 {
|
||||
t.Fatal("numeric keys must sort numerically")
|
||||
}
|
||||
if CompareKeyText("a", "b") >= 0 {
|
||||
t.Fatal("non-numeric keys must sort lexically")
|
||||
}
|
||||
if CompareKeyText("5", "5") != 0 {
|
||||
t.Fatal("equal keys must compare equal")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user