archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
220 lines
6.9 KiB
Go
220 lines
6.9 KiB
Go
package cli
|
|
|
|
import (
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"regexp"
|
|
"strings"
|
|
|
|
"github.com/go-sql-driver/mysql"
|
|
|
|
"archer/internal/shard"
|
|
)
|
|
|
|
// ImportConfig is the resolved configuration of `archer import`. It is kept
|
|
// separate from Config so archive (export) validation rules never leak into
|
|
// replay and vice versa.
|
|
type ImportConfig struct {
|
|
FileDir string
|
|
DestDSN string
|
|
DestTable string
|
|
Columns []string
|
|
AssumeAll bool
|
|
Selects []string
|
|
Selectors []shard.Selector
|
|
Mode string // ignore|insert|replace
|
|
Engine string // load|insert
|
|
Verify string // pre|off
|
|
Workers int
|
|
Charset string
|
|
TimeZone string
|
|
AllowPartial bool
|
|
AllowWarnings bool
|
|
DryRun bool
|
|
Progress ProgressValue
|
|
Statistics bool
|
|
Version bool
|
|
}
|
|
|
|
// ImportDefaults returns the documented defaults for `archer import`.
|
|
func ImportDefaults() *ImportConfig {
|
|
return &ImportConfig{
|
|
Mode: "ignore",
|
|
Engine: "load",
|
|
Verify: "pre",
|
|
Workers: 1,
|
|
Charset: "utf8mb4",
|
|
Progress: ProgressValue{Mode: ProgressAuto},
|
|
}
|
|
}
|
|
|
|
// exportOnlyFlags lists archive-mode flags. They are intentionally absent from
|
|
// the import flag set, but a dedicated message is friendlier than the flag
|
|
// package's generic "flag provided but not defined".
|
|
var exportOnlyFlags = []string{
|
|
"source", "table", "where", "purge", "bulk-delete", "insert-ignore",
|
|
"replace", "chunk-size", "limit", "shard-by", "shard-rows", "shard-column",
|
|
"shard-column-type", "shard-time-zone", "zero-date", "format", "compress",
|
|
"compress-level", "csv-null", "max-lag", "check-interval",
|
|
"check-replica-lag", "retries", "safe-auto-increment", "progress-interval",
|
|
"allow-full-scan", "force-resume", "index",
|
|
}
|
|
|
|
var charsetRE = regexp.MustCompile(`^[A-Za-z0-9_]+$`)
|
|
|
|
type stringList []string
|
|
|
|
func (s *stringList) String() string { return strings.Join(*s, ",") }
|
|
func (s *stringList) Set(v string) error {
|
|
*s = append(*s, v)
|
|
return nil
|
|
}
|
|
|
|
// ParseImport parses args (without the "import" subcommand) into an
|
|
// ImportConfig. Validation is separate.
|
|
func ParseImport(args []string, errOut io.Writer) (*ImportConfig, error) {
|
|
if err := rejectExportFlags(args); err != nil {
|
|
return nil, err
|
|
}
|
|
c := ImportDefaults()
|
|
var (
|
|
columnsArg string
|
|
selects stringList
|
|
)
|
|
fs := flag.NewFlagSet("archer import", flag.ContinueOnError)
|
|
fs.SetOutput(errOut)
|
|
fs.Usage = func() {
|
|
fmt.Fprintln(errOut, "usage: archer import --file <archive-dir> --dest <dsn> [flags]")
|
|
fmt.Fprintln(errOut, "")
|
|
fmt.Fprintln(errOut, "Replay a dump archive into an existing MySQL table.")
|
|
fmt.Fprintln(errOut, "")
|
|
fmt.Fprintln(errOut, "flags:")
|
|
fs.PrintDefaults()
|
|
}
|
|
|
|
fs.StringVar(&c.FileDir, "file", "", "archive directory containing manifest.jsonl (required)")
|
|
fs.StringVar(&c.DestDSN, "dest", "", "target MySQL DSN (required, must include a database)")
|
|
fs.StringVar(&c.DestTable, "dest-table", "", "target table (defaults to the archived source table)")
|
|
fs.StringVar(&columnsArg, "columns", "", "comma separated target columns in file field order")
|
|
fs.BoolVar(&c.AssumeAll, "assume-all-columns", false, "accept positional mapping to all target columns, in ordinal order")
|
|
fs.Var(&selects, "select", "shard selector (repeatable): all, seq:A-B, pk:LO-HI, date:D1..D2")
|
|
fs.StringVar(&c.Mode, "mode", c.Mode, "insert mode: ignore|insert|replace")
|
|
fs.StringVar(&c.Engine, "engine", c.Engine, "load engine: load|insert")
|
|
fs.StringVar(&c.Verify, "verify", c.Verify, "integrity check: pre|off")
|
|
fs.IntVar(&c.Workers, "workers", c.Workers, "parallel shard workers (must be greater than 0)")
|
|
fs.StringVar(&c.Charset, "charset", c.Charset, "character set used for loading")
|
|
fs.StringVar(&c.TimeZone, "time-zone", "", "target session time zone (empty: server default)")
|
|
fs.BoolVar(&c.AllowPartial, "allow-partial", false, "import sealed shards from an archive whose state.json is incomplete")
|
|
fs.BoolVar(&c.AllowWarnings, "allow-warnings", false, "do not treat server warnings as failures")
|
|
fs.BoolVar(&c.DryRun, "dry-run", false, "print selected shards and configuration without connecting or writing")
|
|
fs.Var(&c.Progress, "progress", "progress output: auto|on|off|N")
|
|
fs.BoolVar(&c.Statistics, "statistics", false, "print a summary when finished")
|
|
fs.BoolVar(&c.Version, "version", false, "print version and exit")
|
|
|
|
if err := fs.Parse(args); err != nil {
|
|
return nil, err
|
|
}
|
|
if fs.NArg() > 0 {
|
|
return nil, fmt.Errorf("unexpected argument %q", fs.Arg(0))
|
|
}
|
|
if c.Version {
|
|
return c, nil
|
|
}
|
|
if columnsArg != "" {
|
|
for _, part := range strings.Split(columnsArg, ",") {
|
|
part = strings.TrimSpace(part)
|
|
if part != "" {
|
|
c.Columns = append(c.Columns, part)
|
|
}
|
|
}
|
|
}
|
|
c.Selects = append([]string(nil), selects...)
|
|
return c, nil
|
|
}
|
|
|
|
func rejectExportFlags(args []string) error {
|
|
exportOnly := make(map[string]bool, len(exportOnlyFlags))
|
|
for _, name := range exportOnlyFlags {
|
|
exportOnly[name] = true
|
|
}
|
|
for _, a := range args {
|
|
if a == "--" {
|
|
break
|
|
}
|
|
if !strings.HasPrefix(a, "--") {
|
|
continue
|
|
}
|
|
name := strings.TrimPrefix(a, "--")
|
|
if i := strings.IndexByte(name, '='); i >= 0 {
|
|
name = name[:i]
|
|
}
|
|
if exportOnly[name] {
|
|
return cfgErr("--%s is only valid in archive (export) mode", name)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Validate checks required flags and the import-only dependency rules. Archive
|
|
// contents (completion, recorded columns, selectors against entries) are
|
|
// checked by the importer before any database connection is opened.
|
|
func (c *ImportConfig) Validate() error {
|
|
if c.FileDir == "" {
|
|
return cfgErr("--file is required")
|
|
}
|
|
if c.DestDSN == "" {
|
|
return cfgErr("--dest is required")
|
|
}
|
|
dest, err := normalize(c.DestDSN)
|
|
if err != nil {
|
|
return cfgErr("--dest: %v", err)
|
|
}
|
|
if dest.DBName == "" {
|
|
return cfgErr("--dest must include a database name")
|
|
}
|
|
switch c.Mode {
|
|
case "ignore", "insert", "replace":
|
|
default:
|
|
return cfgErr("--mode must be one of ignore, insert, replace")
|
|
}
|
|
switch c.Engine {
|
|
case "load", "insert":
|
|
default:
|
|
return cfgErr("--engine must be one of load, insert")
|
|
}
|
|
switch c.Verify {
|
|
case "pre", "off":
|
|
default:
|
|
return cfgErr("--verify must be pre or off")
|
|
}
|
|
if c.Workers <= 0 {
|
|
return cfgErr("--workers must be greater than 0")
|
|
}
|
|
if !charsetRE.MatchString(c.Charset) {
|
|
return cfgErr("--charset %q is not a valid character set name", c.Charset)
|
|
}
|
|
if len(c.Columns) > 0 && c.AssumeAll {
|
|
return cfgErr("--columns and --assume-all-columns are mutually exclusive")
|
|
}
|
|
if err := c.Progress.resolve(); err != nil {
|
|
return cfgErr("%v", err)
|
|
}
|
|
sels, err := shard.ParseSelection(c.Selects)
|
|
if err != nil {
|
|
return cfgErr("%v", err)
|
|
}
|
|
for _, s := range sels {
|
|
if s.Kind == "all" && len(sels) > 1 {
|
|
return cfgErr("--select all cannot be combined with other selectors")
|
|
}
|
|
}
|
|
c.Selectors = sels
|
|
return nil
|
|
}
|
|
|
|
// DestConfig parses the target DSN, forcing parseTime off like the export path.
|
|
func (c *ImportConfig) DestConfig() (*mysql.Config, error) {
|
|
return normalize(c.DestDSN)
|
|
}
|