Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4a6b504f
commit
366d61ca25
49 files changed
+5099
-342
No files matched your search
@@ -87,8 +87,8 @@ func TestValidateShardRules(t *testing.T) {
|
||||
|
||||
args = append(baseArgs(), "--shard-by", "date", "--shard-column", "c", "--bulk-delete", "--purge")
|
||||
c, _ = Parse(args, &bytes.Buffer{})
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "bulk-delete") {
|
||||
t.Fatalf("got %v", err)
|
||||
if err := c.Validate(); err != nil {
|
||||
t.Fatalf("date mode bulk delete should be allowed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
|
||||
"archer/internal/shard"
|
||||
)
|
||||
|
||||
// ImportConfig is the resolved configuration of `archer import`. It is kept
|
||||
// separate from Config so archive (export) validation rules never leak into
|
||||
// replay and vice versa.
|
||||
type ImportConfig struct {
|
||||
FileDir string
|
||||
DestDSN string
|
||||
DestTable string
|
||||
Columns []string
|
||||
AssumeAll bool
|
||||
Selects []string
|
||||
Selectors []shard.Selector
|
||||
Mode string // ignore|insert|replace
|
||||
Engine string // load|insert
|
||||
Verify string // pre|off
|
||||
Workers int
|
||||
Charset string
|
||||
TimeZone string
|
||||
AllowPartial bool
|
||||
AllowWarnings bool
|
||||
DryRun bool
|
||||
Progress ProgressValue
|
||||
Statistics bool
|
||||
Version bool
|
||||
}
|
||||
|
||||
// ImportDefaults returns the documented defaults for `archer import`.
|
||||
func ImportDefaults() *ImportConfig {
|
||||
return &ImportConfig{
|
||||
Mode: "ignore",
|
||||
Engine: "load",
|
||||
Verify: "pre",
|
||||
Workers: 1,
|
||||
Charset: "utf8mb4",
|
||||
Progress: ProgressValue{Mode: ProgressAuto},
|
||||
}
|
||||
}
|
||||
|
||||
// exportOnlyFlags lists archive-mode flags. They are intentionally absent from
|
||||
// the import flag set, but a dedicated message is friendlier than the flag
|
||||
// package's generic "flag provided but not defined".
|
||||
var exportOnlyFlags = []string{
|
||||
"source", "table", "where", "purge", "bulk-delete", "insert-ignore",
|
||||
"replace", "chunk-size", "limit", "shard-by", "shard-rows", "shard-column",
|
||||
"shard-column-type", "shard-time-zone", "zero-date", "format", "compress",
|
||||
"compress-level", "csv-null", "max-lag", "check-interval",
|
||||
"check-replica-lag", "retries", "safe-auto-increment", "progress-interval",
|
||||
"allow-full-scan", "force-resume", "index",
|
||||
}
|
||||
|
||||
var charsetRE = regexp.MustCompile(`^[A-Za-z0-9_]+$`)
|
||||
|
||||
type stringList []string
|
||||
|
||||
func (s *stringList) String() string { return strings.Join(*s, ",") }
|
||||
func (s *stringList) Set(v string) error {
|
||||
*s = append(*s, v)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ParseImport parses args (without the "import" subcommand) into an
|
||||
// ImportConfig. Validation is separate.
|
||||
func ParseImport(args []string, errOut io.Writer) (*ImportConfig, error) {
|
||||
if err := rejectExportFlags(args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := ImportDefaults()
|
||||
var (
|
||||
columnsArg string
|
||||
selects stringList
|
||||
)
|
||||
fs := flag.NewFlagSet("archer import", flag.ContinueOnError)
|
||||
fs.SetOutput(errOut)
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintln(errOut, "usage: archer import --file <archive-dir> --dest <dsn> [flags]")
|
||||
fmt.Fprintln(errOut, "")
|
||||
fmt.Fprintln(errOut, "Replay a dump archive into an existing MySQL table.")
|
||||
fmt.Fprintln(errOut, "")
|
||||
fmt.Fprintln(errOut, "flags:")
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
|
||||
fs.StringVar(&c.FileDir, "file", "", "archive directory containing manifest.jsonl (required)")
|
||||
fs.StringVar(&c.DestDSN, "dest", "", "target MySQL DSN (required, must include a database)")
|
||||
fs.StringVar(&c.DestTable, "dest-table", "", "target table (defaults to the archived source table)")
|
||||
fs.StringVar(&columnsArg, "columns", "", "comma separated target columns in file field order")
|
||||
fs.BoolVar(&c.AssumeAll, "assume-all-columns", false, "accept positional mapping to all target columns, in ordinal order")
|
||||
fs.Var(&selects, "select", "shard selector (repeatable): all, seq:A-B, pk:LO-HI, date:D1..D2")
|
||||
fs.StringVar(&c.Mode, "mode", c.Mode, "insert mode: ignore|insert|replace")
|
||||
fs.StringVar(&c.Engine, "engine", c.Engine, "load engine: load|insert")
|
||||
fs.StringVar(&c.Verify, "verify", c.Verify, "integrity check: pre|off")
|
||||
fs.IntVar(&c.Workers, "workers", c.Workers, "parallel shard workers (must be greater than 0)")
|
||||
fs.StringVar(&c.Charset, "charset", c.Charset, "character set used for loading")
|
||||
fs.StringVar(&c.TimeZone, "time-zone", "", "target session time zone (empty: server default)")
|
||||
fs.BoolVar(&c.AllowPartial, "allow-partial", false, "import sealed shards from an archive whose state.json is incomplete")
|
||||
fs.BoolVar(&c.AllowWarnings, "allow-warnings", false, "do not treat server warnings as failures")
|
||||
fs.BoolVar(&c.DryRun, "dry-run", false, "print selected shards and configuration without connecting or writing")
|
||||
fs.Var(&c.Progress, "progress", "progress output: auto|on|off|N")
|
||||
fs.BoolVar(&c.Statistics, "statistics", false, "print a summary when finished")
|
||||
fs.BoolVar(&c.Version, "version", false, "print version and exit")
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if fs.NArg() > 0 {
|
||||
return nil, fmt.Errorf("unexpected argument %q", fs.Arg(0))
|
||||
}
|
||||
if c.Version {
|
||||
return c, nil
|
||||
}
|
||||
if columnsArg != "" {
|
||||
for _, part := range strings.Split(columnsArg, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
c.Columns = append(c.Columns, part)
|
||||
}
|
||||
}
|
||||
}
|
||||
c.Selects = append([]string(nil), selects...)
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func rejectExportFlags(args []string) error {
|
||||
exportOnly := make(map[string]bool, len(exportOnlyFlags))
|
||||
for _, name := range exportOnlyFlags {
|
||||
exportOnly[name] = true
|
||||
}
|
||||
for _, a := range args {
|
||||
if a == "--" {
|
||||
break
|
||||
}
|
||||
if !strings.HasPrefix(a, "--") {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimPrefix(a, "--")
|
||||
if i := strings.IndexByte(name, '='); i >= 0 {
|
||||
name = name[:i]
|
||||
}
|
||||
if exportOnly[name] {
|
||||
return cfgErr("--%s is only valid in archive (export) mode", name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate checks required flags and the import-only dependency rules. Archive
|
||||
// contents (completion, recorded columns, selectors against entries) are
|
||||
// checked by the importer before any database connection is opened.
|
||||
func (c *ImportConfig) Validate() error {
|
||||
if c.FileDir == "" {
|
||||
return cfgErr("--file is required")
|
||||
}
|
||||
if c.DestDSN == "" {
|
||||
return cfgErr("--dest is required")
|
||||
}
|
||||
dest, err := normalize(c.DestDSN)
|
||||
if err != nil {
|
||||
return cfgErr("--dest: %v", err)
|
||||
}
|
||||
if dest.DBName == "" {
|
||||
return cfgErr("--dest must include a database name")
|
||||
}
|
||||
switch c.Mode {
|
||||
case "ignore", "insert", "replace":
|
||||
default:
|
||||
return cfgErr("--mode must be one of ignore, insert, replace")
|
||||
}
|
||||
switch c.Engine {
|
||||
case "load", "insert":
|
||||
default:
|
||||
return cfgErr("--engine must be one of load, insert")
|
||||
}
|
||||
switch c.Verify {
|
||||
case "pre", "off":
|
||||
default:
|
||||
return cfgErr("--verify must be pre or off")
|
||||
}
|
||||
if c.Workers <= 0 {
|
||||
return cfgErr("--workers must be greater than 0")
|
||||
}
|
||||
if !charsetRE.MatchString(c.Charset) {
|
||||
return cfgErr("--charset %q is not a valid character set name", c.Charset)
|
||||
}
|
||||
if len(c.Columns) > 0 && c.AssumeAll {
|
||||
return cfgErr("--columns and --assume-all-columns are mutually exclusive")
|
||||
}
|
||||
if err := c.Progress.resolve(); err != nil {
|
||||
return cfgErr("%v", err)
|
||||
}
|
||||
sels, err := shard.ParseSelection(c.Selects)
|
||||
if err != nil {
|
||||
return cfgErr("%v", err)
|
||||
}
|
||||
for _, s := range sels {
|
||||
if s.Kind == "all" && len(sels) > 1 {
|
||||
return cfgErr("--select all cannot be combined with other selectors")
|
||||
}
|
||||
}
|
||||
c.Selectors = sels
|
||||
return nil
|
||||
}
|
||||
|
||||
// DestConfig parses the target DSN, forcing parseTime off like the export path.
|
||||
func (c *ImportConfig) DestConfig() (*mysql.Config, error) {
|
||||
return normalize(c.DestDSN)
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func importArgs(extra ...string) []string {
|
||||
args := []string{"--file", "archive", "--dest", "u:p@tcp(127.0.0.1:3306)/db"}
|
||||
return append(args, extra...)
|
||||
}
|
||||
|
||||
func parseImport(t *testing.T, args ...string) *ImportConfig {
|
||||
t.Helper()
|
||||
c, err := ParseImport(args, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatalf("parse %v: %v", args, err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func TestImportDefaults(t *testing.T) {
|
||||
c := parseImport(t, importArgs()...)
|
||||
if err := c.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Mode != "ignore" || c.Engine != "load" || c.Verify != "pre" || c.Workers != 1 {
|
||||
t.Fatalf("bad defaults: %+v", c)
|
||||
}
|
||||
if c.Charset != "utf8mb4" || c.Progress.Mode != ProgressAuto {
|
||||
t.Fatalf("bad defaults: %+v", c)
|
||||
}
|
||||
if len(c.Selectors) != 1 || c.Selectors[0].Kind != "all" {
|
||||
t.Fatalf("default selector = %+v", c.Selectors)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportRequired(t *testing.T) {
|
||||
c := parseImport(t, "--dest", "u:p@tcp(h:3306)/db")
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--file") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
c = parseImport(t, "--file", "archive")
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--dest") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
c = parseImport(t, "--file", "archive", "--dest", "u:p@tcp(h:3306)/")
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "database name") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportRejectsExportFlags(t *testing.T) {
|
||||
for _, arg := range []string{"--source", "--where", "--purge", "--table", "--format", "--replace"} {
|
||||
_, err := ParseImport(append(importArgs(), arg, "x"), &bytes.Buffer{})
|
||||
if err == nil {
|
||||
t.Fatalf("%s: expected an error", arg)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "archive (export) mode") {
|
||||
t.Fatalf("%s: got %v", arg, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportRejectsUnknownFlag(t *testing.T) {
|
||||
if _, err := ParseImport(append(importArgs(), "--bogus", "1"), &bytes.Buffer{}); err == nil {
|
||||
t.Fatal("expected unknown flag error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportInvalidValues(t *testing.T) {
|
||||
cases := []struct {
|
||||
extra []string
|
||||
want string
|
||||
}{
|
||||
{[]string{"--mode", "merge"}, "--mode"},
|
||||
{[]string{"--engine", "bulk"}, "--engine"},
|
||||
{[]string{"--verify", "after"}, "--verify"},
|
||||
{[]string{"--workers", "0"}, "--workers"},
|
||||
{[]string{"--charset", "utf8; drop"}, "--charset"},
|
||||
{[]string{"--select", "seq:5-2"}, "--select"},
|
||||
{[]string{"--select", "date:2025-01-31..2025-01-01"}, "--select"},
|
||||
{[]string{"--select", "bogus:1"}, "--select"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
c := parseImport(t, importArgs(tc.extra...)...)
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("%v: got %v", tc.extra, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportMappingConflict(t *testing.T) {
|
||||
c := parseImport(t, importArgs("--columns", "id,title", "--assume-all-columns")...)
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "mutually exclusive") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportSelectAllCannotCombine(t *testing.T) {
|
||||
c := parseImport(t, importArgs("--select", "all", "--select", "seq:1-2")...)
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "cannot be combined") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportRepeatedSelect(t *testing.T) {
|
||||
c := parseImport(t, importArgs("--select", "seq:1-2", "--select", "pk:10-20", "--select", "date:2025-01-01..2025-01-02")...)
|
||||
if err := c.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(c.Selectors) != 3 || c.Selectors[0].Kind != "seq" || c.Selectors[1].Kind != "pk" || c.Selectors[2].Kind != "date" {
|
||||
t.Fatalf("selectors = %+v", c.Selectors)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportUnexpectedPositional(t *testing.T) {
|
||||
if _, err := ParseImport(append(importArgs(), "stray"), &bytes.Buffer{}); err == nil {
|
||||
t.Fatal("expected an error for a positional argument")
|
||||
}
|
||||
}
|
||||
@@ -67,9 +67,6 @@ func (c *Config) Validate() error {
|
||||
if c.BulkDelete && !c.Purge {
|
||||
return cfgErr("--bulk-delete requires --purge")
|
||||
}
|
||||
if c.BulkDelete && c.ShardBy == "date" {
|
||||
return cfgErr("--bulk-delete cannot be used with --shard-by date; delete by IN list instead")
|
||||
}
|
||||
|
||||
switch c.Format {
|
||||
case "dump", "csv", "jsonl":
|
||||
|
||||
Reference in new issue
Block a user