Add archer import and allow date-mode bulk delete

archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-08 21:14:23 +08:00
1 parent bc4a6b504f
commit 366d61ca25
49 files changed
+5099 -342

No files matched your search

+2 -2
View File
@@ -87,8 +87,8 @@ func TestValidateShardRules(t *testing.T) {
args = append(baseArgs(), "--shard-by", "date", "--shard-column", "c", "--bulk-delete", "--purge")
c, _ = Parse(args, &bytes.Buffer{})
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "bulk-delete") {
t.Fatalf("got %v", err)
if err := c.Validate(); err != nil {
t.Fatalf("date mode bulk delete should be allowed: %v", err)
}
}
+219
View File
@@ -0,0 +1,219 @@
package cli
import (
"flag"
"fmt"
"io"
"regexp"
"strings"
"github.com/go-sql-driver/mysql"
"archer/internal/shard"
)
// ImportConfig is the resolved configuration of `archer import`. It is kept
// separate from Config so archive (export) validation rules never leak into
// replay and vice versa.
type ImportConfig struct {
FileDir string
DestDSN string
DestTable string
Columns []string
AssumeAll bool
Selects []string
Selectors []shard.Selector
Mode string // ignore|insert|replace
Engine string // load|insert
Verify string // pre|off
Workers int
Charset string
TimeZone string
AllowPartial bool
AllowWarnings bool
DryRun bool
Progress ProgressValue
Statistics bool
Version bool
}
// ImportDefaults returns the documented defaults for `archer import`.
func ImportDefaults() *ImportConfig {
return &ImportConfig{
Mode: "ignore",
Engine: "load",
Verify: "pre",
Workers: 1,
Charset: "utf8mb4",
Progress: ProgressValue{Mode: ProgressAuto},
}
}
// exportOnlyFlags lists archive-mode flags. They are intentionally absent from
// the import flag set, but a dedicated message is friendlier than the flag
// package's generic "flag provided but not defined".
var exportOnlyFlags = []string{
"source", "table", "where", "purge", "bulk-delete", "insert-ignore",
"replace", "chunk-size", "limit", "shard-by", "shard-rows", "shard-column",
"shard-column-type", "shard-time-zone", "zero-date", "format", "compress",
"compress-level", "csv-null", "max-lag", "check-interval",
"check-replica-lag", "retries", "safe-auto-increment", "progress-interval",
"allow-full-scan", "force-resume", "index",
}
var charsetRE = regexp.MustCompile(`^[A-Za-z0-9_]+$`)
type stringList []string
func (s *stringList) String() string { return strings.Join(*s, ",") }
func (s *stringList) Set(v string) error {
*s = append(*s, v)
return nil
}
// ParseImport parses args (without the "import" subcommand) into an
// ImportConfig. Validation is separate.
func ParseImport(args []string, errOut io.Writer) (*ImportConfig, error) {
if err := rejectExportFlags(args); err != nil {
return nil, err
}
c := ImportDefaults()
var (
columnsArg string
selects stringList
)
fs := flag.NewFlagSet("archer import", flag.ContinueOnError)
fs.SetOutput(errOut)
fs.Usage = func() {
fmt.Fprintln(errOut, "usage: archer import --file <archive-dir> --dest <dsn> [flags]")
fmt.Fprintln(errOut, "")
fmt.Fprintln(errOut, "Replay a dump archive into an existing MySQL table.")
fmt.Fprintln(errOut, "")
fmt.Fprintln(errOut, "flags:")
fs.PrintDefaults()
}
fs.StringVar(&c.FileDir, "file", "", "archive directory containing manifest.jsonl (required)")
fs.StringVar(&c.DestDSN, "dest", "", "target MySQL DSN (required, must include a database)")
fs.StringVar(&c.DestTable, "dest-table", "", "target table (defaults to the archived source table)")
fs.StringVar(&columnsArg, "columns", "", "comma separated target columns in file field order")
fs.BoolVar(&c.AssumeAll, "assume-all-columns", false, "accept positional mapping to all target columns, in ordinal order")
fs.Var(&selects, "select", "shard selector (repeatable): all, seq:A-B, pk:LO-HI, date:D1..D2")
fs.StringVar(&c.Mode, "mode", c.Mode, "insert mode: ignore|insert|replace")
fs.StringVar(&c.Engine, "engine", c.Engine, "load engine: load|insert")
fs.StringVar(&c.Verify, "verify", c.Verify, "integrity check: pre|off")
fs.IntVar(&c.Workers, "workers", c.Workers, "parallel shard workers (must be greater than 0)")
fs.StringVar(&c.Charset, "charset", c.Charset, "character set used for loading")
fs.StringVar(&c.TimeZone, "time-zone", "", "target session time zone (empty: server default)")
fs.BoolVar(&c.AllowPartial, "allow-partial", false, "import sealed shards from an archive whose state.json is incomplete")
fs.BoolVar(&c.AllowWarnings, "allow-warnings", false, "do not treat server warnings as failures")
fs.BoolVar(&c.DryRun, "dry-run", false, "print selected shards and configuration without connecting or writing")
fs.Var(&c.Progress, "progress", "progress output: auto|on|off|N")
fs.BoolVar(&c.Statistics, "statistics", false, "print a summary when finished")
fs.BoolVar(&c.Version, "version", false, "print version and exit")
if err := fs.Parse(args); err != nil {
return nil, err
}
if fs.NArg() > 0 {
return nil, fmt.Errorf("unexpected argument %q", fs.Arg(0))
}
if c.Version {
return c, nil
}
if columnsArg != "" {
for _, part := range strings.Split(columnsArg, ",") {
part = strings.TrimSpace(part)
if part != "" {
c.Columns = append(c.Columns, part)
}
}
}
c.Selects = append([]string(nil), selects...)
return c, nil
}
func rejectExportFlags(args []string) error {
exportOnly := make(map[string]bool, len(exportOnlyFlags))
for _, name := range exportOnlyFlags {
exportOnly[name] = true
}
for _, a := range args {
if a == "--" {
break
}
if !strings.HasPrefix(a, "--") {
continue
}
name := strings.TrimPrefix(a, "--")
if i := strings.IndexByte(name, '='); i >= 0 {
name = name[:i]
}
if exportOnly[name] {
return cfgErr("--%s is only valid in archive (export) mode", name)
}
}
return nil
}
// Validate checks required flags and the import-only dependency rules. Archive
// contents (completion, recorded columns, selectors against entries) are
// checked by the importer before any database connection is opened.
func (c *ImportConfig) Validate() error {
if c.FileDir == "" {
return cfgErr("--file is required")
}
if c.DestDSN == "" {
return cfgErr("--dest is required")
}
dest, err := normalize(c.DestDSN)
if err != nil {
return cfgErr("--dest: %v", err)
}
if dest.DBName == "" {
return cfgErr("--dest must include a database name")
}
switch c.Mode {
case "ignore", "insert", "replace":
default:
return cfgErr("--mode must be one of ignore, insert, replace")
}
switch c.Engine {
case "load", "insert":
default:
return cfgErr("--engine must be one of load, insert")
}
switch c.Verify {
case "pre", "off":
default:
return cfgErr("--verify must be pre or off")
}
if c.Workers <= 0 {
return cfgErr("--workers must be greater than 0")
}
if !charsetRE.MatchString(c.Charset) {
return cfgErr("--charset %q is not a valid character set name", c.Charset)
}
if len(c.Columns) > 0 && c.AssumeAll {
return cfgErr("--columns and --assume-all-columns are mutually exclusive")
}
if err := c.Progress.resolve(); err != nil {
return cfgErr("%v", err)
}
sels, err := shard.ParseSelection(c.Selects)
if err != nil {
return cfgErr("%v", err)
}
for _, s := range sels {
if s.Kind == "all" && len(sels) > 1 {
return cfgErr("--select all cannot be combined with other selectors")
}
}
c.Selectors = sels
return nil
}
// DestConfig parses the target DSN, forcing parseTime off like the export path.
func (c *ImportConfig) DestConfig() (*mysql.Config, error) {
return normalize(c.DestDSN)
}
+122
View File
@@ -0,0 +1,122 @@
package cli
import (
"bytes"
"strings"
"testing"
)
func importArgs(extra ...string) []string {
args := []string{"--file", "archive", "--dest", "u:p@tcp(127.0.0.1:3306)/db"}
return append(args, extra...)
}
func parseImport(t *testing.T, args ...string) *ImportConfig {
t.Helper()
c, err := ParseImport(args, &bytes.Buffer{})
if err != nil {
t.Fatalf("parse %v: %v", args, err)
}
return c
}
func TestImportDefaults(t *testing.T) {
c := parseImport(t, importArgs()...)
if err := c.Validate(); err != nil {
t.Fatal(err)
}
if c.Mode != "ignore" || c.Engine != "load" || c.Verify != "pre" || c.Workers != 1 {
t.Fatalf("bad defaults: %+v", c)
}
if c.Charset != "utf8mb4" || c.Progress.Mode != ProgressAuto {
t.Fatalf("bad defaults: %+v", c)
}
if len(c.Selectors) != 1 || c.Selectors[0].Kind != "all" {
t.Fatalf("default selector = %+v", c.Selectors)
}
}
func TestImportRequired(t *testing.T) {
c := parseImport(t, "--dest", "u:p@tcp(h:3306)/db")
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--file") {
t.Fatalf("got %v", err)
}
c = parseImport(t, "--file", "archive")
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--dest") {
t.Fatalf("got %v", err)
}
c = parseImport(t, "--file", "archive", "--dest", "u:p@tcp(h:3306)/")
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "database name") {
t.Fatalf("got %v", err)
}
}
func TestImportRejectsExportFlags(t *testing.T) {
for _, arg := range []string{"--source", "--where", "--purge", "--table", "--format", "--replace"} {
_, err := ParseImport(append(importArgs(), arg, "x"), &bytes.Buffer{})
if err == nil {
t.Fatalf("%s: expected an error", arg)
}
if !strings.Contains(err.Error(), "archive (export) mode") {
t.Fatalf("%s: got %v", arg, err)
}
}
}
func TestImportRejectsUnknownFlag(t *testing.T) {
if _, err := ParseImport(append(importArgs(), "--bogus", "1"), &bytes.Buffer{}); err == nil {
t.Fatal("expected unknown flag error")
}
}
func TestImportInvalidValues(t *testing.T) {
cases := []struct {
extra []string
want string
}{
{[]string{"--mode", "merge"}, "--mode"},
{[]string{"--engine", "bulk"}, "--engine"},
{[]string{"--verify", "after"}, "--verify"},
{[]string{"--workers", "0"}, "--workers"},
{[]string{"--charset", "utf8; drop"}, "--charset"},
{[]string{"--select", "seq:5-2"}, "--select"},
{[]string{"--select", "date:2025-01-31..2025-01-01"}, "--select"},
{[]string{"--select", "bogus:1"}, "--select"},
}
for _, tc := range cases {
c := parseImport(t, importArgs(tc.extra...)...)
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), tc.want) {
t.Fatalf("%v: got %v", tc.extra, err)
}
}
}
func TestImportMappingConflict(t *testing.T) {
c := parseImport(t, importArgs("--columns", "id,title", "--assume-all-columns")...)
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "mutually exclusive") {
t.Fatalf("got %v", err)
}
}
func TestImportSelectAllCannotCombine(t *testing.T) {
c := parseImport(t, importArgs("--select", "all", "--select", "seq:1-2")...)
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "cannot be combined") {
t.Fatalf("got %v", err)
}
}
func TestImportRepeatedSelect(t *testing.T) {
c := parseImport(t, importArgs("--select", "seq:1-2", "--select", "pk:10-20", "--select", "date:2025-01-01..2025-01-02")...)
if err := c.Validate(); err != nil {
t.Fatal(err)
}
if len(c.Selectors) != 3 || c.Selectors[0].Kind != "seq" || c.Selectors[1].Kind != "pk" || c.Selectors[2].Kind != "date" {
t.Fatalf("selectors = %+v", c.Selectors)
}
}
func TestImportUnexpectedPositional(t *testing.T) {
if _, err := ParseImport(append(importArgs(), "stray"), &bytes.Buffer{}); err == nil {
t.Fatal("expected an error for a positional argument")
}
}
-3
View File
@@ -67,9 +67,6 @@ func (c *Config) Validate() error {
if c.BulkDelete && !c.Purge {
return cfgErr("--bulk-delete requires --purge")
}
if c.BulkDelete && c.ShardBy == "date" {
return cfgErr("--bulk-delete cannot be used with --shard-by date; delete by IN list instead")
}
switch c.Format {
case "dump", "csv", "jsonl":