Initial commit: archer, a Go-native MySQL/MariaDB table archiver

Renamed from arcdb: module path, cmd/archer, internal/archererr,
docs, build scripts and ARCHER_* environment variables.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-06 22:12:17 +08:00
commit bc4a6b504f
99 files changed
+12752

No files matched your search

+352
View File
@@ -0,0 +1,352 @@
package cli
import (
"bytes"
"strings"
"testing"
"time"
"github.com/go-sql-driver/mysql"
)
func baseArgs() []string {
return []string{
"--source", "u:p@tcp(127.0.0.1:3306)/db",
"--table", "t",
"--where", "id > 0",
"--file", "out",
}
}
func TestDefaults(t *testing.T) {
c, err := Parse(baseArgs(), &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
if c.ChunkSize != 2000 || c.ShardBy != "none" || c.Format != "dump" || c.Compress != "none" {
t.Fatalf("bad defaults: %+v", c)
}
if c.MaxLag.String() != "1s" || c.CheckInterval.String() != "1s" || c.Retries != 1 {
t.Fatalf("bad defaults: %+v", c)
}
if !c.SafeAutoIncrement {
t.Fatal("safe auto increment must default true")
}
if err := c.Validate(); err != nil {
t.Fatal(err)
}
}
func TestParseLimitAlias(t *testing.T) {
args := append(baseArgs(), "--limit", "500")
c, err := Parse(args, &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
if c.ChunkSize != 500 {
t.Fatalf("got %d", c.ChunkSize)
}
}
func TestParseInsertModesMutuallyExclusive(t *testing.T) {
args := append(baseArgs(), "--insert-ignore", "--replace")
if _, err := Parse(args, &bytes.Buffer{}); err == nil {
t.Fatal("expected error")
}
}
func TestValidateRequired(t *testing.T) {
c, _ := Parse([]string{"--source", "u:p@tcp(h:3306)/db", "--table", "t", "--file", "out"}, &bytes.Buffer{})
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--where") {
t.Fatalf("got %v", err)
}
}
func TestValidateNeedsTarget(t *testing.T) {
c, _ := Parse([]string{"--source", "u:p@tcp(h:3306)/db", "--table", "t", "--where", "1=1"}, &bytes.Buffer{})
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "output target") {
t.Fatalf("got %v", err)
}
}
func TestValidateShardRules(t *testing.T) {
args := append(baseArgs(), "--shard-by", "count", "--shard-column", "created_at", "--shard-rows", "10")
c, err := Parse(args, &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--shard-column") {
t.Fatalf("got %v", err)
}
args = append(baseArgs(), "--shard-by", "count")
c, _ = Parse(args, &bytes.Buffer{})
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--shard-rows") {
t.Fatalf("got %v", err)
}
args = append(baseArgs(), "--shard-by", "date", "--shard-column", "c", "--bulk-delete", "--purge")
c, _ = Parse(args, &bytes.Buffer{})
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "bulk-delete") {
t.Fatalf("got %v", err)
}
}
func TestValidateBulkDeleteNeedsPurge(t *testing.T) {
args := append(baseArgs(), "--bulk-delete")
c, _ := Parse(args, &bytes.Buffer{})
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "requires --purge") {
t.Fatalf("got %v", err)
}
}
func TestFingerprintExcludesPassword(t *testing.T) {
a, _ := Parse([]string{"--source", "u:one@tcp(h:3306)/db", "--table", "t", "--where", "1=1", "--file", "out"}, &bytes.Buffer{})
b, _ := Parse([]string{"--source", "u:two@tcp(h:3306)/db", "--table", "t", "--where", "1=1", "--file", "out"}, &bytes.Buffer{})
ha, fa, err := a.Fingerprint()
if err != nil {
t.Fatal(err)
}
hb, fb, err := b.Fingerprint()
if err != nil {
t.Fatal(err)
}
if ha != hb {
t.Fatal("password must not affect the fingerprint")
}
if fa["source"] != fb["source"] {
t.Fatalf("password must not affect the source field: %q vs %q", fa["source"], fb["source"])
}
}
func fingerprintFor(t *testing.T, dsn string) (string, map[string]string) {
t.Helper()
c, err := Parse([]string{"--source", dsn, "--table", "t", "--where", "id > 0", "--file", "out"}, &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
h, f, err := c.Fingerprint()
if err != nil {
t.Fatal(err)
}
return h, f
}
func TestFingerprintTracksConnectionCharset(t *testing.T) {
hBase, _ := fingerprintFor(t, "u:p@tcp(h:3306)/db")
hCharset, fCharset := fingerprintFor(t, "u:p@tcp(h:3306)/db?charset=utf8mb4")
if hBase == hCharset {
t.Fatal("charset must affect the fingerprint")
}
if !strings.Contains(fCharset["source"], "charset=utf8mb4") {
t.Fatalf("source field missing charset: %q", fCharset["source"])
}
hCollation, _ := fingerprintFor(t, "u:p@tcp(h:3306)/db?charset=utf8mb4&collation=utf8mb4_bin")
if hCharset == hCollation {
t.Fatal("collation must affect the fingerprint")
}
}
func TestDsnFingerprintRedactsPassword(t *testing.T) {
c, err := mysql.ParseDSN("u:s3cr3t@tcp(h:3306)/db")
if err != nil {
t.Fatal(err)
}
got := dsnFingerprint(c)
if got != "u@tcp(h:3306)/db" {
t.Fatalf("got %q", got)
}
if strings.Contains(got, "s3cr3t") {
t.Fatal("password leaked into the fingerprint")
}
if c.Passwd != "s3cr3t" {
t.Fatal("dsnFingerprint must not modify the input config")
}
if dsnFingerprint(nil) != "" {
t.Fatal("nil config must yield an empty fragment")
}
}
func TestFingerprintDestEmptyAndNoPassword(t *testing.T) {
h, f := fingerprintFor(t, "u:s3cr3t@tcp(h:3306)/db")
if f["dest"] != "" {
t.Fatalf("dest must be empty without --dest: %q", f["dest"])
}
if f["source"] != "u@tcp(h:3306)/db" {
t.Fatalf("source = %q", f["source"])
}
if strings.Contains(h, "s3cr3t") || strings.Contains(f["source"], "s3cr3t") {
t.Fatal("password leaked into the fingerprint")
}
}
func TestFingerprintDetectsTimezoneChange(t *testing.T) {
a, _ := Parse(append(baseArgs(), "--shard-by", "date", "--shard-column", "c", "--shard-time-zone", "UTC"), &bytes.Buffer{})
b, _ := Parse(append(baseArgs(), "--shard-by", "date", "--shard-column", "c", "--shard-time-zone", "Asia/Shanghai"), &bytes.Buffer{})
ha, fa, _ := a.Fingerprint()
hb, _, _ := b.Fingerprint()
if ha == hb {
t.Fatal("time zone must affect the fingerprint")
}
_ = fa
}
func TestDiffFields(t *testing.T) {
old := map[string]string{"a": "1", "b": "2"}
cur := map[string]string{"a": "1", "b": "3", "c": "4"}
d := DiffFields(old, cur)
if len(d) != 2 {
t.Fatalf("got %v", d)
}
}
func TestValidateCompressLevel(t *testing.T) {
for _, level := range []string{"0", "10"} {
c, err := Parse(append(baseArgs(), "--compress", "gzip", "--compress-level", level), &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "between 1 and 9") {
t.Fatalf("level %s: got %v", level, err)
}
}
c, _ := Parse(append(baseArgs(), "--compress", "none", "--compress-level", "5"), &bytes.Buffer{})
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "only valid when compression is enabled") {
t.Fatalf("none + level 5: got %v", err)
}
c, _ = Parse(append(baseArgs(), "--compress", "none", "--compress-level", "3"), &bytes.Buffer{})
if err := c.Validate(); err != nil {
t.Fatalf("none + default level must be accepted: %v", err)
}
for _, name := range []string{"gzip", "zstd", "xz"} {
c, _ = Parse(append(baseArgs(), "--compress", name, "--compress-level", "9"), &bytes.Buffer{})
if err := c.Validate(); err != nil {
t.Fatalf("%s level 9: %v", name, err)
}
}
}
func TestProgressModes(t *testing.T) {
cases := []struct {
arg string
mode ProgressMode
rows int64
valid bool
}{
{"", ProgressAuto, 0, true},
{"auto", ProgressAuto, 0, true},
{"on", ProgressOn, 0, true},
{"off", ProgressOff, 0, true},
{"0", ProgressOff, 0, true},
{"100000", ProgressRows, 100000, true},
{"1", ProgressRows, 1, true},
{"abc", 0, 0, false},
{"-1", 0, 0, false},
}
for _, tc := range cases {
args := baseArgs()
if tc.arg != "" {
args = append(args, "--progress", tc.arg)
}
c, err := Parse(args, &bytes.Buffer{})
if err != nil {
t.Fatalf("%q: parse: %v", tc.arg, err)
}
err = c.Validate()
if !tc.valid {
if err == nil {
t.Fatalf("%q: expected a configuration error", tc.arg)
}
continue
}
if err != nil {
t.Fatalf("%q: validate: %v", tc.arg, err)
}
if c.Progress.Mode != tc.mode || c.Progress.Rows != tc.rows {
t.Fatalf("%q: got mode=%d rows=%d", tc.arg, c.Progress.Mode, c.Progress.Rows)
}
}
c, _ := Parse(baseArgs(), &bytes.Buffer{})
if c.Progress.Mode != ProgressAuto {
t.Fatalf("default progress mode = %d, want auto", c.Progress.Mode)
}
if c.ProgressInterval != time.Second {
t.Fatalf("default progress interval = %s", c.ProgressInterval)
}
}
func TestProgressEnabledDependsOnTTY(t *testing.T) {
on, _ := Parse(append(baseArgs(), "--progress", "on"), &bytes.Buffer{})
off, _ := Parse(append(baseArgs(), "--progress", "off"), &bytes.Buffer{})
auto, _ := Parse(baseArgs(), &bytes.Buffer{})
for _, c := range []*Config{on, off, auto} {
if err := c.Validate(); err != nil {
t.Fatal(err)
}
}
if !on.Progress.Enabled(false) || !on.Progress.Enabled(true) {
t.Fatal("on must always be enabled")
}
if off.Progress.Enabled(true) || off.Progress.Enabled(false) {
t.Fatal("off must never be enabled")
}
if !auto.Progress.Enabled(true) {
t.Fatal("auto must be enabled on a terminal")
}
if auto.Progress.Enabled(false) {
t.Fatal("auto must be silent without a terminal")
}
}
func TestProgressIntervalValidation(t *testing.T) {
c, err := Parse(append(baseArgs(), "--progress-interval", "0s"), &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--progress-interval") {
t.Fatalf("got %v", err)
}
}
func TestFingerprintCompressLevel(t *testing.T) {
implicit, _ := Parse(append(baseArgs(), "--compress", "gzip"), &bytes.Buffer{})
explicit, _ := Parse(append(baseArgs(), "--compress", "gzip", "--compress-level", "3"), &bytes.Buffer{})
hImplicit, fImplicit, err := implicit.Fingerprint()
if err != nil {
t.Fatal(err)
}
hExplicit, _, err := explicit.Fingerprint()
if err != nil {
t.Fatal(err)
}
if hImplicit != hExplicit {
t.Fatal("the default level must not change the fingerprint")
}
if _, ok := fImplicit["compress_level"]; ok {
t.Fatal("the default level must not be fingerprinted")
}
five, _ := Parse(append(baseArgs(), "--compress", "gzip", "--compress-level", "5"), &bytes.Buffer{})
hFive, fFive, err := five.Fingerprint()
if err != nil {
t.Fatal(err)
}
if hFive == hImplicit {
t.Fatal("a non-default level must change the fingerprint")
}
if fFive["compress_level"] != "5" {
t.Fatalf("compress_level = %q", fFive["compress_level"])
}
if diffs := DiffFields(fImplicit, fFive); len(diffs) != 1 || !strings.Contains(diffs[0], "compress_level") {
t.Fatalf("raising the level: %v", diffs)
}
if diffs := DiffFields(fFive, fImplicit); len(diffs) != 1 || !strings.Contains(diffs[0], "compress_level") {
t.Fatalf("lowering the level back to default: %v", diffs)
}
}
+112
View File
@@ -0,0 +1,112 @@
// Package cli parses and validates archer command-line configuration.
package cli
import (
"fmt"
"time"
"github.com/go-sql-driver/mysql"
)
// Config is the fully resolved runtime configuration.
type Config struct {
SourceDSN string
DestDSN string
Table string
DestTable string
Where string
Columns []string
FileDir string
Purge bool
BulkDelete bool
InsertMode string // "", "ignore", "replace"
ChunkSize int
ShardBy string // none|count|date
ShardRows int
ShardColumn string
ShardColumnType string
ShardTimeZone string
ZeroDate string
Format string
Compress string
CompressLevel int
CSVNull string
MaxLag time.Duration
CheckInterval time.Duration
CheckReplicaLag string
Retries int
SafeAutoIncrement bool
Progress ProgressValue
ProgressInterval time.Duration
Statistics bool
DryRun bool
AllowFullScan bool
ForceResume bool
Index string
Version bool
}
// Defaults returns a Config with documented default values applied.
func Defaults() *Config {
return &Config{
ChunkSize: 2000,
ShardBy: "none",
ShardRows: 0,
ShardTimeZone: "UTC",
Format: "dump",
Compress: "none",
CompressLevel: 3,
CSVNull: `\N`,
MaxLag: time.Second,
CheckInterval: time.Second,
Retries: 1,
SafeAutoIncrement: true,
Progress: ProgressValue{Mode: ProgressAuto},
ProgressInterval: time.Second,
}
}
// SourceConfig parses and normalizes the source DSN, forcing parseTime off so
// values arrive as raw bytes.
func (c *Config) SourceConfig() (*mysql.Config, error) {
return normalize(c.SourceDSN)
}
// DestConfig parses and normalizes the dest DSN.
func (c *Config) DestConfig() (*mysql.Config, error) {
if c.DestDSN == "" {
return nil, nil
}
return normalize(c.DestDSN)
}
func normalize(dsn string) (*mysql.Config, error) {
cfg, err := mysql.ParseDSN(dsn)
if err != nil {
return nil, fmt.Errorf("invalid DSN: %w", err)
}
cfg.ParseTime = false
if cfg.Loc == nil {
cfg.Loc = time.UTC
}
return cfg, nil
}
// SameInstance reports whether two normalized DSNs point at the same MySQL
// server, which determines whether dest writes and purges share a transaction.
func SameInstance(src, dst *mysql.Config) bool {
if src == nil || dst == nil {
return false
}
return src.Net == dst.Net && src.Addr == dst.Addr
}
// ShardMode helpers.
func (c *Config) IsDateShard() bool { return c.ShardBy == "date" }
func (c *Config) IsCountShard() bool { return c.ShardBy == "count" }
func (c *Config) HasFile() bool { return c.FileDir != "" }
+102
View File
@@ -0,0 +1,102 @@
package cli
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strconv"
"strings"
"github.com/go-sql-driver/mysql"
)
// Fingerprint returns a stable hash of every configuration input that can
// change scan or shard semantics, plus the normalized fields used to build it.
// The source and dest fields are standard MySQL DSNs that include every
// connection parameter the driver parsed (charset, collation, timeout, ...),
// so a resume notices any of them changing. The password is the one exclusion:
// rotating it must not break resume.
func (c *Config) Fingerprint() (string, map[string]string, error) {
src, err := c.SourceConfig()
if err != nil {
return "", nil, fmt.Errorf("source DSN: %w", err)
}
dst, err := c.DestConfig()
if err != nil {
return "", nil, fmt.Errorf("dest DSN: %w", err)
}
fields := map[string]string{
"source": dsnFingerprint(src),
"dest": dsnFingerprint(dst),
"table": c.Table,
"dest_table": c.DestTable,
"where": c.Where,
"columns": strings.Join(c.Columns, ","),
"purge": strconv.FormatBool(c.Purge),
"bulk_delete": strconv.FormatBool(c.BulkDelete),
"insert_mode": c.InsertMode,
"shard_by": c.ShardBy,
"shard_rows": strconv.Itoa(c.ShardRows),
"shard_column": c.ShardColumn,
"shard_column_type": c.ShardColumnType,
"shard_time_zone": c.ShardTimeZone,
"zero_date": c.ZeroDate,
"format": c.Format,
"compress": c.Compress,
"csv_null": c.CSVNull,
"safe_auto_increment": strconv.FormatBool(c.SafeAutoIncrement),
}
// The level only changes the output when compression is on, and omitting
// the default keeps fingerprints of tasks started before it existed
// byte-identical so they can resume.
if c.Compress != "" && c.Compress != "none" && c.CompressLevel != 3 {
fields["compress_level"] = strconv.Itoa(c.CompressLevel)
}
keys := make([]string, 0, len(fields))
for k := range fields {
keys = append(keys, k)
}
sort.Strings(keys)
var b strings.Builder
for _, k := range keys {
b.WriteString(k)
b.WriteByte('=')
b.WriteString(fields[k])
b.WriteByte('\n')
}
sum := sha256.Sum256([]byte(b.String()))
return hex.EncodeToString(sum[:]), fields, nil
}
// dsnFingerprint renders c as a standard MySQL DSN covering every parameter
// the driver parsed, except the password. Copying by value keeps the driver's
// unexported fields (charsets, timeTruncate, ...) intact, which is the only way
// FormatDSN can serialize them.
func dsnFingerprint(c *mysql.Config) string {
if c == nil {
return ""
}
redacted := *c
redacted.Passwd = ""
return redacted.FormatDSN()
}
// DiffFields lists keys whose values differ between two fingerprints.
func DiffFields(old, cur map[string]string) []string {
seen := map[string]bool{}
var out []string
for k, v := range cur {
seen[k] = true
if old[k] != v {
out = append(out, fmt.Sprintf("%s: %q -> %q", k, old[k], v))
}
}
for k, v := range old {
if !seen[k] {
out = append(out, fmt.Sprintf("%s: %q -> (absent)", k, v))
}
}
sort.Strings(out)
return out
}
+102
View File
@@ -0,0 +1,102 @@
package cli
import (
"flag"
"fmt"
"io"
"strings"
)
// Parse parses args (without the program name) into a Config. It does not
// validate combinations; call Validate for that.
func Parse(args []string, errOut io.Writer) (*Config, error) {
c := Defaults()
var (
columnsArg string
limitArg = -1
insertIgnoreArg bool
replaceArg bool
)
fs := flag.NewFlagSet("archer", flag.ContinueOnError)
fs.SetOutput(errOut)
fs.StringVar(&c.SourceDSN, "source", "", "source MySQL DSN (required)")
fs.StringVar(&c.DestDSN, "dest", "", "destination MySQL DSN")
fs.StringVar(&c.Table, "table", "", "source table (required)")
fs.StringVar(&c.DestTable, "dest-table", "", "destination table (defaults to --table)")
fs.StringVar(&c.Where, "where", "", "row filter for the source table (required)")
fs.StringVar(&columnsArg, "columns", "", "comma separated columns to export (default: all)")
fs.StringVar(&c.FileDir, "file", "", "output directory for shard files")
fs.BoolVar(&c.Purge, "purge", false, "delete archived rows from the source")
fs.BoolVar(&c.BulkDelete, "bulk-delete", false, "delete by key range instead of IN list (requires --purge)")
fs.BoolVar(&insertIgnoreArg, "insert-ignore", false, "use INSERT IGNORE for --dest")
fs.BoolVar(&replaceArg, "replace", false, "use REPLACE for --dest")
fs.IntVar(&c.ChunkSize, "chunk-size", c.ChunkSize, "rows per SELECT / transaction / checkpoint")
fs.IntVar(&limitArg, "limit", -1, "deprecated alias for --chunk-size")
fs.StringVar(&c.ShardBy, "shard-by", c.ShardBy, "shard mode: none|count|date")
fs.IntVar(&c.ShardRows, "shard-rows", c.ShardRows, "rows per shard file (count/date modes)")
fs.StringVar(&c.ShardColumn, "shard-column", "", "date or epoch column used for date sharding")
fs.StringVar(&c.ShardColumnType, "shard-column-type", "", "epoch unit: epoch-seconds|epoch-milliseconds|epoch-microseconds")
fs.StringVar(&c.ShardTimeZone, "shard-time-zone", c.ShardTimeZone, "time zone for epoch shard values")
fs.StringVar(&c.ZeroDate, "zero-date", "", "zero date handling: bucket (default: error)")
fs.StringVar(&c.Format, "format", c.Format, "output format: dump|csv|jsonl")
fs.StringVar(&c.Compress, "compress", c.Compress, "compression: none|gzip|zstd|xz")
fs.IntVar(&c.CompressLevel, "compress-level", c.CompressLevel, "compression effort 1-9 (higher is smaller and slower)")
fs.StringVar(&c.CSVNull, "csv-null", c.CSVNull, "string used for NULL in csv output")
fs.DurationVar(&c.MaxLag, "max-lag", c.MaxLag, "maximum tolerated replica lag")
fs.DurationVar(&c.CheckInterval, "check-interval", c.CheckInterval, "wait between replica lag checks")
fs.StringVar(&c.CheckReplicaLag, "check-replica-lag", "", "DSN of a replica whose lag is checked before each fetch")
fs.IntVar(&c.Retries, "retries", c.Retries, "retries for cross-instance commits")
fs.BoolVar(&c.SafeAutoIncrement, "safe-auto-increment", c.SafeAutoIncrement, "never purge the maximum auto-increment row")
fs.Var(&c.Progress, "progress", "progress output: auto|on|off|N (default auto; N: also every N rows)")
fs.DurationVar(&c.ProgressInterval, "progress-interval", c.ProgressInterval, "minimum interval between progress updates")
fs.BoolVar(&c.Statistics, "statistics", false, "print a summary when finished")
fs.BoolVar(&c.DryRun, "dry-run", false, "print resolved config and SQL without writing")
fs.BoolVar(&c.AllowFullScan, "allow-full-scan", false, "allow date sharding without a usable index")
fs.BoolVar(&c.ForceResume, "force-resume", false, "resume even when the config fingerprint changed")
fs.StringVar(&c.Index, "index", "", "force a unique index for the scan key")
fs.BoolVar(&c.Version, "version", false, "print version and exit")
if err := fs.Parse(args); err != nil {
return nil, err
}
if c.Version {
return c, nil
}
set := map[string]bool{}
fs.Visit(func(f *flag.Flag) { set[f.Name] = true })
if set["limit"] {
if set["chunk-size"] {
return nil, fmt.Errorf("--limit is a deprecated alias of --chunk-size; pass only one")
}
c.ChunkSize = limitArg
}
if columnsArg != "" {
for _, part := range strings.Split(columnsArg, ",") {
part = strings.TrimSpace(part)
if part != "" {
c.Columns = append(c.Columns, part)
}
}
}
if insertIgnoreArg && replaceArg {
return nil, fmt.Errorf("--insert-ignore and --replace are mutually exclusive")
}
switch {
case insertIgnoreArg:
c.InsertMode = "ignore"
case replaceArg:
c.InsertMode = "replace"
}
if c.DestTable == "" {
c.DestTable = c.Table
}
return c, nil
}
+84
View File
@@ -0,0 +1,84 @@
package cli
import (
"fmt"
"strconv"
"strings"
)
// ProgressMode is the resolved meaning of --progress.
type ProgressMode int
const (
ProgressAuto ProgressMode = iota // live on a terminal, silent otherwise
ProgressOff
ProgressOn
ProgressRows // enabled, plus a row-count threshold
)
// ProgressValue is the flag.Value behind --progress. It keeps the raw
// argument so invalid values are reported as configuration errors by Validate
// rather than as bare flag parse errors.
type ProgressValue struct {
Mode ProgressMode
Rows int64
raw string
}
// Set records the raw argument; resolve happens in Validate.
func (p *ProgressValue) Set(s string) error {
p.raw = s
return nil
}
func (p *ProgressValue) String() string {
if p.raw != "" {
return p.raw
}
switch p.Mode {
case ProgressOn:
return "on"
case ProgressOff:
return "off"
case ProgressRows:
return strconv.FormatInt(p.Rows, 10)
default:
return "auto"
}
}
// resolve parses the recorded argument into Mode and Rows.
func (p *ProgressValue) resolve() error {
switch strings.ToLower(strings.TrimSpace(p.raw)) {
case "", "auto":
p.Mode, p.Rows = ProgressAuto, 0
case "on":
p.Mode, p.Rows = ProgressOn, 0
case "off":
p.Mode, p.Rows = ProgressOff, 0
default:
n, err := strconv.ParseInt(strings.TrimSpace(p.raw), 10, 64)
if err != nil || n < 0 {
return fmt.Errorf("--progress must be auto, on, off or a non-negative integer, got %q", p.raw)
}
if n == 0 {
p.Mode, p.Rows = ProgressOff, 0
} else {
p.Mode, p.Rows = ProgressRows, n
}
}
return nil
}
// Enabled reports whether progress output should be produced. tty describes
// whether stderr is a terminal.
func (p ProgressValue) Enabled(tty bool) bool {
switch p.Mode {
case ProgressOn, ProgressRows:
return true
case ProgressAuto:
return tty
default:
return false
}
}
+119
View File
@@ -0,0 +1,119 @@
package cli
import (
"strings"
"time"
"archer/internal/archererr"
)
// Validate checks required flags and the inter-parameter dependency rules.
// The shard column's existence and type are checked later, after metadata is
// loaded, so the error can list candidate columns.
func (c *Config) Validate() error {
if c.SourceDSN == "" {
return cfgErr("--source is required")
}
if c.Table == "" {
return cfgErr("--table is required")
}
if strings.TrimSpace(c.Where) == "" {
return cfgErr("--where is required")
}
if !c.HasFile() && c.DestDSN == "" && !c.Purge {
return cfgErr("at least one output target is required: --file, --dest or --purge")
}
if c.ChunkSize <= 0 {
return cfgErr("--chunk-size must be greater than 0")
}
if c.Retries < 0 {
return cfgErr("--retries must be 0 or greater")
}
if err := c.Progress.resolve(); err != nil {
return cfgErr("%v", err)
}
if c.ProgressInterval <= 0 {
return cfgErr("--progress-interval must be greater than 0")
}
switch c.ShardBy {
case "none", "count", "date":
default:
return cfgErr("--shard-by must be one of none, count, date")
}
switch c.ShardBy {
case "date":
if c.ShardRows < 0 {
return cfgErr("--shard-rows must be 0 or greater")
}
default:
if c.ShardColumn != "" {
return cfgErr("--shard-column is only valid with --shard-by date")
}
if c.ShardColumnType != "" {
return cfgErr("--shard-column-type is only valid with --shard-by date")
}
if c.ZeroDate != "" {
return cfgErr("--zero-date is only valid with --shard-by date")
}
}
if c.ShardBy == "none" && c.ShardRows != 0 {
return cfgErr("--shard-rows is only valid with --shard-by count or date")
}
if c.ShardBy == "count" && c.ShardRows == 0 {
return cfgErr("--shard-rows must be greater than 0 for --shard-by count")
}
if c.BulkDelete && !c.Purge {
return cfgErr("--bulk-delete requires --purge")
}
if c.BulkDelete && c.ShardBy == "date" {
return cfgErr("--bulk-delete cannot be used with --shard-by date; delete by IN list instead")
}
switch c.Format {
case "dump", "csv", "jsonl":
default:
return cfgErr("--format must be one of dump, csv, jsonl")
}
switch c.Compress {
case "none", "gzip", "zstd", "xz":
default:
return cfgErr("--compress must be one of none, gzip, zstd, xz")
}
if c.CompressLevel < 1 || c.CompressLevel > 9 {
return cfgErr("--compress-level must be between 1 and 9")
}
if c.Compress == "none" && c.CompressLevel != 3 {
return cfgErr("--compress-level is only valid when compression is enabled")
}
switch c.ZeroDate {
case "", "bucket":
default:
return cfgErr("--zero-date must be empty or bucket")
}
if _, err := time.LoadLocation(c.ShardTimeZone); err != nil {
return cfgErr("--shard-time-zone %q is not a valid location: %v", c.ShardTimeZone, err)
}
if _, err := c.SourceConfig(); err != nil {
return cfgErr("--source: %v", err)
}
if _, err := c.DestConfig(); err != nil {
return cfgErr("--dest: %v", err)
}
if c.CheckReplicaLag != "" {
if err := validateDSN(c.CheckReplicaLag); err != nil {
return cfgErr("--check-replica-lag: %v", err)
}
}
return nil
}
func validateDSN(dsn string) error {
_, err := normalize(dsn)
return err
}
func cfgErr(format string, args ...any) error {
return archererr.Newf(archererr.KindConfig, format, args...)
}