Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,352 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
func baseArgs() []string {
|
||||
return []string{
|
||||
"--source", "u:p@tcp(127.0.0.1:3306)/db",
|
||||
"--table", "t",
|
||||
"--where", "id > 0",
|
||||
"--file", "out",
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaults(t *testing.T) {
|
||||
c, err := Parse(baseArgs(), &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.ChunkSize != 2000 || c.ShardBy != "none" || c.Format != "dump" || c.Compress != "none" {
|
||||
t.Fatalf("bad defaults: %+v", c)
|
||||
}
|
||||
if c.MaxLag.String() != "1s" || c.CheckInterval.String() != "1s" || c.Retries != 1 {
|
||||
t.Fatalf("bad defaults: %+v", c)
|
||||
}
|
||||
if !c.SafeAutoIncrement {
|
||||
t.Fatal("safe auto increment must default true")
|
||||
}
|
||||
if err := c.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLimitAlias(t *testing.T) {
|
||||
args := append(baseArgs(), "--limit", "500")
|
||||
c, err := Parse(args, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.ChunkSize != 500 {
|
||||
t.Fatalf("got %d", c.ChunkSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseInsertModesMutuallyExclusive(t *testing.T) {
|
||||
args := append(baseArgs(), "--insert-ignore", "--replace")
|
||||
if _, err := Parse(args, &bytes.Buffer{}); err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRequired(t *testing.T) {
|
||||
c, _ := Parse([]string{"--source", "u:p@tcp(h:3306)/db", "--table", "t", "--file", "out"}, &bytes.Buffer{})
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--where") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateNeedsTarget(t *testing.T) {
|
||||
c, _ := Parse([]string{"--source", "u:p@tcp(h:3306)/db", "--table", "t", "--where", "1=1"}, &bytes.Buffer{})
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "output target") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateShardRules(t *testing.T) {
|
||||
args := append(baseArgs(), "--shard-by", "count", "--shard-column", "created_at", "--shard-rows", "10")
|
||||
c, err := Parse(args, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--shard-column") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
|
||||
args = append(baseArgs(), "--shard-by", "count")
|
||||
c, _ = Parse(args, &bytes.Buffer{})
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--shard-rows") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
|
||||
args = append(baseArgs(), "--shard-by", "date", "--shard-column", "c", "--bulk-delete", "--purge")
|
||||
c, _ = Parse(args, &bytes.Buffer{})
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "bulk-delete") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateBulkDeleteNeedsPurge(t *testing.T) {
|
||||
args := append(baseArgs(), "--bulk-delete")
|
||||
c, _ := Parse(args, &bytes.Buffer{})
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "requires --purge") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFingerprintExcludesPassword(t *testing.T) {
|
||||
a, _ := Parse([]string{"--source", "u:one@tcp(h:3306)/db", "--table", "t", "--where", "1=1", "--file", "out"}, &bytes.Buffer{})
|
||||
b, _ := Parse([]string{"--source", "u:two@tcp(h:3306)/db", "--table", "t", "--where", "1=1", "--file", "out"}, &bytes.Buffer{})
|
||||
ha, fa, err := a.Fingerprint()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hb, fb, err := b.Fingerprint()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ha != hb {
|
||||
t.Fatal("password must not affect the fingerprint")
|
||||
}
|
||||
if fa["source"] != fb["source"] {
|
||||
t.Fatalf("password must not affect the source field: %q vs %q", fa["source"], fb["source"])
|
||||
}
|
||||
}
|
||||
|
||||
func fingerprintFor(t *testing.T, dsn string) (string, map[string]string) {
|
||||
t.Helper()
|
||||
c, err := Parse([]string{"--source", dsn, "--table", "t", "--where", "id > 0", "--file", "out"}, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h, f, err := c.Fingerprint()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return h, f
|
||||
}
|
||||
|
||||
func TestFingerprintTracksConnectionCharset(t *testing.T) {
|
||||
hBase, _ := fingerprintFor(t, "u:p@tcp(h:3306)/db")
|
||||
hCharset, fCharset := fingerprintFor(t, "u:p@tcp(h:3306)/db?charset=utf8mb4")
|
||||
if hBase == hCharset {
|
||||
t.Fatal("charset must affect the fingerprint")
|
||||
}
|
||||
if !strings.Contains(fCharset["source"], "charset=utf8mb4") {
|
||||
t.Fatalf("source field missing charset: %q", fCharset["source"])
|
||||
}
|
||||
|
||||
hCollation, _ := fingerprintFor(t, "u:p@tcp(h:3306)/db?charset=utf8mb4&collation=utf8mb4_bin")
|
||||
if hCharset == hCollation {
|
||||
t.Fatal("collation must affect the fingerprint")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDsnFingerprintRedactsPassword(t *testing.T) {
|
||||
c, err := mysql.ParseDSN("u:s3cr3t@tcp(h:3306)/db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := dsnFingerprint(c)
|
||||
if got != "u@tcp(h:3306)/db" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if strings.Contains(got, "s3cr3t") {
|
||||
t.Fatal("password leaked into the fingerprint")
|
||||
}
|
||||
if c.Passwd != "s3cr3t" {
|
||||
t.Fatal("dsnFingerprint must not modify the input config")
|
||||
}
|
||||
if dsnFingerprint(nil) != "" {
|
||||
t.Fatal("nil config must yield an empty fragment")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFingerprintDestEmptyAndNoPassword(t *testing.T) {
|
||||
h, f := fingerprintFor(t, "u:s3cr3t@tcp(h:3306)/db")
|
||||
if f["dest"] != "" {
|
||||
t.Fatalf("dest must be empty without --dest: %q", f["dest"])
|
||||
}
|
||||
if f["source"] != "u@tcp(h:3306)/db" {
|
||||
t.Fatalf("source = %q", f["source"])
|
||||
}
|
||||
if strings.Contains(h, "s3cr3t") || strings.Contains(f["source"], "s3cr3t") {
|
||||
t.Fatal("password leaked into the fingerprint")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFingerprintDetectsTimezoneChange(t *testing.T) {
|
||||
a, _ := Parse(append(baseArgs(), "--shard-by", "date", "--shard-column", "c", "--shard-time-zone", "UTC"), &bytes.Buffer{})
|
||||
b, _ := Parse(append(baseArgs(), "--shard-by", "date", "--shard-column", "c", "--shard-time-zone", "Asia/Shanghai"), &bytes.Buffer{})
|
||||
ha, fa, _ := a.Fingerprint()
|
||||
hb, _, _ := b.Fingerprint()
|
||||
if ha == hb {
|
||||
t.Fatal("time zone must affect the fingerprint")
|
||||
}
|
||||
_ = fa
|
||||
}
|
||||
|
||||
func TestDiffFields(t *testing.T) {
|
||||
old := map[string]string{"a": "1", "b": "2"}
|
||||
cur := map[string]string{"a": "1", "b": "3", "c": "4"}
|
||||
d := DiffFields(old, cur)
|
||||
if len(d) != 2 {
|
||||
t.Fatalf("got %v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateCompressLevel(t *testing.T) {
|
||||
for _, level := range []string{"0", "10"} {
|
||||
c, err := Parse(append(baseArgs(), "--compress", "gzip", "--compress-level", level), &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "between 1 and 9") {
|
||||
t.Fatalf("level %s: got %v", level, err)
|
||||
}
|
||||
}
|
||||
|
||||
c, _ := Parse(append(baseArgs(), "--compress", "none", "--compress-level", "5"), &bytes.Buffer{})
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "only valid when compression is enabled") {
|
||||
t.Fatalf("none + level 5: got %v", err)
|
||||
}
|
||||
|
||||
c, _ = Parse(append(baseArgs(), "--compress", "none", "--compress-level", "3"), &bytes.Buffer{})
|
||||
if err := c.Validate(); err != nil {
|
||||
t.Fatalf("none + default level must be accepted: %v", err)
|
||||
}
|
||||
|
||||
for _, name := range []string{"gzip", "zstd", "xz"} {
|
||||
c, _ = Parse(append(baseArgs(), "--compress", name, "--compress-level", "9"), &bytes.Buffer{})
|
||||
if err := c.Validate(); err != nil {
|
||||
t.Fatalf("%s level 9: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressModes(t *testing.T) {
|
||||
cases := []struct {
|
||||
arg string
|
||||
mode ProgressMode
|
||||
rows int64
|
||||
valid bool
|
||||
}{
|
||||
{"", ProgressAuto, 0, true},
|
||||
{"auto", ProgressAuto, 0, true},
|
||||
{"on", ProgressOn, 0, true},
|
||||
{"off", ProgressOff, 0, true},
|
||||
{"0", ProgressOff, 0, true},
|
||||
{"100000", ProgressRows, 100000, true},
|
||||
{"1", ProgressRows, 1, true},
|
||||
{"abc", 0, 0, false},
|
||||
{"-1", 0, 0, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
args := baseArgs()
|
||||
if tc.arg != "" {
|
||||
args = append(args, "--progress", tc.arg)
|
||||
}
|
||||
c, err := Parse(args, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatalf("%q: parse: %v", tc.arg, err)
|
||||
}
|
||||
err = c.Validate()
|
||||
if !tc.valid {
|
||||
if err == nil {
|
||||
t.Fatalf("%q: expected a configuration error", tc.arg)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("%q: validate: %v", tc.arg, err)
|
||||
}
|
||||
if c.Progress.Mode != tc.mode || c.Progress.Rows != tc.rows {
|
||||
t.Fatalf("%q: got mode=%d rows=%d", tc.arg, c.Progress.Mode, c.Progress.Rows)
|
||||
}
|
||||
}
|
||||
|
||||
c, _ := Parse(baseArgs(), &bytes.Buffer{})
|
||||
if c.Progress.Mode != ProgressAuto {
|
||||
t.Fatalf("default progress mode = %d, want auto", c.Progress.Mode)
|
||||
}
|
||||
if c.ProgressInterval != time.Second {
|
||||
t.Fatalf("default progress interval = %s", c.ProgressInterval)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressEnabledDependsOnTTY(t *testing.T) {
|
||||
on, _ := Parse(append(baseArgs(), "--progress", "on"), &bytes.Buffer{})
|
||||
off, _ := Parse(append(baseArgs(), "--progress", "off"), &bytes.Buffer{})
|
||||
auto, _ := Parse(baseArgs(), &bytes.Buffer{})
|
||||
for _, c := range []*Config{on, off, auto} {
|
||||
if err := c.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if !on.Progress.Enabled(false) || !on.Progress.Enabled(true) {
|
||||
t.Fatal("on must always be enabled")
|
||||
}
|
||||
if off.Progress.Enabled(true) || off.Progress.Enabled(false) {
|
||||
t.Fatal("off must never be enabled")
|
||||
}
|
||||
if !auto.Progress.Enabled(true) {
|
||||
t.Fatal("auto must be enabled on a terminal")
|
||||
}
|
||||
if auto.Progress.Enabled(false) {
|
||||
t.Fatal("auto must be silent without a terminal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressIntervalValidation(t *testing.T) {
|
||||
c, err := Parse(append(baseArgs(), "--progress-interval", "0s"), &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "--progress-interval") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFingerprintCompressLevel(t *testing.T) {
|
||||
implicit, _ := Parse(append(baseArgs(), "--compress", "gzip"), &bytes.Buffer{})
|
||||
explicit, _ := Parse(append(baseArgs(), "--compress", "gzip", "--compress-level", "3"), &bytes.Buffer{})
|
||||
hImplicit, fImplicit, err := implicit.Fingerprint()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hExplicit, _, err := explicit.Fingerprint()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hImplicit != hExplicit {
|
||||
t.Fatal("the default level must not change the fingerprint")
|
||||
}
|
||||
if _, ok := fImplicit["compress_level"]; ok {
|
||||
t.Fatal("the default level must not be fingerprinted")
|
||||
}
|
||||
|
||||
five, _ := Parse(append(baseArgs(), "--compress", "gzip", "--compress-level", "5"), &bytes.Buffer{})
|
||||
hFive, fFive, err := five.Fingerprint()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hFive == hImplicit {
|
||||
t.Fatal("a non-default level must change the fingerprint")
|
||||
}
|
||||
if fFive["compress_level"] != "5" {
|
||||
t.Fatalf("compress_level = %q", fFive["compress_level"])
|
||||
}
|
||||
|
||||
if diffs := DiffFields(fImplicit, fFive); len(diffs) != 1 || !strings.Contains(diffs[0], "compress_level") {
|
||||
t.Fatalf("raising the level: %v", diffs)
|
||||
}
|
||||
if diffs := DiffFields(fFive, fImplicit); len(diffs) != 1 || !strings.Contains(diffs[0], "compress_level") {
|
||||
t.Fatalf("lowering the level back to default: %v", diffs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
// Package cli parses and validates archer command-line configuration.
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// Config is the fully resolved runtime configuration.
|
||||
type Config struct {
|
||||
SourceDSN string
|
||||
DestDSN string
|
||||
Table string
|
||||
DestTable string
|
||||
Where string
|
||||
Columns []string
|
||||
FileDir string
|
||||
Purge bool
|
||||
BulkDelete bool
|
||||
InsertMode string // "", "ignore", "replace"
|
||||
|
||||
ChunkSize int
|
||||
|
||||
ShardBy string // none|count|date
|
||||
ShardRows int
|
||||
ShardColumn string
|
||||
ShardColumnType string
|
||||
ShardTimeZone string
|
||||
ZeroDate string
|
||||
|
||||
Format string
|
||||
Compress string
|
||||
CompressLevel int
|
||||
CSVNull string
|
||||
|
||||
MaxLag time.Duration
|
||||
CheckInterval time.Duration
|
||||
CheckReplicaLag string
|
||||
Retries int
|
||||
|
||||
SafeAutoIncrement bool
|
||||
Progress ProgressValue
|
||||
ProgressInterval time.Duration
|
||||
Statistics bool
|
||||
DryRun bool
|
||||
AllowFullScan bool
|
||||
ForceResume bool
|
||||
Index string
|
||||
Version bool
|
||||
}
|
||||
|
||||
// Defaults returns a Config with documented default values applied.
|
||||
func Defaults() *Config {
|
||||
return &Config{
|
||||
ChunkSize: 2000,
|
||||
ShardBy: "none",
|
||||
ShardRows: 0,
|
||||
ShardTimeZone: "UTC",
|
||||
Format: "dump",
|
||||
Compress: "none",
|
||||
CompressLevel: 3,
|
||||
CSVNull: `\N`,
|
||||
MaxLag: time.Second,
|
||||
CheckInterval: time.Second,
|
||||
Retries: 1,
|
||||
SafeAutoIncrement: true,
|
||||
Progress: ProgressValue{Mode: ProgressAuto},
|
||||
ProgressInterval: time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
// SourceConfig parses and normalizes the source DSN, forcing parseTime off so
|
||||
// values arrive as raw bytes.
|
||||
func (c *Config) SourceConfig() (*mysql.Config, error) {
|
||||
return normalize(c.SourceDSN)
|
||||
}
|
||||
|
||||
// DestConfig parses and normalizes the dest DSN.
|
||||
func (c *Config) DestConfig() (*mysql.Config, error) {
|
||||
if c.DestDSN == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return normalize(c.DestDSN)
|
||||
}
|
||||
|
||||
func normalize(dsn string) (*mysql.Config, error) {
|
||||
cfg, err := mysql.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid DSN: %w", err)
|
||||
}
|
||||
cfg.ParseTime = false
|
||||
if cfg.Loc == nil {
|
||||
cfg.Loc = time.UTC
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// SameInstance reports whether two normalized DSNs point at the same MySQL
|
||||
// server, which determines whether dest writes and purges share a transaction.
|
||||
func SameInstance(src, dst *mysql.Config) bool {
|
||||
if src == nil || dst == nil {
|
||||
return false
|
||||
}
|
||||
return src.Net == dst.Net && src.Addr == dst.Addr
|
||||
}
|
||||
|
||||
// ShardMode helpers.
|
||||
func (c *Config) IsDateShard() bool { return c.ShardBy == "date" }
|
||||
func (c *Config) IsCountShard() bool { return c.ShardBy == "count" }
|
||||
func (c *Config) HasFile() bool { return c.FileDir != "" }
|
||||
@@ -0,0 +1,102 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// Fingerprint returns a stable hash of every configuration input that can
|
||||
// change scan or shard semantics, plus the normalized fields used to build it.
|
||||
// The source and dest fields are standard MySQL DSNs that include every
|
||||
// connection parameter the driver parsed (charset, collation, timeout, ...),
|
||||
// so a resume notices any of them changing. The password is the one exclusion:
|
||||
// rotating it must not break resume.
|
||||
func (c *Config) Fingerprint() (string, map[string]string, error) {
|
||||
src, err := c.SourceConfig()
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("source DSN: %w", err)
|
||||
}
|
||||
dst, err := c.DestConfig()
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("dest DSN: %w", err)
|
||||
}
|
||||
fields := map[string]string{
|
||||
"source": dsnFingerprint(src),
|
||||
"dest": dsnFingerprint(dst),
|
||||
"table": c.Table,
|
||||
"dest_table": c.DestTable,
|
||||
"where": c.Where,
|
||||
"columns": strings.Join(c.Columns, ","),
|
||||
"purge": strconv.FormatBool(c.Purge),
|
||||
"bulk_delete": strconv.FormatBool(c.BulkDelete),
|
||||
"insert_mode": c.InsertMode,
|
||||
"shard_by": c.ShardBy,
|
||||
"shard_rows": strconv.Itoa(c.ShardRows),
|
||||
"shard_column": c.ShardColumn,
|
||||
"shard_column_type": c.ShardColumnType,
|
||||
"shard_time_zone": c.ShardTimeZone,
|
||||
"zero_date": c.ZeroDate,
|
||||
"format": c.Format,
|
||||
"compress": c.Compress,
|
||||
"csv_null": c.CSVNull,
|
||||
"safe_auto_increment": strconv.FormatBool(c.SafeAutoIncrement),
|
||||
}
|
||||
// The level only changes the output when compression is on, and omitting
|
||||
// the default keeps fingerprints of tasks started before it existed
|
||||
// byte-identical so they can resume.
|
||||
if c.Compress != "" && c.Compress != "none" && c.CompressLevel != 3 {
|
||||
fields["compress_level"] = strconv.Itoa(c.CompressLevel)
|
||||
}
|
||||
keys := make([]string, 0, len(fields))
|
||||
for k := range fields {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var b strings.Builder
|
||||
for _, k := range keys {
|
||||
b.WriteString(k)
|
||||
b.WriteByte('=')
|
||||
b.WriteString(fields[k])
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
sum := sha256.Sum256([]byte(b.String()))
|
||||
return hex.EncodeToString(sum[:]), fields, nil
|
||||
}
|
||||
|
||||
// dsnFingerprint renders c as a standard MySQL DSN covering every parameter
|
||||
// the driver parsed, except the password. Copying by value keeps the driver's
|
||||
// unexported fields (charsets, timeTruncate, ...) intact, which is the only way
|
||||
// FormatDSN can serialize them.
|
||||
func dsnFingerprint(c *mysql.Config) string {
|
||||
if c == nil {
|
||||
return ""
|
||||
}
|
||||
redacted := *c
|
||||
redacted.Passwd = ""
|
||||
return redacted.FormatDSN()
|
||||
}
|
||||
|
||||
// DiffFields lists keys whose values differ between two fingerprints.
|
||||
func DiffFields(old, cur map[string]string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for k, v := range cur {
|
||||
seen[k] = true
|
||||
if old[k] != v {
|
||||
out = append(out, fmt.Sprintf("%s: %q -> %q", k, old[k], v))
|
||||
}
|
||||
}
|
||||
for k, v := range old {
|
||||
if !seen[k] {
|
||||
out = append(out, fmt.Sprintf("%s: %q -> (absent)", k, v))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Parse parses args (without the program name) into a Config. It does not
|
||||
// validate combinations; call Validate for that.
|
||||
func Parse(args []string, errOut io.Writer) (*Config, error) {
|
||||
c := Defaults()
|
||||
var (
|
||||
columnsArg string
|
||||
limitArg = -1
|
||||
insertIgnoreArg bool
|
||||
replaceArg bool
|
||||
)
|
||||
fs := flag.NewFlagSet("archer", flag.ContinueOnError)
|
||||
fs.SetOutput(errOut)
|
||||
|
||||
fs.StringVar(&c.SourceDSN, "source", "", "source MySQL DSN (required)")
|
||||
fs.StringVar(&c.DestDSN, "dest", "", "destination MySQL DSN")
|
||||
fs.StringVar(&c.Table, "table", "", "source table (required)")
|
||||
fs.StringVar(&c.DestTable, "dest-table", "", "destination table (defaults to --table)")
|
||||
fs.StringVar(&c.Where, "where", "", "row filter for the source table (required)")
|
||||
fs.StringVar(&columnsArg, "columns", "", "comma separated columns to export (default: all)")
|
||||
fs.StringVar(&c.FileDir, "file", "", "output directory for shard files")
|
||||
fs.BoolVar(&c.Purge, "purge", false, "delete archived rows from the source")
|
||||
fs.BoolVar(&c.BulkDelete, "bulk-delete", false, "delete by key range instead of IN list (requires --purge)")
|
||||
fs.BoolVar(&insertIgnoreArg, "insert-ignore", false, "use INSERT IGNORE for --dest")
|
||||
fs.BoolVar(&replaceArg, "replace", false, "use REPLACE for --dest")
|
||||
|
||||
fs.IntVar(&c.ChunkSize, "chunk-size", c.ChunkSize, "rows per SELECT / transaction / checkpoint")
|
||||
fs.IntVar(&limitArg, "limit", -1, "deprecated alias for --chunk-size")
|
||||
|
||||
fs.StringVar(&c.ShardBy, "shard-by", c.ShardBy, "shard mode: none|count|date")
|
||||
fs.IntVar(&c.ShardRows, "shard-rows", c.ShardRows, "rows per shard file (count/date modes)")
|
||||
fs.StringVar(&c.ShardColumn, "shard-column", "", "date or epoch column used for date sharding")
|
||||
fs.StringVar(&c.ShardColumnType, "shard-column-type", "", "epoch unit: epoch-seconds|epoch-milliseconds|epoch-microseconds")
|
||||
fs.StringVar(&c.ShardTimeZone, "shard-time-zone", c.ShardTimeZone, "time zone for epoch shard values")
|
||||
fs.StringVar(&c.ZeroDate, "zero-date", "", "zero date handling: bucket (default: error)")
|
||||
|
||||
fs.StringVar(&c.Format, "format", c.Format, "output format: dump|csv|jsonl")
|
||||
fs.StringVar(&c.Compress, "compress", c.Compress, "compression: none|gzip|zstd|xz")
|
||||
fs.IntVar(&c.CompressLevel, "compress-level", c.CompressLevel, "compression effort 1-9 (higher is smaller and slower)")
|
||||
fs.StringVar(&c.CSVNull, "csv-null", c.CSVNull, "string used for NULL in csv output")
|
||||
|
||||
fs.DurationVar(&c.MaxLag, "max-lag", c.MaxLag, "maximum tolerated replica lag")
|
||||
fs.DurationVar(&c.CheckInterval, "check-interval", c.CheckInterval, "wait between replica lag checks")
|
||||
fs.StringVar(&c.CheckReplicaLag, "check-replica-lag", "", "DSN of a replica whose lag is checked before each fetch")
|
||||
fs.IntVar(&c.Retries, "retries", c.Retries, "retries for cross-instance commits")
|
||||
|
||||
fs.BoolVar(&c.SafeAutoIncrement, "safe-auto-increment", c.SafeAutoIncrement, "never purge the maximum auto-increment row")
|
||||
fs.Var(&c.Progress, "progress", "progress output: auto|on|off|N (default auto; N: also every N rows)")
|
||||
fs.DurationVar(&c.ProgressInterval, "progress-interval", c.ProgressInterval, "minimum interval between progress updates")
|
||||
fs.BoolVar(&c.Statistics, "statistics", false, "print a summary when finished")
|
||||
fs.BoolVar(&c.DryRun, "dry-run", false, "print resolved config and SQL without writing")
|
||||
fs.BoolVar(&c.AllowFullScan, "allow-full-scan", false, "allow date sharding without a usable index")
|
||||
fs.BoolVar(&c.ForceResume, "force-resume", false, "resume even when the config fingerprint changed")
|
||||
fs.StringVar(&c.Index, "index", "", "force a unique index for the scan key")
|
||||
fs.BoolVar(&c.Version, "version", false, "print version and exit")
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.Version {
|
||||
return c, nil
|
||||
}
|
||||
|
||||
set := map[string]bool{}
|
||||
fs.Visit(func(f *flag.Flag) { set[f.Name] = true })
|
||||
|
||||
if set["limit"] {
|
||||
if set["chunk-size"] {
|
||||
return nil, fmt.Errorf("--limit is a deprecated alias of --chunk-size; pass only one")
|
||||
}
|
||||
c.ChunkSize = limitArg
|
||||
}
|
||||
if columnsArg != "" {
|
||||
for _, part := range strings.Split(columnsArg, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
c.Columns = append(c.Columns, part)
|
||||
}
|
||||
}
|
||||
}
|
||||
if insertIgnoreArg && replaceArg {
|
||||
return nil, fmt.Errorf("--insert-ignore and --replace are mutually exclusive")
|
||||
}
|
||||
switch {
|
||||
case insertIgnoreArg:
|
||||
c.InsertMode = "ignore"
|
||||
case replaceArg:
|
||||
c.InsertMode = "replace"
|
||||
}
|
||||
if c.DestTable == "" {
|
||||
c.DestTable = c.Table
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ProgressMode is the resolved meaning of --progress.
|
||||
type ProgressMode int
|
||||
|
||||
const (
|
||||
ProgressAuto ProgressMode = iota // live on a terminal, silent otherwise
|
||||
ProgressOff
|
||||
ProgressOn
|
||||
ProgressRows // enabled, plus a row-count threshold
|
||||
)
|
||||
|
||||
// ProgressValue is the flag.Value behind --progress. It keeps the raw
|
||||
// argument so invalid values are reported as configuration errors by Validate
|
||||
// rather than as bare flag parse errors.
|
||||
type ProgressValue struct {
|
||||
Mode ProgressMode
|
||||
Rows int64
|
||||
raw string
|
||||
}
|
||||
|
||||
// Set records the raw argument; resolve happens in Validate.
|
||||
func (p *ProgressValue) Set(s string) error {
|
||||
p.raw = s
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *ProgressValue) String() string {
|
||||
if p.raw != "" {
|
||||
return p.raw
|
||||
}
|
||||
switch p.Mode {
|
||||
case ProgressOn:
|
||||
return "on"
|
||||
case ProgressOff:
|
||||
return "off"
|
||||
case ProgressRows:
|
||||
return strconv.FormatInt(p.Rows, 10)
|
||||
default:
|
||||
return "auto"
|
||||
}
|
||||
}
|
||||
|
||||
// resolve parses the recorded argument into Mode and Rows.
|
||||
func (p *ProgressValue) resolve() error {
|
||||
switch strings.ToLower(strings.TrimSpace(p.raw)) {
|
||||
case "", "auto":
|
||||
p.Mode, p.Rows = ProgressAuto, 0
|
||||
case "on":
|
||||
p.Mode, p.Rows = ProgressOn, 0
|
||||
case "off":
|
||||
p.Mode, p.Rows = ProgressOff, 0
|
||||
default:
|
||||
n, err := strconv.ParseInt(strings.TrimSpace(p.raw), 10, 64)
|
||||
if err != nil || n < 0 {
|
||||
return fmt.Errorf("--progress must be auto, on, off or a non-negative integer, got %q", p.raw)
|
||||
}
|
||||
if n == 0 {
|
||||
p.Mode, p.Rows = ProgressOff, 0
|
||||
} else {
|
||||
p.Mode, p.Rows = ProgressRows, n
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Enabled reports whether progress output should be produced. tty describes
|
||||
// whether stderr is a terminal.
|
||||
func (p ProgressValue) Enabled(tty bool) bool {
|
||||
switch p.Mode {
|
||||
case ProgressOn, ProgressRows:
|
||||
return true
|
||||
case ProgressAuto:
|
||||
return tty
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archer/internal/archererr"
|
||||
)
|
||||
|
||||
// Validate checks required flags and the inter-parameter dependency rules.
|
||||
// The shard column's existence and type are checked later, after metadata is
|
||||
// loaded, so the error can list candidate columns.
|
||||
func (c *Config) Validate() error {
|
||||
if c.SourceDSN == "" {
|
||||
return cfgErr("--source is required")
|
||||
}
|
||||
if c.Table == "" {
|
||||
return cfgErr("--table is required")
|
||||
}
|
||||
if strings.TrimSpace(c.Where) == "" {
|
||||
return cfgErr("--where is required")
|
||||
}
|
||||
if !c.HasFile() && c.DestDSN == "" && !c.Purge {
|
||||
return cfgErr("at least one output target is required: --file, --dest or --purge")
|
||||
}
|
||||
if c.ChunkSize <= 0 {
|
||||
return cfgErr("--chunk-size must be greater than 0")
|
||||
}
|
||||
if c.Retries < 0 {
|
||||
return cfgErr("--retries must be 0 or greater")
|
||||
}
|
||||
if err := c.Progress.resolve(); err != nil {
|
||||
return cfgErr("%v", err)
|
||||
}
|
||||
if c.ProgressInterval <= 0 {
|
||||
return cfgErr("--progress-interval must be greater than 0")
|
||||
}
|
||||
|
||||
switch c.ShardBy {
|
||||
case "none", "count", "date":
|
||||
default:
|
||||
return cfgErr("--shard-by must be one of none, count, date")
|
||||
}
|
||||
|
||||
switch c.ShardBy {
|
||||
case "date":
|
||||
if c.ShardRows < 0 {
|
||||
return cfgErr("--shard-rows must be 0 or greater")
|
||||
}
|
||||
default:
|
||||
if c.ShardColumn != "" {
|
||||
return cfgErr("--shard-column is only valid with --shard-by date")
|
||||
}
|
||||
if c.ShardColumnType != "" {
|
||||
return cfgErr("--shard-column-type is only valid with --shard-by date")
|
||||
}
|
||||
if c.ZeroDate != "" {
|
||||
return cfgErr("--zero-date is only valid with --shard-by date")
|
||||
}
|
||||
}
|
||||
if c.ShardBy == "none" && c.ShardRows != 0 {
|
||||
return cfgErr("--shard-rows is only valid with --shard-by count or date")
|
||||
}
|
||||
if c.ShardBy == "count" && c.ShardRows == 0 {
|
||||
return cfgErr("--shard-rows must be greater than 0 for --shard-by count")
|
||||
}
|
||||
if c.BulkDelete && !c.Purge {
|
||||
return cfgErr("--bulk-delete requires --purge")
|
||||
}
|
||||
if c.BulkDelete && c.ShardBy == "date" {
|
||||
return cfgErr("--bulk-delete cannot be used with --shard-by date; delete by IN list instead")
|
||||
}
|
||||
|
||||
switch c.Format {
|
||||
case "dump", "csv", "jsonl":
|
||||
default:
|
||||
return cfgErr("--format must be one of dump, csv, jsonl")
|
||||
}
|
||||
switch c.Compress {
|
||||
case "none", "gzip", "zstd", "xz":
|
||||
default:
|
||||
return cfgErr("--compress must be one of none, gzip, zstd, xz")
|
||||
}
|
||||
if c.CompressLevel < 1 || c.CompressLevel > 9 {
|
||||
return cfgErr("--compress-level must be between 1 and 9")
|
||||
}
|
||||
if c.Compress == "none" && c.CompressLevel != 3 {
|
||||
return cfgErr("--compress-level is only valid when compression is enabled")
|
||||
}
|
||||
switch c.ZeroDate {
|
||||
case "", "bucket":
|
||||
default:
|
||||
return cfgErr("--zero-date must be empty or bucket")
|
||||
}
|
||||
if _, err := time.LoadLocation(c.ShardTimeZone); err != nil {
|
||||
return cfgErr("--shard-time-zone %q is not a valid location: %v", c.ShardTimeZone, err)
|
||||
}
|
||||
if _, err := c.SourceConfig(); err != nil {
|
||||
return cfgErr("--source: %v", err)
|
||||
}
|
||||
if _, err := c.DestConfig(); err != nil {
|
||||
return cfgErr("--dest: %v", err)
|
||||
}
|
||||
if c.CheckReplicaLag != "" {
|
||||
if err := validateDSN(c.CheckReplicaLag); err != nil {
|
||||
return cfgErr("--check-replica-lag: %v", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateDSN(dsn string) error {
|
||||
_, err := normalize(dsn)
|
||||
return err
|
||||
}
|
||||
|
||||
func cfgErr(format string, args ...any) error {
|
||||
return archererr.Newf(archererr.KindConfig, format, args...)
|
||||
}
|
||||
Reference in new issue
Block a user