Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,162 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
|
||||
"archer/internal/purge"
|
||||
"archer/internal/shard"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
func (a *App) printDryRun(w io.Writer) error {
|
||||
cfg := a.cfg
|
||||
comp, err := shard.NewCompressor(cfg.Compress, cfg.CompressLevel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(w, "== resolved configuration ==")
|
||||
fmt.Fprintf(w, "source: %s\n", redactDSN(cfg.SourceDSN))
|
||||
if cfg.DestDSN != "" {
|
||||
fmt.Fprintf(w, "dest: %s\n", redactDSN(cfg.DestDSN))
|
||||
}
|
||||
fmt.Fprintf(w, "table: %s\n", cfg.Table)
|
||||
if cfg.DestDSN != "" {
|
||||
fmt.Fprintf(w, "dest table: %s\n", cfg.DestTable)
|
||||
}
|
||||
fmt.Fprintf(w, "where: %s\n", cfg.Where)
|
||||
if len(cfg.Columns) > 0 {
|
||||
fmt.Fprintf(w, "columns: %s\n", strings.Join(cfg.Columns, ", "))
|
||||
} else {
|
||||
fmt.Fprintf(w, "columns: (all)\n")
|
||||
}
|
||||
targets := []string{}
|
||||
if cfg.HasFile() {
|
||||
targets = append(targets, "file="+cfg.FileDir)
|
||||
}
|
||||
if cfg.DestDSN != "" {
|
||||
targets = append(targets, "dest")
|
||||
}
|
||||
if cfg.Purge {
|
||||
targets = append(targets, "purge")
|
||||
}
|
||||
fmt.Fprintf(w, "targets: %s\n", strings.Join(targets, ", "))
|
||||
fmt.Fprintf(w, "chunk size: %d\n", cfg.ChunkSize)
|
||||
fmt.Fprintf(w, "shard by: %s\n", cfg.ShardBy)
|
||||
if cfg.ShardBy != "none" {
|
||||
fmt.Fprintf(w, "shard rows: %d\n", cfg.ShardRows)
|
||||
}
|
||||
if a.probe.shard != nil {
|
||||
fmt.Fprintf(w, "shard column: %s (%s)\n", a.probe.shard.Column, a.probe.shard.Type)
|
||||
fmt.Fprintf(w, "shard time zone: %s\n", cfg.ShardTimeZone)
|
||||
}
|
||||
fmt.Fprintf(w, "format: %s\n", cfg.Format)
|
||||
if comp.Name == "none" {
|
||||
fmt.Fprintf(w, "compress: none\n")
|
||||
} else {
|
||||
fmt.Fprintf(w, "compress: %s (level %d)\n", comp.Name, comp.Level)
|
||||
}
|
||||
if cfg.InsertMode != "" {
|
||||
fmt.Fprintf(w, "insert mode: %s\n", cfg.InsertMode)
|
||||
}
|
||||
fmt.Fprintf(w, "max lag: %s (check interval %s)\n", cfg.MaxLag, cfg.CheckInterval)
|
||||
fmt.Fprintf(w, "retries: %d\n", cfg.Retries)
|
||||
fmt.Fprintf(w, "safe auto inc: %t\n", cfg.SafeAutoIncrement)
|
||||
fmt.Fprintf(w, "config hash: %s\n", a.cfgHash)
|
||||
|
||||
fmt.Fprintln(w, "\n== index probe ==")
|
||||
fmt.Fprintf(w, "scan key: %s (%s)\n", a.probe.key.Name, strings.Join(a.probe.key.Columns, ", "))
|
||||
if a.probe.path == "A" {
|
||||
fmt.Fprintf(w, "scan path: A (keyset chunks using index %s)\n", a.probe.joint.Name)
|
||||
} else if a.probe.path == "B" {
|
||||
if a.probe.single != nil {
|
||||
fmt.Fprintf(w, "scan path: B (day-at-a-time using index %s)\n", a.probe.single.Name)
|
||||
} else {
|
||||
fmt.Fprintln(w, "scan path: B (day-at-a-time, full table scan allowed)")
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintln(w, "scan path: primary key seek")
|
||||
}
|
||||
|
||||
fmt.Fprintln(w, "\n== first SELECT ==")
|
||||
for _, q := range a.dryRunSelects() {
|
||||
fmt.Fprintf(w, " %s\n", q)
|
||||
}
|
||||
|
||||
fmt.Fprintln(w, "\n== write statements ==")
|
||||
if cfg.DestDSN != "" {
|
||||
fmt.Fprintf(w, " %s\n", a.dryRunInsert())
|
||||
}
|
||||
if cfg.Purge {
|
||||
p := purge.New(nil, a.srcCfg.DBName, cfg.Table, a.probe.key.Columns, cfg.Where, cfg.BulkDelete, cfg.SafeAutoIncrement, false)
|
||||
for _, q := range p.DryRunSQL() {
|
||||
fmt.Fprintf(w, " %s\n", q)
|
||||
}
|
||||
}
|
||||
if cfg.HasFile() {
|
||||
fmt.Fprintf(w, " file: %s/[...]<seq>.<pk_lo>-<pk_hi>.%s%s (written as .partial until sealed)\n",
|
||||
cfg.FileDir, cfg.Format, comp.Ext)
|
||||
}
|
||||
fmt.Fprintln(w, "\nnote: file output is at-least-once; a crash and rerun may duplicate rows in shard files")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) dryRunSelects() []string {
|
||||
cols := sqlutil.Columns(a.probe.selectCols)
|
||||
table := sqlutil.Qualified(a.srcCfg.DBName, a.cfg.Table)
|
||||
where := a.cfg.Where
|
||||
key := sqlutil.Columns(a.probe.key.Columns)
|
||||
limit := fmt.Sprintf(" LIMIT %d", a.cfg.ChunkSize)
|
||||
switch a.probe.path {
|
||||
case "A":
|
||||
shard := sqlutil.Ident(a.probe.shard.Column)
|
||||
colsWithShard := sqlutil.Columns(append([]string{a.probe.shard.Column}, a.probe.key.Columns...))
|
||||
return []string{
|
||||
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s IS NULL ORDER BY %s%s", cols, table, where, shard, key, limit),
|
||||
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s IS NOT NULL AND (%s) > (?, ...) ORDER BY %s%s", cols, table, where, shard, colsWithShard, colsWithShard, limit),
|
||||
}
|
||||
case "B":
|
||||
shard := sqlutil.Ident(a.probe.shard.Column)
|
||||
colsWithShard := sqlutil.Columns(append([]string{a.probe.shard.Column}, a.probe.key.Columns...))
|
||||
return []string{
|
||||
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s IS NULL ORDER BY %s", cols, table, where, shard, key),
|
||||
fmt.Sprintf("SELECT %s FROM %s WHERE (%s) AND %s >= ? AND %s < ? ORDER BY %s", cols, table, where, shard, shard, colsWithShard),
|
||||
}
|
||||
default:
|
||||
return []string{fmt.Sprintf("SELECT %s FROM %s WHERE (%s) ORDER BY %s%s", cols, table, where, key, limit)}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) dryRunInsert() string {
|
||||
destCfg, err := a.cfg.DestConfig()
|
||||
dbName := ""
|
||||
if err == nil && destCfg != nil {
|
||||
dbName = destCfg.DBName
|
||||
}
|
||||
verb := "INSERT INTO"
|
||||
switch a.cfg.InsertMode {
|
||||
case "ignore":
|
||||
verb = "INSERT IGNORE INTO"
|
||||
case "replace":
|
||||
verb = "REPLACE INTO"
|
||||
}
|
||||
names := make([]string, len(a.probe.outCols))
|
||||
for i, c := range a.probe.outCols {
|
||||
names[i] = c.Name
|
||||
}
|
||||
tuple := sqlutil.ParamTuple(len(names))
|
||||
return fmt.Sprintf("%s %s (%s) VALUES %s, %s, ...",
|
||||
verb, sqlutil.Qualified(dbName, a.cfg.DestTable), sqlutil.Columns(names), tuple, tuple)
|
||||
}
|
||||
|
||||
func redactDSN(dsn string) string {
|
||||
c, err := mysql.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
return "<invalid dsn>"
|
||||
}
|
||||
c.Passwd = ""
|
||||
return c.FormatDSN()
|
||||
}
|
||||
Reference in new issue
Block a user