Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,129 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/cli"
|
||||
"archer/internal/row"
|
||||
"archer/internal/schema"
|
||||
)
|
||||
|
||||
// probeResult is everything learned about the source table needed to plan the
|
||||
// scan and the encoders.
|
||||
type probeResult struct {
|
||||
table *schema.Table
|
||||
key schema.KeyInfo
|
||||
shard *schema.ShardSpec
|
||||
path string // pk | A | B
|
||||
joint *schema.Index
|
||||
single *schema.Index
|
||||
selectCols []string
|
||||
selectMeta []row.Column
|
||||
keyIdx []int
|
||||
shardIdx int
|
||||
outCols []row.Column
|
||||
outIdx []int
|
||||
}
|
||||
|
||||
func probeTable(ctx context.Context, db *sql.DB, cfg *cli.Config, dbName string, loc *time.Location) (*probeResult, error) {
|
||||
table, err := schema.Load(ctx, db, dbName, cfg.Table)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
key, err := table.ChooseKey(cfg.Index)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
outCols, err := table.OutputColumns(cfg.Columns)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
p := &probeResult{table: table, key: key, path: "pk", shardIdx: -1, outCols: outCols}
|
||||
|
||||
if cfg.IsDateShard() {
|
||||
spec, err := schema.ResolveShardColumn(ctx, db, table, cfg.ShardColumn, cfg.ShardColumnType, cfg.Where, loc)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
p.shard = spec
|
||||
prefix := append([]string{spec.Column}, key.Columns...)
|
||||
p.joint = table.FindIndexPrefix(prefix)
|
||||
p.single = table.FindIndexPrefix([]string{spec.Column})
|
||||
switch {
|
||||
case p.joint != nil:
|
||||
p.path = "A"
|
||||
case p.single != nil:
|
||||
p.path = "B"
|
||||
case cfg.AllowFullScan:
|
||||
p.path = "B"
|
||||
default:
|
||||
return nil, archererr.Newf(archererr.KindValidation,
|
||||
"no usable index on %s for date sharding\n joint: create an index on (%s) for keyset chunks\n or: create an index on (%s) for day-at-a-time streaming\n or: pass --allow-full-scan to accept a full table scan\n%s",
|
||||
table.Name, strings.Join(prefix, ", "), spec.Column, describeIndexes(table))
|
||||
}
|
||||
}
|
||||
|
||||
needed := map[string]bool{}
|
||||
for _, c := range outCols {
|
||||
needed[strings.ToLower(c.Name)] = true
|
||||
}
|
||||
for _, c := range key.Columns {
|
||||
needed[strings.ToLower(c)] = true
|
||||
}
|
||||
if p.shard != nil {
|
||||
needed[strings.ToLower(p.shard.Column)] = true
|
||||
}
|
||||
for _, c := range table.Columns {
|
||||
if needed[strings.ToLower(c.Name)] {
|
||||
p.selectCols = append(p.selectCols, c.Name)
|
||||
p.selectMeta = append(p.selectMeta, row.Column{Name: c.Name, Type: c.DataType})
|
||||
}
|
||||
}
|
||||
pos := map[string]int{}
|
||||
for i, name := range p.selectCols {
|
||||
pos[name] = i
|
||||
}
|
||||
p.keyIdx = make([]int, len(key.Columns))
|
||||
for i, c := range key.Columns {
|
||||
idx, ok := pos[c]
|
||||
if !ok {
|
||||
return nil, archererr.Newf(archererr.KindValidation, "internal: key column %s missing from select list", c)
|
||||
}
|
||||
p.keyIdx[i] = idx
|
||||
}
|
||||
if p.shard != nil {
|
||||
idx, ok := pos[p.shard.Column]
|
||||
if !ok {
|
||||
return nil, archererr.Newf(archererr.KindValidation, "internal: shard column %s missing from select list", p.shard.Column)
|
||||
}
|
||||
p.shardIdx = idx
|
||||
}
|
||||
p.outIdx = make([]int, len(outCols))
|
||||
for i, c := range outCols {
|
||||
p.outIdx[i] = pos[c.Name]
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func describeIndexes(t *schema.Table) string {
|
||||
if len(t.Indexes) == 0 {
|
||||
return " indexes: (none)"
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString(" indexes:\n")
|
||||
for _, ix := range t.Indexes {
|
||||
kind := "index"
|
||||
if ix.Name == "PRIMARY" {
|
||||
kind = "primary key"
|
||||
} else if ix.Unique {
|
||||
kind = "unique index"
|
||||
}
|
||||
fmt.Fprintf(&b, " - %s (%s) %s\n", ix.Name, strings.Join(ix.Columns, ", "), kind)
|
||||
}
|
||||
return strings.TrimRight(b.String(), "\n")
|
||||
}
|
||||
Reference in new issue
Block a user