Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,263 @@
|
||||
// Package schema loads MySQL table metadata and answers questions about
|
||||
// columns and indexes needed to choose a scan strategy.
|
||||
package schema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
// Column is one column of the source table.
|
||||
type Column struct {
|
||||
Name string
|
||||
Ordinal int
|
||||
DataType string
|
||||
ColumnType string
|
||||
Nullable bool
|
||||
Extra string
|
||||
Key string
|
||||
}
|
||||
|
||||
func (c Column) IsAutoIncrement() bool {
|
||||
return strings.Contains(strings.ToLower(c.Extra), "auto_increment")
|
||||
}
|
||||
|
||||
// Index is one index of the source table, in index-column order.
|
||||
type Index struct {
|
||||
Name string
|
||||
Unique bool
|
||||
Columns []string
|
||||
}
|
||||
|
||||
// Table is the loaded metadata of the source table.
|
||||
type Table struct {
|
||||
DB string
|
||||
Name string
|
||||
Columns []Column
|
||||
byName map[string]Column
|
||||
Indexes []Index
|
||||
}
|
||||
|
||||
func (t *Table) Column(name string) (Column, bool) {
|
||||
c, ok := t.byName[strings.ToLower(name)]
|
||||
return c, ok
|
||||
}
|
||||
|
||||
func (t *Table) ColumnNames() []string {
|
||||
out := make([]string, len(t.Columns))
|
||||
for i, c := range t.Columns {
|
||||
out[i] = c.Name
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TimeColumns returns every DATE/DATETIME/TIMESTAMP/IPv4-ish candidate used to
|
||||
// explain to the user which shard columns are available.
|
||||
func (t *Table) TimeColumns() []Column {
|
||||
var out []Column
|
||||
for _, c := range t.Columns {
|
||||
if c.DataType == "date" || c.DataType == "datetime" || c.DataType == "timestamp" {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Load reads column and index metadata from information_schema.
|
||||
func Load(ctx context.Context, db *sql.DB, dbName, table string) (*Table, error) {
|
||||
t := &Table{DB: dbName, Name: table, byName: map[string]Column{}}
|
||||
|
||||
rows, err := db.QueryContext(ctx, `
|
||||
SELECT COLUMN_NAME, ORDINAL_POSITION, LOWER(DATA_TYPE), COLUMN_TYPE, IS_NULLABLE, EXTRA, COLUMN_KEY
|
||||
FROM information_schema.COLUMNS
|
||||
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
|
||||
ORDER BY ORDINAL_POSITION`, dbName, table)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load columns: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var c Column
|
||||
var nullable string
|
||||
if err := rows.Scan(&c.Name, &c.Ordinal, &c.DataType, &c.ColumnType, &nullable, &c.Extra, &c.Key); err != nil {
|
||||
return nil, fmt.Errorf("scan column: %w", err)
|
||||
}
|
||||
c.Nullable = strings.EqualFold(nullable, "YES")
|
||||
t.Columns = append(t.Columns, c)
|
||||
t.byName[strings.ToLower(c.Name)] = c
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("load columns: %w", err)
|
||||
}
|
||||
if len(t.Columns) == 0 {
|
||||
return nil, fmt.Errorf("table %s.%s not found or has no columns", dbName, table)
|
||||
}
|
||||
|
||||
idxRows, err := db.QueryContext(ctx, `
|
||||
SELECT INDEX_NAME, NON_UNIQUE, SEQ_IN_INDEX, COLUMN_NAME
|
||||
FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
|
||||
ORDER BY INDEX_NAME, SEQ_IN_INDEX`, dbName, table)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load indexes: %w", err)
|
||||
}
|
||||
defer idxRows.Close()
|
||||
|
||||
byIdx := map[string]*Index{}
|
||||
for idxRows.Next() {
|
||||
var name, col string
|
||||
var nonUnique, seq int
|
||||
if err := idxRows.Scan(&name, &nonUnique, &seq, &col); err != nil {
|
||||
return nil, fmt.Errorf("scan index: %w", err)
|
||||
}
|
||||
if col == "" {
|
||||
continue
|
||||
}
|
||||
ix := byIdx[name]
|
||||
if ix == nil {
|
||||
ix = &Index{Name: name, Unique: nonUnique == 0}
|
||||
byIdx[name] = ix
|
||||
t.Indexes = append(t.Indexes, *ix)
|
||||
}
|
||||
ix.Columns = append(ix.Columns, col)
|
||||
}
|
||||
if err := idxRows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("load indexes: %w", err)
|
||||
}
|
||||
// Rebuild pointers into the slice since append may have copied structs.
|
||||
for i := range t.Indexes {
|
||||
t.Indexes[i] = *byIdx[t.Indexes[i].Name]
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// KeyInfo describes the unique index chosen as scan key.
|
||||
type KeyInfo struct {
|
||||
Name string
|
||||
Columns []string
|
||||
IsPrimary bool
|
||||
}
|
||||
|
||||
func (t *Table) PrimaryKey() *Index {
|
||||
for i := range t.Indexes {
|
||||
if t.Indexes[i].Name == "PRIMARY" {
|
||||
return &t.Indexes[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ChooseKey selects the scan key: an explicitly requested index, else the
|
||||
// primary key, else the first unique index. Every key column must be NOT NULL.
|
||||
func (t *Table) ChooseKey(forced string) (KeyInfo, error) {
|
||||
var chosen *Index
|
||||
switch {
|
||||
case forced != "":
|
||||
for i := range t.Indexes {
|
||||
if strings.EqualFold(t.Indexes[i].Name, forced) {
|
||||
chosen = &t.Indexes[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if chosen == nil {
|
||||
return KeyInfo{}, fmt.Errorf("index %q does not exist on %s", forced, t.Name)
|
||||
}
|
||||
if !chosen.Unique {
|
||||
return KeyInfo{}, fmt.Errorf("index %q is not unique; a unique index is required for seek pagination", forced)
|
||||
}
|
||||
case t.PrimaryKey() != nil:
|
||||
chosen = t.PrimaryKey()
|
||||
default:
|
||||
for i := range t.Indexes {
|
||||
if t.Indexes[i].Unique {
|
||||
chosen = &t.Indexes[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if chosen == nil {
|
||||
return KeyInfo{}, fmt.Errorf("table %s has no primary key or unique index; create one or pass --index", t.Name)
|
||||
}
|
||||
}
|
||||
for _, col := range chosen.Columns {
|
||||
c, ok := t.Column(col)
|
||||
if !ok {
|
||||
return KeyInfo{}, fmt.Errorf("index %q references unknown column %q", chosen.Name, col)
|
||||
}
|
||||
if c.Nullable {
|
||||
return KeyInfo{}, fmt.Errorf("index %q column %q is nullable; seek pagination needs a NOT NULL unique key", chosen.Name, col)
|
||||
}
|
||||
}
|
||||
return KeyInfo{Name: chosen.Name, Columns: chosen.Columns, IsPrimary: chosen.Name == "PRIMARY"}, nil
|
||||
}
|
||||
|
||||
// FindIndexExact returns the index whose columns are exactly cols.
|
||||
func (t *Table) FindIndexExact(cols []string) *Index {
|
||||
for i := range t.Indexes {
|
||||
if equalFoldSlice(t.Indexes[i].Columns, cols) {
|
||||
return &t.Indexes[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FindIndexPrefix returns the first index whose leading columns are exactly
|
||||
// prefix, and its full column list.
|
||||
func (t *Table) FindIndexPrefix(prefix []string) *Index {
|
||||
for i := range t.Indexes {
|
||||
ix := &t.Indexes[i]
|
||||
if len(ix.Columns) < len(prefix) {
|
||||
continue
|
||||
}
|
||||
if equalFoldSlice(ix.Columns[:len(prefix)], prefix) {
|
||||
return ix
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HasColumnInIndex reports whether col is the first column of some index.
|
||||
func (t *Table) HasColumnPrefixIndex(col string) *Index {
|
||||
return t.FindIndexPrefix([]string{col})
|
||||
}
|
||||
|
||||
func equalFoldSlice(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if !strings.EqualFold(a[i], b[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// OutputColumns resolves the requested column list (empty means all) and
|
||||
// validates names against the table.
|
||||
func (t *Table) OutputColumns(requested []string) ([]row.Column, error) {
|
||||
if len(requested) == 0 {
|
||||
out := make([]row.Column, len(t.Columns))
|
||||
for i, c := range t.Columns {
|
||||
out[i] = row.Column{Name: c.Name, Type: c.DataType}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
out := make([]row.Column, 0, len(requested))
|
||||
seen := map[string]bool{}
|
||||
for _, name := range requested {
|
||||
c, ok := t.Column(name)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown column %q", name)
|
||||
}
|
||||
if seen[c.Name] {
|
||||
continue
|
||||
}
|
||||
seen[c.Name] = true
|
||||
out = append(out, row.Column{Name: c.Name, Type: c.DataType})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
Reference in new issue
Block a user