Initial commit: archer, a Go-native MySQL/MariaDB table archiver

Renamed from arcdb: module path, cmd/archer, internal/archererr,
docs, build scripts and ARCHER_* environment variables.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-06 22:12:17 +08:00
commit bc4a6b504f
99 files changed
+12752

No files matched your search

+219
View File
@@ -0,0 +1,219 @@
package schema
import (
"context"
"database/sql"
"fmt"
"strconv"
"strings"
"time"
"archer/internal/row"
"archer/internal/sqlutil"
)
// ShardKind is how a shard column's date is derived from its raw value.
type ShardKind int
const (
ShardNone ShardKind = iota
ShardDate
ShardDateTime
ShardEpoch
)
// ShardSpec is the resolved shard column and how to interpret it.
type ShardSpec struct {
Column string
Kind ShardKind
Unit int64 // epoch multiplier to nanoseconds: 1e9, 1e6, 1e3
Type string
}
// ResolveShardColumn validates --shard-column against the table and the unit
// flag. It returns a descriptive error listing candidates when the column is
// missing or the type is unsupported.
func ResolveShardColumn(ctx context.Context, db *sql.DB, t *Table, column, unitFlag, where string, loc *time.Location) (*ShardSpec, error) {
if column == "" {
return nil, fmt.Errorf("--shard-column is required for --shard-by date\ncandidate columns:\n%s", describeTimeColumns(t))
}
c, ok := t.Column(column)
if !ok {
return nil, fmt.Errorf("shard column %q does not exist; candidate columns:\n%s", column, describeTimeColumns(t))
}
switch c.DataType {
case "date":
return &ShardSpec{Column: c.Name, Kind: ShardDate, Unit: 1, Type: c.DataType}, nil
case "datetime", "timestamp":
return &ShardSpec{Column: c.Name, Kind: ShardDateTime, Unit: 1, Type: c.DataType}, nil
}
if row.IsInteger(c.DataType) {
unit, err := ParseEpochUnit(unitFlag)
if err != nil {
minV, maxV, ok, serr := SampleEpochRange(ctx, db, t, c.Name, where)
if serr != nil {
return nil, fmt.Errorf("%w (also failed to sample %s: %v)", err, c.Name, serr)
}
if !ok {
return nil, fmt.Errorf("%w; column %s has no non-NULL values to sample", err, c.Name)
}
return nil, fmt.Errorf("%w\nsampled %s range: %d .. %d\n seconds: %s .. %s\n milliseconds: %s .. %s\n microseconds: %s .. %s",
err, c.Name, minV, maxV,
epochString(minV, 1, loc), epochString(maxV, 1, loc),
epochString(minV, 1_000, loc), epochString(maxV, 1_000, loc),
epochString(minV, 1_000_000, loc), epochString(maxV, 1_000_000, loc))
}
return &ShardSpec{Column: c.Name, Kind: ShardEpoch, Unit: unit, Type: c.DataType}, nil
}
return nil, fmt.Errorf("column %q has unsupported type %s; v1 supports DATE, DATETIME, TIMESTAMP and integer epoch columns with --shard-column-type\ncandidate columns:\n%s",
c.Name, c.DataType, describeTimeColumns(t))
}
func describeTimeColumns(t *Table) string {
var b strings.Builder
found := false
for _, c := range t.TimeColumns() {
found = true
fmt.Fprintf(&b, " - %s %s%s\n", c.Name, c.ColumnType, nullableSuffix(c))
}
for _, c := range t.Columns {
if row.IsInteger(c.DataType) {
found = true
fmt.Fprintf(&b, " - %s %s (integer; requires --shard-column-type)%s\n", c.Name, c.ColumnType, nullableSuffix(c))
}
}
if !found {
return " (none)"
}
return strings.TrimRight(b.String(), "\n")
}
func nullableSuffix(c Column) string {
if c.Nullable {
return " NULL"
}
return " NOT NULL"
}
// ParseEpochUnit maps --shard-column-type to raw units per second: 1 for
// seconds, 1000 for milliseconds, 1000000 for microseconds.
func ParseEpochUnit(flag string) (int64, error) {
switch strings.ToLower(strings.TrimSpace(flag)) {
case "epoch-seconds", "seconds", "second", "s", "1":
return 1, nil
case "epoch-milliseconds", "epoch-millis", "milliseconds", "millisecond", "ms", "1000":
return 1_000, nil
case "epoch-microseconds", "epoch-micros", "microseconds", "microsecond", "us", "1000000":
return 1_000_000, nil
case "":
return 0, fmt.Errorf("integer shard column requires --shard-column-type (epoch-seconds, epoch-milliseconds or epoch-microseconds)")
default:
return 0, fmt.Errorf("unsupported --shard-column-type %q; use epoch-seconds, epoch-milliseconds or epoch-microseconds", flag)
}
}
// SampleEpochRange returns MIN/MAX of an integer column under the base where.
func SampleEpochRange(ctx context.Context, db *sql.DB, t *Table, col, where string) (int64, int64, bool, error) {
q := fmt.Sprintf("SELECT MIN(%s), MAX(%s) FROM %s", sqlutil.Ident(col), sqlutil.Ident(col), sqlutil.Qualified(t.DB, t.Name))
if strings.TrimSpace(where) != "" {
q += " WHERE (" + where + ")"
}
var minRaw, maxRaw sql.NullString
if err := db.QueryRowContext(ctx, q).Scan(&minRaw, &maxRaw); err != nil {
return 0, 0, false, err
}
if !minRaw.Valid || !maxRaw.Valid {
return 0, 0, false, nil
}
minV, err := strconv.ParseInt(strings.TrimSpace(minRaw.String), 10, 64)
if err != nil {
return 0, 0, false, fmt.Errorf("parse min %q: %w", minRaw.String, err)
}
maxV, err := strconv.ParseInt(strings.TrimSpace(maxRaw.String), 10, 64)
if err != nil {
return 0, 0, false, fmt.Errorf("parse max %q: %w", maxRaw.String, err)
}
return minV, maxV, true, nil
}
func epochString(v, unit int64, loc *time.Location) string {
return EpochTime(v, unit, loc).Format("2006-01-02 15:04:05")
}
// EpochTime converts an epoch value with the given nanoseconds-per-unit to a
// time in loc.
func EpochTime(v, unit int64, loc *time.Location) time.Time {
sec := v / unit
frac := v % unit
if frac < 0 {
sec--
frac += unit
}
nsec := frac * (1_000_000_000 / unit)
return time.Unix(sec, nsec).In(loc)
}
// TimeToEpoch is the inverse of EpochTime.
func TimeToEpoch(t time.Time, unit int64) int64 {
return t.Unix()*unit + int64(t.Nanosecond())/(1_000_000_000/unit)
}
const nullBucket = "_null"
const zeroBucket = "0000-00-00"
// BucketOf maps a raw shard value to its date bucket. key is used only in
// error messages.
func BucketOf(v row.Value, spec *ShardSpec, loc *time.Location, zeroDate string, key row.Key) (string, error) {
if v.IsNull() {
return nullBucket, nil
}
switch spec.Kind {
case ShardEpoch:
n, err := strconv.ParseInt(strings.TrimSpace(string(v.Raw)), 10, 64)
if err != nil {
return "", fmt.Errorf("row %s: cannot parse epoch value %q in column %s: %w", key, string(v.Raw), spec.Column, err)
}
return EpochTime(n, spec.Unit, loc).Format("2006-01-02"), nil
case ShardDate, ShardDateTime:
s := string(v.Raw)
if len(s) < 10 {
return "", fmt.Errorf("row %s: cannot parse date value %q in column %s", key, s, spec.Column)
}
day := s[:10]
if strings.HasPrefix(day, "0000-") {
if zeroDate == "bucket" {
return zeroBucket, nil
}
return "", fmt.Errorf("row %s has zero date %q in column %s; pass --zero-date bucket to route it to date=0000-00-00", key, s, spec.Column)
}
if _, err := time.Parse("2006-01-02", day); err != nil {
return "", fmt.Errorf("row %s: invalid date %q in column %s", key, s, spec.Column)
}
return day, nil
}
return "", fmt.Errorf("internal: shard kind not resolved")
}
// BucketLess orders buckets in scan order: NULL first, then zero dates, then
// real dates.
func BucketLess(a, b string) bool {
ca, ka := bucketClass(a)
cb, kb := bucketClass(b)
if ca != cb {
return ca < cb
}
return ka < kb
}
func bucketClass(b string) (int, string) {
switch b {
case "":
return -1, ""
case nullBucket:
return 0, ""
case zeroBucket:
return 1, ""
default:
return 2, b
}
}
+111
View File
@@ -0,0 +1,111 @@
package schema
import (
"testing"
"time"
"archer/internal/row"
)
func TestBucketOfDateTime(t *testing.T) {
spec := &ShardSpec{Column: "created_at", Kind: ShardDateTime}
loc := time.UTC
cases := []struct {
raw []byte
want string
}{
{[]byte("2025-06-01 13:45:00"), "2025-06-01"},
{[]byte("2025-12-31 00:00:00"), "2025-12-31"},
}
for _, c := range cases {
got, err := BucketOf(row.Value{Raw: c.raw}, spec, loc, "", row.KeyOf("1"))
if err != nil || got != c.want {
t.Fatalf("%s: got %q err %v want %q", c.raw, got, err, c.want)
}
}
}
func TestBucketOfNull(t *testing.T) {
spec := &ShardSpec{Column: "c", Kind: ShardDateTime}
got, err := BucketOf(row.Value{}, spec, time.UTC, "", row.KeyOf("1"))
if err != nil || got != "_null" {
t.Fatalf("got %q err %v", got, err)
}
}
func TestBucketOfZeroDate(t *testing.T) {
spec := &ShardSpec{Column: "c", Kind: ShardDateTime}
if _, err := BucketOf(row.Value{Raw: []byte("0000-00-00 00:00:00")}, spec, time.UTC, "", row.KeyOf("7")); err == nil {
t.Fatal("expected zero-date error")
}
got, err := BucketOf(row.Value{Raw: []byte("0000-00-00 00:00:00")}, spec, time.UTC, "bucket", row.KeyOf("7"))
if err != nil || got != "0000-00-00" {
t.Fatalf("got %q err %v", got, err)
}
}
func TestBucketOfEpoch(t *testing.T) {
loc, err := time.LoadLocation("UTC")
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
unit int64
raw string
want string
}{
{1, "1717200000", "2024-06-01"},
{1_000, "1717200000000", "2024-06-01"},
{1_000_000, "1717200000000000", "2024-06-01"},
} {
spec := &ShardSpec{Column: "ts", Kind: ShardEpoch, Unit: tc.unit}
got, err := BucketOf(row.Value{Raw: []byte(tc.raw)}, spec, loc, "", row.KeyOf("1"))
if err != nil || got != tc.want {
t.Fatalf("unit %d: got %q err %v want %q", tc.unit, got, err, tc.want)
}
}
}
func TestBucketLess(t *testing.T) {
order := []string{"_null", "0000-00-00", "2024-01-01", "2024-01-02", "2025-01-01"}
for i := 0; i+1 < len(order); i++ {
if !BucketLess(order[i], order[i+1]) {
t.Fatalf("expected %q < %q", order[i], order[i+1])
}
if BucketLess(order[i+1], order[i]) {
t.Fatalf("expected %q not < %q", order[i+1], order[i])
}
}
}
func TestParseEpochUnit(t *testing.T) {
cases := map[string]int64{
"epoch-seconds": 1,
"seconds": 1,
"epoch-milliseconds": 1_000,
"ms": 1_000,
"epoch-microseconds": 1_000_000,
"us": 1_000_000,
}
for in, want := range cases {
got, err := ParseEpochUnit(in)
if err != nil || got != want {
t.Fatalf("%s: got %d err %v want %d", in, got, err, want)
}
}
if _, err := ParseEpochUnit(""); err == nil {
t.Fatal("empty unit should error")
}
}
func TestEpochRoundTrip(t *testing.T) {
loc := time.UTC
for _, unit := range []int64{1, 1_000, 1_000_000} {
now := time.Date(2024, 6, 1, 12, 30, 15, 0, loc)
v := TimeToEpoch(now, unit)
back := EpochTime(v, unit, loc)
if !back.Equal(now) {
t.Fatalf("unit %d: got %s want %s", unit, back, now)
}
}
}
+263
View File
@@ -0,0 +1,263 @@
// Package schema loads MySQL table metadata and answers questions about
// columns and indexes needed to choose a scan strategy.
package schema
import (
"context"
"database/sql"
"fmt"
"strings"
"archer/internal/row"
)
// Column is one column of the source table.
type Column struct {
Name string
Ordinal int
DataType string
ColumnType string
Nullable bool
Extra string
Key string
}
func (c Column) IsAutoIncrement() bool {
return strings.Contains(strings.ToLower(c.Extra), "auto_increment")
}
// Index is one index of the source table, in index-column order.
type Index struct {
Name string
Unique bool
Columns []string
}
// Table is the loaded metadata of the source table.
type Table struct {
DB string
Name string
Columns []Column
byName map[string]Column
Indexes []Index
}
func (t *Table) Column(name string) (Column, bool) {
c, ok := t.byName[strings.ToLower(name)]
return c, ok
}
func (t *Table) ColumnNames() []string {
out := make([]string, len(t.Columns))
for i, c := range t.Columns {
out[i] = c.Name
}
return out
}
// TimeColumns returns every DATE/DATETIME/TIMESTAMP/IPv4-ish candidate used to
// explain to the user which shard columns are available.
func (t *Table) TimeColumns() []Column {
var out []Column
for _, c := range t.Columns {
if c.DataType == "date" || c.DataType == "datetime" || c.DataType == "timestamp" {
out = append(out, c)
}
}
return out
}
// Load reads column and index metadata from information_schema.
func Load(ctx context.Context, db *sql.DB, dbName, table string) (*Table, error) {
t := &Table{DB: dbName, Name: table, byName: map[string]Column{}}
rows, err := db.QueryContext(ctx, `
SELECT COLUMN_NAME, ORDINAL_POSITION, LOWER(DATA_TYPE), COLUMN_TYPE, IS_NULLABLE, EXTRA, COLUMN_KEY
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
ORDER BY ORDINAL_POSITION`, dbName, table)
if err != nil {
return nil, fmt.Errorf("load columns: %w", err)
}
defer rows.Close()
for rows.Next() {
var c Column
var nullable string
if err := rows.Scan(&c.Name, &c.Ordinal, &c.DataType, &c.ColumnType, &nullable, &c.Extra, &c.Key); err != nil {
return nil, fmt.Errorf("scan column: %w", err)
}
c.Nullable = strings.EqualFold(nullable, "YES")
t.Columns = append(t.Columns, c)
t.byName[strings.ToLower(c.Name)] = c
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("load columns: %w", err)
}
if len(t.Columns) == 0 {
return nil, fmt.Errorf("table %s.%s not found or has no columns", dbName, table)
}
idxRows, err := db.QueryContext(ctx, `
SELECT INDEX_NAME, NON_UNIQUE, SEQ_IN_INDEX, COLUMN_NAME
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
ORDER BY INDEX_NAME, SEQ_IN_INDEX`, dbName, table)
if err != nil {
return nil, fmt.Errorf("load indexes: %w", err)
}
defer idxRows.Close()
byIdx := map[string]*Index{}
for idxRows.Next() {
var name, col string
var nonUnique, seq int
if err := idxRows.Scan(&name, &nonUnique, &seq, &col); err != nil {
return nil, fmt.Errorf("scan index: %w", err)
}
if col == "" {
continue
}
ix := byIdx[name]
if ix == nil {
ix = &Index{Name: name, Unique: nonUnique == 0}
byIdx[name] = ix
t.Indexes = append(t.Indexes, *ix)
}
ix.Columns = append(ix.Columns, col)
}
if err := idxRows.Err(); err != nil {
return nil, fmt.Errorf("load indexes: %w", err)
}
// Rebuild pointers into the slice since append may have copied structs.
for i := range t.Indexes {
t.Indexes[i] = *byIdx[t.Indexes[i].Name]
}
return t, nil
}
// KeyInfo describes the unique index chosen as scan key.
type KeyInfo struct {
Name string
Columns []string
IsPrimary bool
}
func (t *Table) PrimaryKey() *Index {
for i := range t.Indexes {
if t.Indexes[i].Name == "PRIMARY" {
return &t.Indexes[i]
}
}
return nil
}
// ChooseKey selects the scan key: an explicitly requested index, else the
// primary key, else the first unique index. Every key column must be NOT NULL.
func (t *Table) ChooseKey(forced string) (KeyInfo, error) {
var chosen *Index
switch {
case forced != "":
for i := range t.Indexes {
if strings.EqualFold(t.Indexes[i].Name, forced) {
chosen = &t.Indexes[i]
break
}
}
if chosen == nil {
return KeyInfo{}, fmt.Errorf("index %q does not exist on %s", forced, t.Name)
}
if !chosen.Unique {
return KeyInfo{}, fmt.Errorf("index %q is not unique; a unique index is required for seek pagination", forced)
}
case t.PrimaryKey() != nil:
chosen = t.PrimaryKey()
default:
for i := range t.Indexes {
if t.Indexes[i].Unique {
chosen = &t.Indexes[i]
break
}
}
if chosen == nil {
return KeyInfo{}, fmt.Errorf("table %s has no primary key or unique index; create one or pass --index", t.Name)
}
}
for _, col := range chosen.Columns {
c, ok := t.Column(col)
if !ok {
return KeyInfo{}, fmt.Errorf("index %q references unknown column %q", chosen.Name, col)
}
if c.Nullable {
return KeyInfo{}, fmt.Errorf("index %q column %q is nullable; seek pagination needs a NOT NULL unique key", chosen.Name, col)
}
}
return KeyInfo{Name: chosen.Name, Columns: chosen.Columns, IsPrimary: chosen.Name == "PRIMARY"}, nil
}
// FindIndexExact returns the index whose columns are exactly cols.
func (t *Table) FindIndexExact(cols []string) *Index {
for i := range t.Indexes {
if equalFoldSlice(t.Indexes[i].Columns, cols) {
return &t.Indexes[i]
}
}
return nil
}
// FindIndexPrefix returns the first index whose leading columns are exactly
// prefix, and its full column list.
func (t *Table) FindIndexPrefix(prefix []string) *Index {
for i := range t.Indexes {
ix := &t.Indexes[i]
if len(ix.Columns) < len(prefix) {
continue
}
if equalFoldSlice(ix.Columns[:len(prefix)], prefix) {
return ix
}
}
return nil
}
// HasColumnInIndex reports whether col is the first column of some index.
func (t *Table) HasColumnPrefixIndex(col string) *Index {
return t.FindIndexPrefix([]string{col})
}
func equalFoldSlice(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if !strings.EqualFold(a[i], b[i]) {
return false
}
}
return true
}
// OutputColumns resolves the requested column list (empty means all) and
// validates names against the table.
func (t *Table) OutputColumns(requested []string) ([]row.Column, error) {
if len(requested) == 0 {
out := make([]row.Column, len(t.Columns))
for i, c := range t.Columns {
out[i] = row.Column{Name: c.Name, Type: c.DataType}
}
return out, nil
}
out := make([]row.Column, 0, len(requested))
seen := map[string]bool{}
for _, name := range requested {
c, ok := t.Column(name)
if !ok {
return nil, fmt.Errorf("unknown column %q", name)
}
if seen[c.Name] {
continue
}
seen[c.Name] = true
out = append(out, row.Column{Name: c.Name, Type: c.DataType})
}
return out, nil
}