Initial commit: archer, a Go-native MySQL/MariaDB table archiver

Renamed from arcdb: module path, cmd/archer, internal/archererr,
docs, build scripts and ARCHER_* environment variables.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-06 22:12:17 +08:00
commit bc4a6b504f
99 files changed
+12752

No files matched your search

+122
View File
@@ -0,0 +1,122 @@
// Package estimate derives a cheap total-size estimate for progress display.
//
// Only bounded MIN/MAX queries are used; COUNT(*) is never attempted. A query
// that fails, times out, or whose metric does not apply silently yields no
// estimate, leaving the caller to render rows and rate only.
package estimate
import (
"context"
"database/sql"
"strconv"
"strings"
"time"
"archer/internal/progress"
"archer/internal/row"
"archer/internal/schema"
)
// Spec describes which metric a run can expose.
type Spec struct {
Kind string // "day", "key" or "" for none
Column string
ShardKind schema.ShardKind
Unit int64
Loc *time.Location
}
// RangeQuery returns MIN and MAX of a column as raw text. Implementations must
// honor ctx so the bounded timeout is effective.
type RangeQuery func(ctx context.Context, column string) (minRaw, maxRaw sql.NullString, err error)
// Bounded runs q under a timeout and converts the result into a display
// estimate. It returns nil instead of an error on every failure path.
func Bounded(ctx context.Context, q RangeQuery, spec Spec, timeout time.Duration) *progress.Estimate {
if q == nil || spec.Kind == "" || timeout <= 0 {
return nil
}
qctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
minRaw, maxRaw, err := q(qctx, spec.Column)
if err != nil || !minRaw.Valid || !maxRaw.Valid {
return nil
}
var lo, hi float64
var ok bool
switch spec.Kind {
case "day":
lo, hi, ok = dayRange(minRaw.String, maxRaw.String, spec.ShardKind, spec.Unit, spec.Loc)
case "key":
lo, hi, ok = keyRange(minRaw.String, maxRaw.String)
}
if !ok || hi <= lo {
return nil
}
return &progress.Estimate{Kind: spec.Kind, Lo: lo, Hi: hi}
}
// KeyColumn reports the scan key column when it is a single integer column,
// which is the only non-date shape a cheap key-range estimate applies to.
func KeyColumn(cols, types []string) (string, bool) {
if len(cols) != 1 || len(types) != 1 {
return "", false
}
if !row.IsInteger(types[0]) {
return "", false
}
return cols[0], true
}
func dayRange(minRaw, maxRaw string, kind schema.ShardKind, unit int64, loc *time.Location) (float64, float64, bool) {
if loc == nil {
loc = time.UTC
}
switch kind {
case schema.ShardDate, schema.ShardDateTime:
lo, ok1 := daySeconds(minRaw, loc)
hi, ok2 := daySeconds(maxRaw, loc)
return lo, hi, ok1 && ok2
case schema.ShardEpoch:
minV, err1 := strconv.ParseInt(strings.TrimSpace(minRaw), 10, 64)
maxV, err2 := strconv.ParseInt(strings.TrimSpace(maxRaw), 10, 64)
if err1 != nil || err2 != nil {
return 0, 0, false
}
lo := dayStart(schema.EpochTime(minV, unit, loc), loc).Unix()
hi := dayStart(schema.EpochTime(maxV, unit, loc), loc).Unix()
return float64(lo), float64(hi), true
}
return 0, 0, false
}
func daySeconds(raw string, loc *time.Location) (float64, bool) {
s := strings.TrimSpace(raw)
if len(s) < 10 {
return 0, false
}
day := s[:10]
if strings.HasPrefix(day, "0000-") {
return 0, false
}
t, err := time.ParseInLocation("2006-01-02", day, loc)
if err != nil {
return 0, false
}
return float64(t.Unix()), true
}
func keyRange(minRaw, maxRaw string) (float64, float64, bool) {
minV, err1 := strconv.ParseInt(strings.TrimSpace(minRaw), 10, 64)
maxV, err2 := strconv.ParseInt(strings.TrimSpace(maxRaw), 10, 64)
if err1 != nil || err2 != nil {
return 0, 0, false
}
return float64(minV), float64(maxV), true
}
func dayStart(t time.Time, loc *time.Location) time.Time {
return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, loc)
}
+113
View File
@@ -0,0 +1,113 @@
package estimate
import (
"context"
"database/sql"
"strconv"
"testing"
"time"
"archer/internal/schema"
)
func validRange(minRaw, maxRaw string) RangeQuery {
return func(context.Context, string) (sql.NullString, sql.NullString, error) {
return sql.NullString{String: minRaw, Valid: true}, sql.NullString{String: maxRaw, Valid: true}, nil
}
}
func TestKeyColumnApplicability(t *testing.T) {
cases := []struct {
cols, types []string
want bool
}{
{[]string{"id"}, []string{"bigint"}, true},
{[]string{"id"}, []string{"int"}, true},
{[]string{"a", "b"}, []string{"bigint", "bigint"}, false},
{[]string{"code"}, []string{"varchar"}, false},
{[]string{"payload"}, []string{"blob"}, false},
{[]string{"bin"}, []string{"varbinary"}, false},
{[]string{"ts"}, []string{"datetime"}, false},
}
for _, tc := range cases {
col, ok := KeyColumn(tc.cols, tc.types)
if ok != tc.want {
t.Fatalf("%v/%v: got %v want %v", tc.cols, tc.types, ok, tc.want)
}
if ok && col != tc.cols[0] {
t.Fatalf("%v: wrong column %q", tc.cols, col)
}
}
}
func TestBoundedDayRangeFromDates(t *testing.T) {
spec := Spec{Kind: "day", Column: "created_at", ShardKind: schema.ShardDate, Loc: time.UTC}
got := Bounded(context.Background(), validRange("2025-01-01", "2025-01-11"), spec, time.Second)
if got == nil || got.Kind != "day" {
t.Fatalf("got %+v", got)
}
if d := time.Duration(got.Hi-got.Lo) * time.Second; d != 10*24*time.Hour {
t.Fatalf("span = %s", d)
}
}
func TestBoundedDayRangeFromDatetimeAndEpoch(t *testing.T) {
dt := Spec{Kind: "day", Column: "c", ShardKind: schema.ShardDateTime, Loc: time.UTC}
if got := Bounded(context.Background(), validRange("2025-01-01 10:00:00", "2025-01-03 23:00:00"), dt, time.Second); got == nil {
t.Fatal("datetime range unavailable")
}
ep := Spec{Kind: "day", Column: "ts", ShardKind: schema.ShardEpoch, Unit: 1, Loc: time.UTC}
lo := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).Unix()
hi := time.Date(2025, 1, 4, 0, 0, 0, 0, time.UTC).Unix()
got := Bounded(context.Background(), validRange(strconv.FormatInt(lo, 10), strconv.FormatInt(hi, 10)), ep, time.Second)
if got == nil || got.Hi-got.Lo != 3*86400 {
t.Fatalf("epoch range = %+v", got)
}
}
func TestBoundedKeyRange(t *testing.T) {
spec := Spec{Kind: "key", Column: "id"}
got := Bounded(context.Background(), validRange("100", "1100"), spec, time.Second)
if got == nil || got.Kind != "key" || got.Lo != 100 || got.Hi != 1100 {
t.Fatalf("got %+v", got)
}
}
func TestBoundedSkipsUnavailableMetrics(t *testing.T) {
cases := []struct {
name string
q RangeQuery
spec Spec
}{
{"no metric", validRange("1", "10"), Spec{}},
{"query error", func(context.Context, string) (sql.NullString, sql.NullString, error) {
return sql.NullString{}, sql.NullString{}, context.DeadlineExceeded
}, Spec{Kind: "key", Column: "id"}},
{"null bounds", func(context.Context, string) (sql.NullString, sql.NullString, error) {
return sql.NullString{}, sql.NullString{}, nil
}, Spec{Kind: "key", Column: "id"}},
{"empty table", validRange("5", "5"), Spec{Kind: "key", Column: "id"}},
{"zero date", validRange("0000-00-00", "2025-01-01"), Spec{Kind: "day", ShardKind: schema.ShardDate, Loc: time.UTC}},
{"string key", validRange("a", "z"), Spec{Kind: "key", Column: "id"}},
}
for _, tc := range cases {
if got := Bounded(context.Background(), tc.q, tc.spec, time.Second); got != nil {
t.Fatalf("%s: expected nil, got %+v", tc.name, got)
}
}
}
func TestBoundedTimeoutDegrades(t *testing.T) {
blocked := func(ctx context.Context, _ string) (sql.NullString, sql.NullString, error) {
<-ctx.Done()
return sql.NullString{}, sql.NullString{}, ctx.Err()
}
start := time.Now()
got := Bounded(context.Background(), blocked, Spec{Kind: "key", Column: "id"}, 20*time.Millisecond)
if got != nil {
t.Fatalf("timed-out estimate must be nil, got %+v", got)
}
if elapsed := time.Since(start); elapsed > time.Second {
t.Fatalf("timeout took too long: %s", elapsed)
}
}