Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,122 @@
|
||||
// Package estimate derives a cheap total-size estimate for progress display.
|
||||
//
|
||||
// Only bounded MIN/MAX queries are used; COUNT(*) is never attempted. A query
|
||||
// that fails, times out, or whose metric does not apply silently yields no
|
||||
// estimate, leaving the caller to render rows and rate only.
|
||||
package estimate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archer/internal/progress"
|
||||
"archer/internal/row"
|
||||
"archer/internal/schema"
|
||||
)
|
||||
|
||||
// Spec describes which metric a run can expose.
|
||||
type Spec struct {
|
||||
Kind string // "day", "key" or "" for none
|
||||
Column string
|
||||
ShardKind schema.ShardKind
|
||||
Unit int64
|
||||
Loc *time.Location
|
||||
}
|
||||
|
||||
// RangeQuery returns MIN and MAX of a column as raw text. Implementations must
|
||||
// honor ctx so the bounded timeout is effective.
|
||||
type RangeQuery func(ctx context.Context, column string) (minRaw, maxRaw sql.NullString, err error)
|
||||
|
||||
// Bounded runs q under a timeout and converts the result into a display
|
||||
// estimate. It returns nil instead of an error on every failure path.
|
||||
func Bounded(ctx context.Context, q RangeQuery, spec Spec, timeout time.Duration) *progress.Estimate {
|
||||
if q == nil || spec.Kind == "" || timeout <= 0 {
|
||||
return nil
|
||||
}
|
||||
qctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
minRaw, maxRaw, err := q(qctx, spec.Column)
|
||||
if err != nil || !minRaw.Valid || !maxRaw.Valid {
|
||||
return nil
|
||||
}
|
||||
|
||||
var lo, hi float64
|
||||
var ok bool
|
||||
switch spec.Kind {
|
||||
case "day":
|
||||
lo, hi, ok = dayRange(minRaw.String, maxRaw.String, spec.ShardKind, spec.Unit, spec.Loc)
|
||||
case "key":
|
||||
lo, hi, ok = keyRange(minRaw.String, maxRaw.String)
|
||||
}
|
||||
if !ok || hi <= lo {
|
||||
return nil
|
||||
}
|
||||
return &progress.Estimate{Kind: spec.Kind, Lo: lo, Hi: hi}
|
||||
}
|
||||
|
||||
// KeyColumn reports the scan key column when it is a single integer column,
|
||||
// which is the only non-date shape a cheap key-range estimate applies to.
|
||||
func KeyColumn(cols, types []string) (string, bool) {
|
||||
if len(cols) != 1 || len(types) != 1 {
|
||||
return "", false
|
||||
}
|
||||
if !row.IsInteger(types[0]) {
|
||||
return "", false
|
||||
}
|
||||
return cols[0], true
|
||||
}
|
||||
|
||||
func dayRange(minRaw, maxRaw string, kind schema.ShardKind, unit int64, loc *time.Location) (float64, float64, bool) {
|
||||
if loc == nil {
|
||||
loc = time.UTC
|
||||
}
|
||||
switch kind {
|
||||
case schema.ShardDate, schema.ShardDateTime:
|
||||
lo, ok1 := daySeconds(minRaw, loc)
|
||||
hi, ok2 := daySeconds(maxRaw, loc)
|
||||
return lo, hi, ok1 && ok2
|
||||
case schema.ShardEpoch:
|
||||
minV, err1 := strconv.ParseInt(strings.TrimSpace(minRaw), 10, 64)
|
||||
maxV, err2 := strconv.ParseInt(strings.TrimSpace(maxRaw), 10, 64)
|
||||
if err1 != nil || err2 != nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
lo := dayStart(schema.EpochTime(minV, unit, loc), loc).Unix()
|
||||
hi := dayStart(schema.EpochTime(maxV, unit, loc), loc).Unix()
|
||||
return float64(lo), float64(hi), true
|
||||
}
|
||||
return 0, 0, false
|
||||
}
|
||||
|
||||
func daySeconds(raw string, loc *time.Location) (float64, bool) {
|
||||
s := strings.TrimSpace(raw)
|
||||
if len(s) < 10 {
|
||||
return 0, false
|
||||
}
|
||||
day := s[:10]
|
||||
if strings.HasPrefix(day, "0000-") {
|
||||
return 0, false
|
||||
}
|
||||
t, err := time.ParseInLocation("2006-01-02", day, loc)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return float64(t.Unix()), true
|
||||
}
|
||||
|
||||
func keyRange(minRaw, maxRaw string) (float64, float64, bool) {
|
||||
minV, err1 := strconv.ParseInt(strings.TrimSpace(minRaw), 10, 64)
|
||||
maxV, err2 := strconv.ParseInt(strings.TrimSpace(maxRaw), 10, 64)
|
||||
if err1 != nil || err2 != nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
return float64(minV), float64(maxV), true
|
||||
}
|
||||
|
||||
func dayStart(t time.Time, loc *time.Location) time.Time {
|
||||
return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, loc)
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package estimate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"archer/internal/schema"
|
||||
)
|
||||
|
||||
func validRange(minRaw, maxRaw string) RangeQuery {
|
||||
return func(context.Context, string) (sql.NullString, sql.NullString, error) {
|
||||
return sql.NullString{String: minRaw, Valid: true}, sql.NullString{String: maxRaw, Valid: true}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyColumnApplicability(t *testing.T) {
|
||||
cases := []struct {
|
||||
cols, types []string
|
||||
want bool
|
||||
}{
|
||||
{[]string{"id"}, []string{"bigint"}, true},
|
||||
{[]string{"id"}, []string{"int"}, true},
|
||||
{[]string{"a", "b"}, []string{"bigint", "bigint"}, false},
|
||||
{[]string{"code"}, []string{"varchar"}, false},
|
||||
{[]string{"payload"}, []string{"blob"}, false},
|
||||
{[]string{"bin"}, []string{"varbinary"}, false},
|
||||
{[]string{"ts"}, []string{"datetime"}, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
col, ok := KeyColumn(tc.cols, tc.types)
|
||||
if ok != tc.want {
|
||||
t.Fatalf("%v/%v: got %v want %v", tc.cols, tc.types, ok, tc.want)
|
||||
}
|
||||
if ok && col != tc.cols[0] {
|
||||
t.Fatalf("%v: wrong column %q", tc.cols, col)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedDayRangeFromDates(t *testing.T) {
|
||||
spec := Spec{Kind: "day", Column: "created_at", ShardKind: schema.ShardDate, Loc: time.UTC}
|
||||
got := Bounded(context.Background(), validRange("2025-01-01", "2025-01-11"), spec, time.Second)
|
||||
if got == nil || got.Kind != "day" {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
if d := time.Duration(got.Hi-got.Lo) * time.Second; d != 10*24*time.Hour {
|
||||
t.Fatalf("span = %s", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedDayRangeFromDatetimeAndEpoch(t *testing.T) {
|
||||
dt := Spec{Kind: "day", Column: "c", ShardKind: schema.ShardDateTime, Loc: time.UTC}
|
||||
if got := Bounded(context.Background(), validRange("2025-01-01 10:00:00", "2025-01-03 23:00:00"), dt, time.Second); got == nil {
|
||||
t.Fatal("datetime range unavailable")
|
||||
}
|
||||
ep := Spec{Kind: "day", Column: "ts", ShardKind: schema.ShardEpoch, Unit: 1, Loc: time.UTC}
|
||||
lo := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).Unix()
|
||||
hi := time.Date(2025, 1, 4, 0, 0, 0, 0, time.UTC).Unix()
|
||||
got := Bounded(context.Background(), validRange(strconv.FormatInt(lo, 10), strconv.FormatInt(hi, 10)), ep, time.Second)
|
||||
if got == nil || got.Hi-got.Lo != 3*86400 {
|
||||
t.Fatalf("epoch range = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedKeyRange(t *testing.T) {
|
||||
spec := Spec{Kind: "key", Column: "id"}
|
||||
got := Bounded(context.Background(), validRange("100", "1100"), spec, time.Second)
|
||||
if got == nil || got.Kind != "key" || got.Lo != 100 || got.Hi != 1100 {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedSkipsUnavailableMetrics(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
q RangeQuery
|
||||
spec Spec
|
||||
}{
|
||||
{"no metric", validRange("1", "10"), Spec{}},
|
||||
{"query error", func(context.Context, string) (sql.NullString, sql.NullString, error) {
|
||||
return sql.NullString{}, sql.NullString{}, context.DeadlineExceeded
|
||||
}, Spec{Kind: "key", Column: "id"}},
|
||||
{"null bounds", func(context.Context, string) (sql.NullString, sql.NullString, error) {
|
||||
return sql.NullString{}, sql.NullString{}, nil
|
||||
}, Spec{Kind: "key", Column: "id"}},
|
||||
{"empty table", validRange("5", "5"), Spec{Kind: "key", Column: "id"}},
|
||||
{"zero date", validRange("0000-00-00", "2025-01-01"), Spec{Kind: "day", ShardKind: schema.ShardDate, Loc: time.UTC}},
|
||||
{"string key", validRange("a", "z"), Spec{Kind: "key", Column: "id"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := Bounded(context.Background(), tc.q, tc.spec, time.Second); got != nil {
|
||||
t.Fatalf("%s: expected nil, got %+v", tc.name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedTimeoutDegrades(t *testing.T) {
|
||||
blocked := func(ctx context.Context, _ string) (sql.NullString, sql.NullString, error) {
|
||||
<-ctx.Done()
|
||||
return sql.NullString{}, sql.NullString{}, ctx.Err()
|
||||
}
|
||||
start := time.Now()
|
||||
got := Bounded(context.Background(), blocked, Spec{Kind: "key", Column: "id"}, 20*time.Millisecond)
|
||||
if got != nil {
|
||||
t.Fatalf("timed-out estimate must be nil, got %+v", got)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed > time.Second {
|
||||
t.Fatalf("timeout took too long: %s", elapsed)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user