Files
archer/internal/schema/date.go
T
voidandClaude Sonnet 4.6 bc4a6b504f Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr,
docs, build scripts and ARCHER_* environment variables.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-11-06 22:12:17 +08:00

220 lines
7.0 KiB
Go

package schema
import (
"context"
"database/sql"
"fmt"
"strconv"
"strings"
"time"
"archer/internal/row"
"archer/internal/sqlutil"
)
// ShardKind is how a shard column's date is derived from its raw value.
type ShardKind int
const (
ShardNone ShardKind = iota
ShardDate
ShardDateTime
ShardEpoch
)
// ShardSpec is the resolved shard column and how to interpret it.
type ShardSpec struct {
Column string
Kind ShardKind
Unit int64 // epoch multiplier to nanoseconds: 1e9, 1e6, 1e3
Type string
}
// ResolveShardColumn validates --shard-column against the table and the unit
// flag. It returns a descriptive error listing candidates when the column is
// missing or the type is unsupported.
func ResolveShardColumn(ctx context.Context, db *sql.DB, t *Table, column, unitFlag, where string, loc *time.Location) (*ShardSpec, error) {
if column == "" {
return nil, fmt.Errorf("--shard-column is required for --shard-by date\ncandidate columns:\n%s", describeTimeColumns(t))
}
c, ok := t.Column(column)
if !ok {
return nil, fmt.Errorf("shard column %q does not exist; candidate columns:\n%s", column, describeTimeColumns(t))
}
switch c.DataType {
case "date":
return &ShardSpec{Column: c.Name, Kind: ShardDate, Unit: 1, Type: c.DataType}, nil
case "datetime", "timestamp":
return &ShardSpec{Column: c.Name, Kind: ShardDateTime, Unit: 1, Type: c.DataType}, nil
}
if row.IsInteger(c.DataType) {
unit, err := ParseEpochUnit(unitFlag)
if err != nil {
minV, maxV, ok, serr := SampleEpochRange(ctx, db, t, c.Name, where)
if serr != nil {
return nil, fmt.Errorf("%w (also failed to sample %s: %v)", err, c.Name, serr)
}
if !ok {
return nil, fmt.Errorf("%w; column %s has no non-NULL values to sample", err, c.Name)
}
return nil, fmt.Errorf("%w\nsampled %s range: %d .. %d\n seconds: %s .. %s\n milliseconds: %s .. %s\n microseconds: %s .. %s",
err, c.Name, minV, maxV,
epochString(minV, 1, loc), epochString(maxV, 1, loc),
epochString(minV, 1_000, loc), epochString(maxV, 1_000, loc),
epochString(minV, 1_000_000, loc), epochString(maxV, 1_000_000, loc))
}
return &ShardSpec{Column: c.Name, Kind: ShardEpoch, Unit: unit, Type: c.DataType}, nil
}
return nil, fmt.Errorf("column %q has unsupported type %s; v1 supports DATE, DATETIME, TIMESTAMP and integer epoch columns with --shard-column-type\ncandidate columns:\n%s",
c.Name, c.DataType, describeTimeColumns(t))
}
func describeTimeColumns(t *Table) string {
var b strings.Builder
found := false
for _, c := range t.TimeColumns() {
found = true
fmt.Fprintf(&b, " - %s %s%s\n", c.Name, c.ColumnType, nullableSuffix(c))
}
for _, c := range t.Columns {
if row.IsInteger(c.DataType) {
found = true
fmt.Fprintf(&b, " - %s %s (integer; requires --shard-column-type)%s\n", c.Name, c.ColumnType, nullableSuffix(c))
}
}
if !found {
return " (none)"
}
return strings.TrimRight(b.String(), "\n")
}
func nullableSuffix(c Column) string {
if c.Nullable {
return " NULL"
}
return " NOT NULL"
}
// ParseEpochUnit maps --shard-column-type to raw units per second: 1 for
// seconds, 1000 for milliseconds, 1000000 for microseconds.
func ParseEpochUnit(flag string) (int64, error) {
switch strings.ToLower(strings.TrimSpace(flag)) {
case "epoch-seconds", "seconds", "second", "s", "1":
return 1, nil
case "epoch-milliseconds", "epoch-millis", "milliseconds", "millisecond", "ms", "1000":
return 1_000, nil
case "epoch-microseconds", "epoch-micros", "microseconds", "microsecond", "us", "1000000":
return 1_000_000, nil
case "":
return 0, fmt.Errorf("integer shard column requires --shard-column-type (epoch-seconds, epoch-milliseconds or epoch-microseconds)")
default:
return 0, fmt.Errorf("unsupported --shard-column-type %q; use epoch-seconds, epoch-milliseconds or epoch-microseconds", flag)
}
}
// SampleEpochRange returns MIN/MAX of an integer column under the base where.
func SampleEpochRange(ctx context.Context, db *sql.DB, t *Table, col, where string) (int64, int64, bool, error) {
q := fmt.Sprintf("SELECT MIN(%s), MAX(%s) FROM %s", sqlutil.Ident(col), sqlutil.Ident(col), sqlutil.Qualified(t.DB, t.Name))
if strings.TrimSpace(where) != "" {
q += " WHERE (" + where + ")"
}
var minRaw, maxRaw sql.NullString
if err := db.QueryRowContext(ctx, q).Scan(&minRaw, &maxRaw); err != nil {
return 0, 0, false, err
}
if !minRaw.Valid || !maxRaw.Valid {
return 0, 0, false, nil
}
minV, err := strconv.ParseInt(strings.TrimSpace(minRaw.String), 10, 64)
if err != nil {
return 0, 0, false, fmt.Errorf("parse min %q: %w", minRaw.String, err)
}
maxV, err := strconv.ParseInt(strings.TrimSpace(maxRaw.String), 10, 64)
if err != nil {
return 0, 0, false, fmt.Errorf("parse max %q: %w", maxRaw.String, err)
}
return minV, maxV, true, nil
}
func epochString(v, unit int64, loc *time.Location) string {
return EpochTime(v, unit, loc).Format("2006-01-02 15:04:05")
}
// EpochTime converts an epoch value with the given nanoseconds-per-unit to a
// time in loc.
func EpochTime(v, unit int64, loc *time.Location) time.Time {
sec := v / unit
frac := v % unit
if frac < 0 {
sec--
frac += unit
}
nsec := frac * (1_000_000_000 / unit)
return time.Unix(sec, nsec).In(loc)
}
// TimeToEpoch is the inverse of EpochTime.
func TimeToEpoch(t time.Time, unit int64) int64 {
return t.Unix()*unit + int64(t.Nanosecond())/(1_000_000_000/unit)
}
const nullBucket = "_null"
const zeroBucket = "0000-00-00"
// BucketOf maps a raw shard value to its date bucket. key is used only in
// error messages.
func BucketOf(v row.Value, spec *ShardSpec, loc *time.Location, zeroDate string, key row.Key) (string, error) {
if v.IsNull() {
return nullBucket, nil
}
switch spec.Kind {
case ShardEpoch:
n, err := strconv.ParseInt(strings.TrimSpace(string(v.Raw)), 10, 64)
if err != nil {
return "", fmt.Errorf("row %s: cannot parse epoch value %q in column %s: %w", key, string(v.Raw), spec.Column, err)
}
return EpochTime(n, spec.Unit, loc).Format("2006-01-02"), nil
case ShardDate, ShardDateTime:
s := string(v.Raw)
if len(s) < 10 {
return "", fmt.Errorf("row %s: cannot parse date value %q in column %s", key, s, spec.Column)
}
day := s[:10]
if strings.HasPrefix(day, "0000-") {
if zeroDate == "bucket" {
return zeroBucket, nil
}
return "", fmt.Errorf("row %s has zero date %q in column %s; pass --zero-date bucket to route it to date=0000-00-00", key, s, spec.Column)
}
if _, err := time.Parse("2006-01-02", day); err != nil {
return "", fmt.Errorf("row %s: invalid date %q in column %s", key, s, spec.Column)
}
return day, nil
}
return "", fmt.Errorf("internal: shard kind not resolved")
}
// BucketLess orders buckets in scan order: NULL first, then zero dates, then
// real dates.
func BucketLess(a, b string) bool {
ca, ka := bucketClass(a)
cb, kb := bucketClass(b)
if ca != cb {
return ca < cb
}
return ka < kb
}
func bucketClass(b string) (int, string) {
switch b {
case "":
return -1, ""
case nullBucket:
return 0, ""
case zeroBucket:
return 1, ""
default:
return 2, b
}
}