Initial commit: archer, a Go-native MySQL/MariaDB table archiver

Renamed from arcdb: module path, cmd/archer, internal/archererr,
docs, build scripts and ARCHER_* environment variables.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-06 22:12:17 +08:00
commit bc4a6b504f
99 files changed
+12752

No files matched your search

+350
View File
@@ -0,0 +1,350 @@
package scan
import (
"context"
"database/sql"
"fmt"
"io"
"strconv"
"strings"
"time"
"archer/internal/row"
"archer/internal/schema"
"archer/internal/sqlutil"
)
const zeroDay = "0000-00-00"
// DateBScanner enumerates days one at a time and streams each day's rows over
// a dedicated connection. It is used when only a single-column index on the
// shard column exists.
type DateBScanner struct {
o *Options
conn *sql.Conn
phase string // null | day
rows *sql.Rows
cursor row.Key
done bool
daysInited bool
hasDays bool
zeroPending bool
zeroDone bool
nextDay time.Time
maxDay time.Time
curDay string
nextDayStr string
resumeDay string
resumeKey row.Key
resumeUse bool
}
func NewDateB(ctx context.Context, o Options) (*DateBScanner, error) {
conn, err := o.DB.Conn(ctx)
if err != nil {
return nil, fmt.Errorf("acquire streaming connection: %w", err)
}
s := &DateBScanner{o: &o, conn: conn, phase: "null"}
if o.Resume != nil {
switch o.Resume.Phase {
case "null":
s.phase = "null"
s.cursor = o.Resume.Key
case "day":
s.phase = "day"
s.resumeDay = o.Resume.DayStart
s.resumeKey = o.Resume.Key
}
}
return s, nil
}
func (s *DateBScanner) Next(ctx context.Context) (*row.Chunk, error) {
for {
if s.done {
return nil, io.EOF
}
if s.rows == nil {
if err := s.openNext(ctx); err != nil {
if err == io.EOF {
s.done = true
return nil, io.EOF
}
return nil, err
}
}
c, exhausted, err := s.o.readChunk(s.rows, s.o.ChunkSize)
if err != nil {
return nil, err
}
if err := s.o.fillBuckets(c); err != nil {
return nil, err
}
if c.Len() > 0 {
if s.phase == "day" {
s.cursor = s.o.compositeCursor(c.Rows[len(c.Rows)-1])
} else {
s.cursor = c.Keys[len(c.Keys)-1]
}
c.LastKey = s.cursor
}
if exhausted {
if err := s.rows.Close(); err != nil {
return nil, err
}
s.rows = nil
if s.phase == "null" {
s.phase = "day"
s.cursor = nil
}
}
if c.Len() == 0 {
continue
}
return c, nil
}
}
func (s *DateBScanner) openNext(ctx context.Context) error {
if s.phase == "null" {
return s.openNull(ctx)
}
if !s.daysInited {
if err := s.initDays(ctx); err != nil {
return err
}
}
if s.zeroPending && !s.zeroDone {
s.zeroPending = false
s.zeroDone = true
s.curDay = zeroDay
s.nextDayStr = zeroDay
return s.openZero(ctx)
}
if !s.advanceDay() {
return io.EOF
}
return s.openDay(ctx)
}
func (s *DateBScanner) openNull(ctx context.Context) error {
var b strings.Builder
b.WriteString(s.o.selectSQL())
b.WriteString(" AND ")
b.WriteString(sqlutil.Ident(s.o.Shard.Column))
b.WriteString(" IS NULL")
var args []any
if len(s.cursor) > 0 {
b.WriteString(" AND ")
b.WriteString(sqlutil.GreaterThan(s.o.KeyCols))
args = append(args, s.cursor.Args()...)
}
b.WriteString(" ORDER BY ")
b.WriteString(sqlutil.Columns(s.o.KeyCols))
return s.query(ctx, b.String(), args)
}
func (s *DateBScanner) openZero(ctx context.Context) error {
var b strings.Builder
b.WriteString(s.o.selectSQL())
b.WriteString(" AND ")
b.WriteString(sqlutil.Ident(s.o.Shard.Column))
b.WriteString(" < ?")
upper := "0001-01-01"
if s.o.Shard.Kind == schema.ShardDateTime {
upper = "0001-01-01 00:00:00"
}
args := []any{upper}
cols := append([]string{s.o.Shard.Column}, s.o.KeyCols...)
if len(s.cursor) > 0 {
b.WriteString(" AND ")
b.WriteString(sqlutil.GreaterThan(cols))
args = append(args, s.cursor.Args()...)
}
b.WriteString(" ORDER BY ")
b.WriteString(sqlutil.Columns(cols))
s.nextDayStr = zeroDay
return s.query(ctx, b.String(), args)
}
func (s *DateBScanner) openDay(ctx context.Context) error {
var b strings.Builder
b.WriteString(s.o.selectSQL())
b.WriteString(" AND ")
col := sqlutil.Ident(s.o.Shard.Column)
b.WriteString(col)
b.WriteString(" >= ? AND ")
b.WriteString(col)
b.WriteString(" < ?")
var lo, hi any
switch s.o.Shard.Kind {
case schema.ShardDate:
lo = s.curDay
hi = s.nextDayStr
case schema.ShardDateTime:
lo = s.curDay + " 00:00:00"
hi = s.nextDayStr + " 00:00:00"
case schema.ShardEpoch:
lo = schema.TimeToEpoch(s.dayTime(s.curDay), s.o.Shard.Unit)
hi = schema.TimeToEpoch(s.dayTime(s.curDay).AddDate(0, 0, 1), s.o.Shard.Unit)
}
args := []any{lo, hi}
cols := append([]string{s.o.Shard.Column}, s.o.KeyCols...)
if len(s.cursor) > 0 {
b.WriteString(" AND ")
b.WriteString(sqlutil.GreaterThan(cols))
args = append(args, s.cursor.Args()...)
}
b.WriteString(" ORDER BY ")
b.WriteString(sqlutil.Columns(cols))
return s.query(ctx, b.String(), args)
}
func (s *DateBScanner) query(ctx context.Context, q string, args []any) error {
rows, err := s.conn.QueryContext(ctx, q, args...)
if err != nil {
return err
}
s.rows = rows
return nil
}
func (s *DateBScanner) dayTime(day string) time.Time {
t, _ := time.ParseInLocation("2006-01-02", day, s.o.ShardLoc)
return t
}
func (s *DateBScanner) initDays(ctx context.Context) error {
s.daysInited = true
q := fmt.Sprintf("SELECT MIN(%s), MAX(%s) FROM %s WHERE (%s) AND %s IS NOT NULL",
sqlutil.Ident(s.o.Shard.Column), sqlutil.Ident(s.o.Shard.Column),
s.o.qualified(), s.o.Where, sqlutil.Ident(s.o.Shard.Column))
var minRaw, maxRaw sql.NullString
if err := s.conn.QueryRowContext(ctx, q).Scan(&minRaw, &maxRaw); err != nil {
return fmt.Errorf("read shard column range: %w", err)
}
if !minRaw.Valid || !maxRaw.Valid {
return nil
}
switch s.o.Shard.Kind {
case schema.ShardDate, schema.ShardDateTime:
minDay := s.first10(minRaw.String)
maxDay := s.first10(maxRaw.String)
zeroExists := strings.HasPrefix(minDay, "0000-")
if zeroExists {
minDay = "0001-01-01"
}
s.zeroPending = s.wantZero(zeroExists)
if strings.HasPrefix(maxDay, "0000-") {
return nil
}
minT, err := time.ParseInLocation("2006-01-02", minDay, s.o.ShardLoc)
if err != nil {
return fmt.Errorf("parse shard range minimum %q: %w", minRaw.String, err)
}
maxT, err := time.ParseInLocation("2006-01-02", maxDay, s.o.ShardLoc)
if err != nil {
return fmt.Errorf("parse shard range maximum %q: %w", maxRaw.String, err)
}
s.minMax(minT, maxT)
case schema.ShardEpoch:
minV, err := strconv.ParseInt(strings.TrimSpace(minRaw.String), 10, 64)
if err != nil {
return fmt.Errorf("parse shard range minimum: %w", err)
}
maxV, err := strconv.ParseInt(strings.TrimSpace(maxRaw.String), 10, 64)
if err != nil {
return fmt.Errorf("parse shard range maximum: %w", err)
}
minT := schema.EpochTime(minV, s.o.Shard.Unit, s.o.ShardLoc)
maxT := schema.EpochTime(maxV, s.o.Shard.Unit, s.o.ShardLoc)
s.minMax(dayStart(minT, s.o.ShardLoc), dayStart(maxT, s.o.ShardLoc))
}
return nil
}
func (s *DateBScanner) wantZero(zeroExists bool) bool {
if s.resumeDay == "" {
return zeroExists
}
return s.resumeDay == zeroDay
}
func (s *DateBScanner) minMax(minT, maxT time.Time) {
s.hasDays = true
s.nextDay = minT
s.maxDay = maxT
if s.resumeDay != "" && s.resumeDay != zeroDay {
if d, err := time.ParseInLocation("2006-01-02", s.resumeDay, s.o.ShardLoc); err == nil && d.After(minT) {
s.nextDay = d
}
}
if s.resumeDay == zeroDay {
s.nextDay = minT
}
}
func (s *DateBScanner) advanceDay() bool {
if !s.hasDays {
return false
}
for !s.nextDay.After(s.maxDay) {
day := s.nextDay.Format("2006-01-02")
if s.resumeDay != "" && s.resumeDay != zeroDay && day < s.resumeDay && !s.resumeUse {
s.nextDay = s.nextDay.AddDate(0, 0, 1)
continue
}
s.curDay = day
s.nextDayStr = s.nextDay.AddDate(0, 0, 1).Format("2006-01-02")
s.nextDay = s.nextDay.AddDate(0, 0, 1)
if s.resumeDay != "" && day == s.resumeDay && !s.resumeUse {
s.cursor = s.resumeKey
s.resumeUse = true
} else {
s.cursor = nil
}
return true
}
return false
}
func (s *DateBScanner) first10(v string) string {
if len(v) < 10 {
return v
}
return v[:10]
}
func dayStart(t time.Time, loc *time.Location) time.Time {
return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, loc)
}
func (s *DateBScanner) ResumeState() CursorState {
phase := s.phase
day := ""
if phase == "day" {
day = s.curDay
}
return CursorState{Phase: phase, Key: s.cursor, DayStart: day}
}
func (s *DateBScanner) Close() error {
var firstErr error
if s.rows != nil {
if err := s.rows.Close(); err != nil {
firstErr = err
}
s.rows = nil
}
if s.conn != nil {
if err := s.conn.Close(); err != nil && firstErr == nil {
firstErr = err
}
s.conn = nil
}
return firstErr
}