Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,350 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archer/internal/row"
|
||||
"archer/internal/schema"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
const zeroDay = "0000-00-00"
|
||||
|
||||
// DateBScanner enumerates days one at a time and streams each day's rows over
|
||||
// a dedicated connection. It is used when only a single-column index on the
|
||||
// shard column exists.
|
||||
type DateBScanner struct {
|
||||
o *Options
|
||||
conn *sql.Conn
|
||||
|
||||
phase string // null | day
|
||||
rows *sql.Rows
|
||||
cursor row.Key
|
||||
done bool
|
||||
|
||||
daysInited bool
|
||||
hasDays bool
|
||||
zeroPending bool
|
||||
zeroDone bool
|
||||
nextDay time.Time
|
||||
maxDay time.Time
|
||||
curDay string
|
||||
nextDayStr string
|
||||
|
||||
resumeDay string
|
||||
resumeKey row.Key
|
||||
resumeUse bool
|
||||
}
|
||||
|
||||
func NewDateB(ctx context.Context, o Options) (*DateBScanner, error) {
|
||||
conn, err := o.DB.Conn(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("acquire streaming connection: %w", err)
|
||||
}
|
||||
s := &DateBScanner{o: &o, conn: conn, phase: "null"}
|
||||
if o.Resume != nil {
|
||||
switch o.Resume.Phase {
|
||||
case "null":
|
||||
s.phase = "null"
|
||||
s.cursor = o.Resume.Key
|
||||
case "day":
|
||||
s.phase = "day"
|
||||
s.resumeDay = o.Resume.DayStart
|
||||
s.resumeKey = o.Resume.Key
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *DateBScanner) Next(ctx context.Context) (*row.Chunk, error) {
|
||||
for {
|
||||
if s.done {
|
||||
return nil, io.EOF
|
||||
}
|
||||
if s.rows == nil {
|
||||
if err := s.openNext(ctx); err != nil {
|
||||
if err == io.EOF {
|
||||
s.done = true
|
||||
return nil, io.EOF
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
c, exhausted, err := s.o.readChunk(s.rows, s.o.ChunkSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.o.fillBuckets(c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.Len() > 0 {
|
||||
if s.phase == "day" {
|
||||
s.cursor = s.o.compositeCursor(c.Rows[len(c.Rows)-1])
|
||||
} else {
|
||||
s.cursor = c.Keys[len(c.Keys)-1]
|
||||
}
|
||||
c.LastKey = s.cursor
|
||||
}
|
||||
if exhausted {
|
||||
if err := s.rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.rows = nil
|
||||
if s.phase == "null" {
|
||||
s.phase = "day"
|
||||
s.cursor = nil
|
||||
}
|
||||
}
|
||||
if c.Len() == 0 {
|
||||
continue
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DateBScanner) openNext(ctx context.Context) error {
|
||||
if s.phase == "null" {
|
||||
return s.openNull(ctx)
|
||||
}
|
||||
if !s.daysInited {
|
||||
if err := s.initDays(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if s.zeroPending && !s.zeroDone {
|
||||
s.zeroPending = false
|
||||
s.zeroDone = true
|
||||
s.curDay = zeroDay
|
||||
s.nextDayStr = zeroDay
|
||||
return s.openZero(ctx)
|
||||
}
|
||||
if !s.advanceDay() {
|
||||
return io.EOF
|
||||
}
|
||||
return s.openDay(ctx)
|
||||
}
|
||||
|
||||
func (s *DateBScanner) openNull(ctx context.Context) error {
|
||||
var b strings.Builder
|
||||
b.WriteString(s.o.selectSQL())
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.Ident(s.o.Shard.Column))
|
||||
b.WriteString(" IS NULL")
|
||||
var args []any
|
||||
if len(s.cursor) > 0 {
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.GreaterThan(s.o.KeyCols))
|
||||
args = append(args, s.cursor.Args()...)
|
||||
}
|
||||
b.WriteString(" ORDER BY ")
|
||||
b.WriteString(sqlutil.Columns(s.o.KeyCols))
|
||||
return s.query(ctx, b.String(), args)
|
||||
}
|
||||
|
||||
func (s *DateBScanner) openZero(ctx context.Context) error {
|
||||
var b strings.Builder
|
||||
b.WriteString(s.o.selectSQL())
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.Ident(s.o.Shard.Column))
|
||||
b.WriteString(" < ?")
|
||||
upper := "0001-01-01"
|
||||
if s.o.Shard.Kind == schema.ShardDateTime {
|
||||
upper = "0001-01-01 00:00:00"
|
||||
}
|
||||
args := []any{upper}
|
||||
cols := append([]string{s.o.Shard.Column}, s.o.KeyCols...)
|
||||
if len(s.cursor) > 0 {
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.GreaterThan(cols))
|
||||
args = append(args, s.cursor.Args()...)
|
||||
}
|
||||
b.WriteString(" ORDER BY ")
|
||||
b.WriteString(sqlutil.Columns(cols))
|
||||
s.nextDayStr = zeroDay
|
||||
return s.query(ctx, b.String(), args)
|
||||
}
|
||||
|
||||
func (s *DateBScanner) openDay(ctx context.Context) error {
|
||||
var b strings.Builder
|
||||
b.WriteString(s.o.selectSQL())
|
||||
b.WriteString(" AND ")
|
||||
col := sqlutil.Ident(s.o.Shard.Column)
|
||||
b.WriteString(col)
|
||||
b.WriteString(" >= ? AND ")
|
||||
b.WriteString(col)
|
||||
b.WriteString(" < ?")
|
||||
|
||||
var lo, hi any
|
||||
switch s.o.Shard.Kind {
|
||||
case schema.ShardDate:
|
||||
lo = s.curDay
|
||||
hi = s.nextDayStr
|
||||
case schema.ShardDateTime:
|
||||
lo = s.curDay + " 00:00:00"
|
||||
hi = s.nextDayStr + " 00:00:00"
|
||||
case schema.ShardEpoch:
|
||||
lo = schema.TimeToEpoch(s.dayTime(s.curDay), s.o.Shard.Unit)
|
||||
hi = schema.TimeToEpoch(s.dayTime(s.curDay).AddDate(0, 0, 1), s.o.Shard.Unit)
|
||||
}
|
||||
args := []any{lo, hi}
|
||||
cols := append([]string{s.o.Shard.Column}, s.o.KeyCols...)
|
||||
if len(s.cursor) > 0 {
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.GreaterThan(cols))
|
||||
args = append(args, s.cursor.Args()...)
|
||||
}
|
||||
b.WriteString(" ORDER BY ")
|
||||
b.WriteString(sqlutil.Columns(cols))
|
||||
return s.query(ctx, b.String(), args)
|
||||
}
|
||||
|
||||
func (s *DateBScanner) query(ctx context.Context, q string, args []any) error {
|
||||
rows, err := s.conn.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.rows = rows
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DateBScanner) dayTime(day string) time.Time {
|
||||
t, _ := time.ParseInLocation("2006-01-02", day, s.o.ShardLoc)
|
||||
return t
|
||||
}
|
||||
|
||||
func (s *DateBScanner) initDays(ctx context.Context) error {
|
||||
s.daysInited = true
|
||||
q := fmt.Sprintf("SELECT MIN(%s), MAX(%s) FROM %s WHERE (%s) AND %s IS NOT NULL",
|
||||
sqlutil.Ident(s.o.Shard.Column), sqlutil.Ident(s.o.Shard.Column),
|
||||
s.o.qualified(), s.o.Where, sqlutil.Ident(s.o.Shard.Column))
|
||||
var minRaw, maxRaw sql.NullString
|
||||
if err := s.conn.QueryRowContext(ctx, q).Scan(&minRaw, &maxRaw); err != nil {
|
||||
return fmt.Errorf("read shard column range: %w", err)
|
||||
}
|
||||
if !minRaw.Valid || !maxRaw.Valid {
|
||||
return nil
|
||||
}
|
||||
switch s.o.Shard.Kind {
|
||||
case schema.ShardDate, schema.ShardDateTime:
|
||||
minDay := s.first10(minRaw.String)
|
||||
maxDay := s.first10(maxRaw.String)
|
||||
zeroExists := strings.HasPrefix(minDay, "0000-")
|
||||
if zeroExists {
|
||||
minDay = "0001-01-01"
|
||||
}
|
||||
s.zeroPending = s.wantZero(zeroExists)
|
||||
if strings.HasPrefix(maxDay, "0000-") {
|
||||
return nil
|
||||
}
|
||||
minT, err := time.ParseInLocation("2006-01-02", minDay, s.o.ShardLoc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse shard range minimum %q: %w", minRaw.String, err)
|
||||
}
|
||||
maxT, err := time.ParseInLocation("2006-01-02", maxDay, s.o.ShardLoc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse shard range maximum %q: %w", maxRaw.String, err)
|
||||
}
|
||||
s.minMax(minT, maxT)
|
||||
case schema.ShardEpoch:
|
||||
minV, err := strconv.ParseInt(strings.TrimSpace(minRaw.String), 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse shard range minimum: %w", err)
|
||||
}
|
||||
maxV, err := strconv.ParseInt(strings.TrimSpace(maxRaw.String), 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse shard range maximum: %w", err)
|
||||
}
|
||||
minT := schema.EpochTime(minV, s.o.Shard.Unit, s.o.ShardLoc)
|
||||
maxT := schema.EpochTime(maxV, s.o.Shard.Unit, s.o.ShardLoc)
|
||||
s.minMax(dayStart(minT, s.o.ShardLoc), dayStart(maxT, s.o.ShardLoc))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DateBScanner) wantZero(zeroExists bool) bool {
|
||||
if s.resumeDay == "" {
|
||||
return zeroExists
|
||||
}
|
||||
return s.resumeDay == zeroDay
|
||||
}
|
||||
|
||||
func (s *DateBScanner) minMax(minT, maxT time.Time) {
|
||||
s.hasDays = true
|
||||
s.nextDay = minT
|
||||
s.maxDay = maxT
|
||||
if s.resumeDay != "" && s.resumeDay != zeroDay {
|
||||
if d, err := time.ParseInLocation("2006-01-02", s.resumeDay, s.o.ShardLoc); err == nil && d.After(minT) {
|
||||
s.nextDay = d
|
||||
}
|
||||
}
|
||||
if s.resumeDay == zeroDay {
|
||||
s.nextDay = minT
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DateBScanner) advanceDay() bool {
|
||||
if !s.hasDays {
|
||||
return false
|
||||
}
|
||||
for !s.nextDay.After(s.maxDay) {
|
||||
day := s.nextDay.Format("2006-01-02")
|
||||
if s.resumeDay != "" && s.resumeDay != zeroDay && day < s.resumeDay && !s.resumeUse {
|
||||
s.nextDay = s.nextDay.AddDate(0, 0, 1)
|
||||
continue
|
||||
}
|
||||
s.curDay = day
|
||||
s.nextDayStr = s.nextDay.AddDate(0, 0, 1).Format("2006-01-02")
|
||||
s.nextDay = s.nextDay.AddDate(0, 0, 1)
|
||||
if s.resumeDay != "" && day == s.resumeDay && !s.resumeUse {
|
||||
s.cursor = s.resumeKey
|
||||
s.resumeUse = true
|
||||
} else {
|
||||
s.cursor = nil
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *DateBScanner) first10(v string) string {
|
||||
if len(v) < 10 {
|
||||
return v
|
||||
}
|
||||
return v[:10]
|
||||
}
|
||||
|
||||
func dayStart(t time.Time, loc *time.Location) time.Time {
|
||||
return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, loc)
|
||||
}
|
||||
|
||||
func (s *DateBScanner) ResumeState() CursorState {
|
||||
phase := s.phase
|
||||
day := ""
|
||||
if phase == "day" {
|
||||
day = s.curDay
|
||||
}
|
||||
return CursorState{Phase: phase, Key: s.cursor, DayStart: day}
|
||||
}
|
||||
|
||||
func (s *DateBScanner) Close() error {
|
||||
var firstErr error
|
||||
if s.rows != nil {
|
||||
if err := s.rows.Close(); err != nil {
|
||||
firstErr = err
|
||||
}
|
||||
s.rows = nil
|
||||
}
|
||||
if s.conn != nil {
|
||||
if err := s.conn.Close(); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
s.conn = nil
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
Reference in new issue
Block a user