Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,148 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"archer/internal/row"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
// DateAScanner uses an index that starts with (shard_column, key...) and
|
||||
// performs keyset pagination over it. NULL rows are scanned in a separate
|
||||
// first phase because row-constructor comparisons do not advance past NULL.
|
||||
type DateAScanner struct {
|
||||
o *Options
|
||||
phase string // null | value
|
||||
cursor row.Key
|
||||
done bool
|
||||
}
|
||||
|
||||
func NewDateA(o Options) *DateAScanner {
|
||||
s := &DateAScanner{o: &o, phase: "null"}
|
||||
if o.Resume != nil {
|
||||
switch o.Resume.Phase {
|
||||
case "null":
|
||||
s.phase = "null"
|
||||
s.cursor = o.Resume.Key
|
||||
case "value":
|
||||
s.phase = "value"
|
||||
s.cursor = o.Resume.Key
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *DateAScanner) Next(ctx context.Context) (*row.Chunk, error) {
|
||||
for {
|
||||
if s.done {
|
||||
return nil, io.EOF
|
||||
}
|
||||
var (
|
||||
c *row.Chunk
|
||||
exhausted bool
|
||||
err error
|
||||
)
|
||||
if s.phase == "null" {
|
||||
c, exhausted, err = s.queryNull(ctx)
|
||||
} else {
|
||||
c, exhausted, err = s.queryValue(ctx)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.Len() == 0 {
|
||||
if s.phase == "null" {
|
||||
s.phase = "value"
|
||||
s.cursor = nil
|
||||
continue
|
||||
}
|
||||
s.done = true
|
||||
return nil, io.EOF
|
||||
}
|
||||
if s.phase == "null" {
|
||||
if exhausted {
|
||||
s.phase = "value"
|
||||
s.cursor = nil
|
||||
} else {
|
||||
s.cursor = c.LastKey
|
||||
}
|
||||
} else {
|
||||
s.cursor = s.o.compositeCursor(c.Rows[len(c.Rows)-1])
|
||||
c.LastKey = s.cursor
|
||||
if exhausted {
|
||||
s.done = true
|
||||
}
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DateAScanner) queryNull(ctx context.Context) (*row.Chunk, bool, error) {
|
||||
var b strings.Builder
|
||||
b.WriteString(s.o.selectSQL())
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.Ident(s.o.Shard.Column))
|
||||
b.WriteString(" IS NULL")
|
||||
var args []any
|
||||
if len(s.cursor) > 0 {
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.GreaterThan(s.o.KeyCols))
|
||||
args = append(args, s.cursor.Args()...)
|
||||
}
|
||||
b.WriteString(" ORDER BY ")
|
||||
b.WriteString(sqlutil.Columns(s.o.KeyCols))
|
||||
b.WriteString(" LIMIT ?")
|
||||
args = append(args, s.o.ChunkSize)
|
||||
return s.run(ctx, b.String(), args)
|
||||
}
|
||||
|
||||
func (s *DateAScanner) queryValue(ctx context.Context) (*row.Chunk, bool, error) {
|
||||
var b strings.Builder
|
||||
b.WriteString(s.o.selectSQL())
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.Ident(s.o.Shard.Column))
|
||||
b.WriteString(" IS NOT NULL")
|
||||
cols := append([]string{s.o.Shard.Column}, s.o.KeyCols...)
|
||||
var args []any
|
||||
if len(s.cursor) > 0 {
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.GreaterThan(cols))
|
||||
args = append(args, s.cursor.Args()...)
|
||||
}
|
||||
b.WriteString(" ORDER BY ")
|
||||
b.WriteString(sqlutil.Columns(cols))
|
||||
b.WriteString(" LIMIT ?")
|
||||
args = append(args, s.o.ChunkSize)
|
||||
c, exhausted, err := s.run(ctx, b.String(), args)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if c.Len() > 0 {
|
||||
c.LastKey = s.o.compositeCursor(c.Rows[len(c.Rows)-1])
|
||||
}
|
||||
return c, exhausted, nil
|
||||
}
|
||||
|
||||
func (s *DateAScanner) run(ctx context.Context, q string, args []any) (*row.Chunk, bool, error) {
|
||||
rows, err := s.o.DB.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
c, exhausted, err := s.o.readChunk(rows, s.o.ChunkSize)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if err := s.o.fillBuckets(c); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return c, exhausted, nil
|
||||
}
|
||||
|
||||
func (s *DateAScanner) ResumeState() CursorState {
|
||||
return CursorState{Phase: s.phase, Key: s.cursor}
|
||||
}
|
||||
|
||||
func (s *DateAScanner) Close() error { return nil }
|
||||
@@ -0,0 +1,350 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archer/internal/row"
|
||||
"archer/internal/schema"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
const zeroDay = "0000-00-00"
|
||||
|
||||
// DateBScanner enumerates days one at a time and streams each day's rows over
|
||||
// a dedicated connection. It is used when only a single-column index on the
|
||||
// shard column exists.
|
||||
type DateBScanner struct {
|
||||
o *Options
|
||||
conn *sql.Conn
|
||||
|
||||
phase string // null | day
|
||||
rows *sql.Rows
|
||||
cursor row.Key
|
||||
done bool
|
||||
|
||||
daysInited bool
|
||||
hasDays bool
|
||||
zeroPending bool
|
||||
zeroDone bool
|
||||
nextDay time.Time
|
||||
maxDay time.Time
|
||||
curDay string
|
||||
nextDayStr string
|
||||
|
||||
resumeDay string
|
||||
resumeKey row.Key
|
||||
resumeUse bool
|
||||
}
|
||||
|
||||
func NewDateB(ctx context.Context, o Options) (*DateBScanner, error) {
|
||||
conn, err := o.DB.Conn(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("acquire streaming connection: %w", err)
|
||||
}
|
||||
s := &DateBScanner{o: &o, conn: conn, phase: "null"}
|
||||
if o.Resume != nil {
|
||||
switch o.Resume.Phase {
|
||||
case "null":
|
||||
s.phase = "null"
|
||||
s.cursor = o.Resume.Key
|
||||
case "day":
|
||||
s.phase = "day"
|
||||
s.resumeDay = o.Resume.DayStart
|
||||
s.resumeKey = o.Resume.Key
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *DateBScanner) Next(ctx context.Context) (*row.Chunk, error) {
|
||||
for {
|
||||
if s.done {
|
||||
return nil, io.EOF
|
||||
}
|
||||
if s.rows == nil {
|
||||
if err := s.openNext(ctx); err != nil {
|
||||
if err == io.EOF {
|
||||
s.done = true
|
||||
return nil, io.EOF
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
c, exhausted, err := s.o.readChunk(s.rows, s.o.ChunkSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.o.fillBuckets(c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.Len() > 0 {
|
||||
if s.phase == "day" {
|
||||
s.cursor = s.o.compositeCursor(c.Rows[len(c.Rows)-1])
|
||||
} else {
|
||||
s.cursor = c.Keys[len(c.Keys)-1]
|
||||
}
|
||||
c.LastKey = s.cursor
|
||||
}
|
||||
if exhausted {
|
||||
if err := s.rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.rows = nil
|
||||
if s.phase == "null" {
|
||||
s.phase = "day"
|
||||
s.cursor = nil
|
||||
}
|
||||
}
|
||||
if c.Len() == 0 {
|
||||
continue
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DateBScanner) openNext(ctx context.Context) error {
|
||||
if s.phase == "null" {
|
||||
return s.openNull(ctx)
|
||||
}
|
||||
if !s.daysInited {
|
||||
if err := s.initDays(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if s.zeroPending && !s.zeroDone {
|
||||
s.zeroPending = false
|
||||
s.zeroDone = true
|
||||
s.curDay = zeroDay
|
||||
s.nextDayStr = zeroDay
|
||||
return s.openZero(ctx)
|
||||
}
|
||||
if !s.advanceDay() {
|
||||
return io.EOF
|
||||
}
|
||||
return s.openDay(ctx)
|
||||
}
|
||||
|
||||
func (s *DateBScanner) openNull(ctx context.Context) error {
|
||||
var b strings.Builder
|
||||
b.WriteString(s.o.selectSQL())
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.Ident(s.o.Shard.Column))
|
||||
b.WriteString(" IS NULL")
|
||||
var args []any
|
||||
if len(s.cursor) > 0 {
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.GreaterThan(s.o.KeyCols))
|
||||
args = append(args, s.cursor.Args()...)
|
||||
}
|
||||
b.WriteString(" ORDER BY ")
|
||||
b.WriteString(sqlutil.Columns(s.o.KeyCols))
|
||||
return s.query(ctx, b.String(), args)
|
||||
}
|
||||
|
||||
func (s *DateBScanner) openZero(ctx context.Context) error {
|
||||
var b strings.Builder
|
||||
b.WriteString(s.o.selectSQL())
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.Ident(s.o.Shard.Column))
|
||||
b.WriteString(" < ?")
|
||||
upper := "0001-01-01"
|
||||
if s.o.Shard.Kind == schema.ShardDateTime {
|
||||
upper = "0001-01-01 00:00:00"
|
||||
}
|
||||
args := []any{upper}
|
||||
cols := append([]string{s.o.Shard.Column}, s.o.KeyCols...)
|
||||
if len(s.cursor) > 0 {
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.GreaterThan(cols))
|
||||
args = append(args, s.cursor.Args()...)
|
||||
}
|
||||
b.WriteString(" ORDER BY ")
|
||||
b.WriteString(sqlutil.Columns(cols))
|
||||
s.nextDayStr = zeroDay
|
||||
return s.query(ctx, b.String(), args)
|
||||
}
|
||||
|
||||
func (s *DateBScanner) openDay(ctx context.Context) error {
|
||||
var b strings.Builder
|
||||
b.WriteString(s.o.selectSQL())
|
||||
b.WriteString(" AND ")
|
||||
col := sqlutil.Ident(s.o.Shard.Column)
|
||||
b.WriteString(col)
|
||||
b.WriteString(" >= ? AND ")
|
||||
b.WriteString(col)
|
||||
b.WriteString(" < ?")
|
||||
|
||||
var lo, hi any
|
||||
switch s.o.Shard.Kind {
|
||||
case schema.ShardDate:
|
||||
lo = s.curDay
|
||||
hi = s.nextDayStr
|
||||
case schema.ShardDateTime:
|
||||
lo = s.curDay + " 00:00:00"
|
||||
hi = s.nextDayStr + " 00:00:00"
|
||||
case schema.ShardEpoch:
|
||||
lo = schema.TimeToEpoch(s.dayTime(s.curDay), s.o.Shard.Unit)
|
||||
hi = schema.TimeToEpoch(s.dayTime(s.curDay).AddDate(0, 0, 1), s.o.Shard.Unit)
|
||||
}
|
||||
args := []any{lo, hi}
|
||||
cols := append([]string{s.o.Shard.Column}, s.o.KeyCols...)
|
||||
if len(s.cursor) > 0 {
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.GreaterThan(cols))
|
||||
args = append(args, s.cursor.Args()...)
|
||||
}
|
||||
b.WriteString(" ORDER BY ")
|
||||
b.WriteString(sqlutil.Columns(cols))
|
||||
return s.query(ctx, b.String(), args)
|
||||
}
|
||||
|
||||
func (s *DateBScanner) query(ctx context.Context, q string, args []any) error {
|
||||
rows, err := s.conn.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.rows = rows
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DateBScanner) dayTime(day string) time.Time {
|
||||
t, _ := time.ParseInLocation("2006-01-02", day, s.o.ShardLoc)
|
||||
return t
|
||||
}
|
||||
|
||||
func (s *DateBScanner) initDays(ctx context.Context) error {
|
||||
s.daysInited = true
|
||||
q := fmt.Sprintf("SELECT MIN(%s), MAX(%s) FROM %s WHERE (%s) AND %s IS NOT NULL",
|
||||
sqlutil.Ident(s.o.Shard.Column), sqlutil.Ident(s.o.Shard.Column),
|
||||
s.o.qualified(), s.o.Where, sqlutil.Ident(s.o.Shard.Column))
|
||||
var minRaw, maxRaw sql.NullString
|
||||
if err := s.conn.QueryRowContext(ctx, q).Scan(&minRaw, &maxRaw); err != nil {
|
||||
return fmt.Errorf("read shard column range: %w", err)
|
||||
}
|
||||
if !minRaw.Valid || !maxRaw.Valid {
|
||||
return nil
|
||||
}
|
||||
switch s.o.Shard.Kind {
|
||||
case schema.ShardDate, schema.ShardDateTime:
|
||||
minDay := s.first10(minRaw.String)
|
||||
maxDay := s.first10(maxRaw.String)
|
||||
zeroExists := strings.HasPrefix(minDay, "0000-")
|
||||
if zeroExists {
|
||||
minDay = "0001-01-01"
|
||||
}
|
||||
s.zeroPending = s.wantZero(zeroExists)
|
||||
if strings.HasPrefix(maxDay, "0000-") {
|
||||
return nil
|
||||
}
|
||||
minT, err := time.ParseInLocation("2006-01-02", minDay, s.o.ShardLoc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse shard range minimum %q: %w", minRaw.String, err)
|
||||
}
|
||||
maxT, err := time.ParseInLocation("2006-01-02", maxDay, s.o.ShardLoc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse shard range maximum %q: %w", maxRaw.String, err)
|
||||
}
|
||||
s.minMax(minT, maxT)
|
||||
case schema.ShardEpoch:
|
||||
minV, err := strconv.ParseInt(strings.TrimSpace(minRaw.String), 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse shard range minimum: %w", err)
|
||||
}
|
||||
maxV, err := strconv.ParseInt(strings.TrimSpace(maxRaw.String), 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse shard range maximum: %w", err)
|
||||
}
|
||||
minT := schema.EpochTime(minV, s.o.Shard.Unit, s.o.ShardLoc)
|
||||
maxT := schema.EpochTime(maxV, s.o.Shard.Unit, s.o.ShardLoc)
|
||||
s.minMax(dayStart(minT, s.o.ShardLoc), dayStart(maxT, s.o.ShardLoc))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DateBScanner) wantZero(zeroExists bool) bool {
|
||||
if s.resumeDay == "" {
|
||||
return zeroExists
|
||||
}
|
||||
return s.resumeDay == zeroDay
|
||||
}
|
||||
|
||||
func (s *DateBScanner) minMax(minT, maxT time.Time) {
|
||||
s.hasDays = true
|
||||
s.nextDay = minT
|
||||
s.maxDay = maxT
|
||||
if s.resumeDay != "" && s.resumeDay != zeroDay {
|
||||
if d, err := time.ParseInLocation("2006-01-02", s.resumeDay, s.o.ShardLoc); err == nil && d.After(minT) {
|
||||
s.nextDay = d
|
||||
}
|
||||
}
|
||||
if s.resumeDay == zeroDay {
|
||||
s.nextDay = minT
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DateBScanner) advanceDay() bool {
|
||||
if !s.hasDays {
|
||||
return false
|
||||
}
|
||||
for !s.nextDay.After(s.maxDay) {
|
||||
day := s.nextDay.Format("2006-01-02")
|
||||
if s.resumeDay != "" && s.resumeDay != zeroDay && day < s.resumeDay && !s.resumeUse {
|
||||
s.nextDay = s.nextDay.AddDate(0, 0, 1)
|
||||
continue
|
||||
}
|
||||
s.curDay = day
|
||||
s.nextDayStr = s.nextDay.AddDate(0, 0, 1).Format("2006-01-02")
|
||||
s.nextDay = s.nextDay.AddDate(0, 0, 1)
|
||||
if s.resumeDay != "" && day == s.resumeDay && !s.resumeUse {
|
||||
s.cursor = s.resumeKey
|
||||
s.resumeUse = true
|
||||
} else {
|
||||
s.cursor = nil
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *DateBScanner) first10(v string) string {
|
||||
if len(v) < 10 {
|
||||
return v
|
||||
}
|
||||
return v[:10]
|
||||
}
|
||||
|
||||
func dayStart(t time.Time, loc *time.Location) time.Time {
|
||||
return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, loc)
|
||||
}
|
||||
|
||||
func (s *DateBScanner) ResumeState() CursorState {
|
||||
phase := s.phase
|
||||
day := ""
|
||||
if phase == "day" {
|
||||
day = s.curDay
|
||||
}
|
||||
return CursorState{Phase: phase, Key: s.cursor, DayStart: day}
|
||||
}
|
||||
|
||||
func (s *DateBScanner) Close() error {
|
||||
var firstErr error
|
||||
if s.rows != nil {
|
||||
if err := s.rows.Close(); err != nil {
|
||||
firstErr = err
|
||||
}
|
||||
s.rows = nil
|
||||
}
|
||||
if s.conn != nil {
|
||||
if err := s.conn.Close(); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
s.conn = nil
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"archer/internal/row"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
// PKScanner walks the table by its chosen unique key using
|
||||
// key > :last ORDER BY key LIMIT :chunk_size.
|
||||
type PKScanner struct {
|
||||
o *Options
|
||||
cursor row.Key
|
||||
done bool
|
||||
}
|
||||
|
||||
func NewPK(o Options) *PKScanner {
|
||||
s := &PKScanner{o: &o}
|
||||
if o.Resume != nil && (o.Resume.Phase == "" || o.Resume.Phase == "pk") {
|
||||
s.cursor = o.Resume.Key
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *PKScanner) Next(ctx context.Context) (*row.Chunk, error) {
|
||||
if s.done {
|
||||
return nil, io.EOF
|
||||
}
|
||||
q, args := s.query()
|
||||
rows, err := s.o.DB.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
chunk, exhausted, err := s.o.readChunk(rows, s.o.ChunkSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(chunk.Rows) == 0 {
|
||||
s.done = true
|
||||
return nil, io.EOF
|
||||
}
|
||||
if exhausted {
|
||||
s.done = true
|
||||
}
|
||||
s.cursor = chunk.LastKey
|
||||
return chunk, nil
|
||||
}
|
||||
|
||||
func (s *PKScanner) query() (string, []any) {
|
||||
var b strings.Builder
|
||||
b.WriteString(s.o.selectSQL())
|
||||
var args []any
|
||||
if len(s.cursor) > 0 {
|
||||
b.WriteString(" AND ")
|
||||
b.WriteString(sqlutil.GreaterThan(s.o.KeyCols))
|
||||
args = append(args, s.cursor.Args()...)
|
||||
}
|
||||
b.WriteString(" ORDER BY ")
|
||||
b.WriteString(sqlutil.Columns(s.o.KeyCols))
|
||||
b.WriteString(" LIMIT ?")
|
||||
args = append(args, s.o.ChunkSize)
|
||||
return b.String(), args
|
||||
}
|
||||
|
||||
func (s *PKScanner) Close() error { return nil }
|
||||
|
||||
func (s *PKScanner) ResumeState() CursorState {
|
||||
return CursorState{Phase: "pk", Key: s.cursor}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
// Package scan implements keyset (seek) chunked scanning of a source table.
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"archer/internal/row"
|
||||
"archer/internal/schema"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
// CursorState is the resumable position of a scanner.
|
||||
type CursorState struct {
|
||||
Phase string
|
||||
Key row.Key
|
||||
DayStart string
|
||||
}
|
||||
|
||||
// Scanner produces chunks in scan order.
|
||||
type Scanner interface {
|
||||
Next(ctx context.Context) (*row.Chunk, error) // io.EOF when exhausted
|
||||
ResumeState() CursorState
|
||||
Close() error
|
||||
}
|
||||
|
||||
// Resume is the persisted scan position.
|
||||
type Resume struct {
|
||||
Phase string // pk|null|value|day
|
||||
Key row.Key
|
||||
DayStart string
|
||||
}
|
||||
|
||||
// Options configures a Scanner.
|
||||
type Options struct {
|
||||
DB *sql.DB
|
||||
DBName string
|
||||
Table string
|
||||
Where string
|
||||
SelectCols []string
|
||||
KeyCols []string
|
||||
KeyIdx []int
|
||||
ChunkSize int
|
||||
|
||||
Shard *schema.ShardSpec // nil outside date mode
|
||||
ShardIdx int // -1 outside date mode
|
||||
ShardLoc *time.Location
|
||||
ZeroDate string
|
||||
Resume *Resume
|
||||
}
|
||||
|
||||
func (o *Options) qualified() string { return sqlutil.Qualified(o.DBName, o.Table) }
|
||||
|
||||
func (o *Options) selectSQL() string {
|
||||
return "SELECT " + sqlutil.Columns(o.SelectCols) +
|
||||
" FROM " + o.qualified() +
|
||||
" WHERE (" + o.Where + ")"
|
||||
}
|
||||
|
||||
func (o *Options) keyOf(r row.Row) row.Key {
|
||||
k := make(row.Key, len(o.KeyIdx))
|
||||
for i, idx := range o.KeyIdx {
|
||||
v := r.Values[idx]
|
||||
if v.IsNull() {
|
||||
k[i] = row.KeyPart{Null: true}
|
||||
} else {
|
||||
k[i] = row.KeyPart{Val: string(v.Raw)}
|
||||
}
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
func (o *Options) compositeCursor(r row.Row) row.Key {
|
||||
k := o.keyOf(r)
|
||||
out := make(row.Key, 0, 1+len(k))
|
||||
v := r.Values[o.ShardIdx]
|
||||
if v.IsNull() {
|
||||
out = append(out, row.KeyPart{Null: true})
|
||||
} else {
|
||||
out = append(out, row.KeyPart{Val: string(v.Raw)})
|
||||
}
|
||||
return append(out, k...)
|
||||
}
|
||||
|
||||
func (o *Options) bucket(r row.Row) (string, error) {
|
||||
return schema.BucketOf(r.Values[o.ShardIdx], o.Shard, o.ShardLoc, o.ZeroDate, o.keyOf(r))
|
||||
}
|
||||
|
||||
// readChunk consumes up to limit rows from an open result set.
|
||||
func (o *Options) readChunk(rows *sql.Rows, limit int) (*row.Chunk, bool, error) {
|
||||
chunk := &row.Chunk{}
|
||||
raw := make([]sql.RawBytes, len(o.SelectCols))
|
||||
ptrs := make([]any, len(raw))
|
||||
for i := range raw {
|
||||
ptrs[i] = &raw[i]
|
||||
}
|
||||
exhausted := false
|
||||
for len(chunk.Rows) < limit {
|
||||
if !rows.Next() {
|
||||
exhausted = true
|
||||
break
|
||||
}
|
||||
if err := rows.Scan(ptrs...); err != nil {
|
||||
return nil, false, fmt.Errorf("scan row: %w", err)
|
||||
}
|
||||
vals := make([]row.Value, len(raw))
|
||||
for i, b := range raw {
|
||||
if b == nil {
|
||||
continue
|
||||
}
|
||||
c := make([]byte, len(b))
|
||||
copy(c, b)
|
||||
vals[i] = row.Value{Raw: c}
|
||||
}
|
||||
chunk.Rows = append(chunk.Rows, row.Row{Values: vals})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, false, fmt.Errorf("read rows: %w", err)
|
||||
}
|
||||
for i := range chunk.Rows {
|
||||
chunk.Keys = append(chunk.Keys, o.keyOf(chunk.Rows[i]))
|
||||
}
|
||||
if len(chunk.Rows) > 0 {
|
||||
chunk.FirstKey = chunk.Keys[0]
|
||||
chunk.LastKey = chunk.Keys[len(chunk.Keys)-1]
|
||||
}
|
||||
return chunk, exhausted, nil
|
||||
}
|
||||
|
||||
func (o *Options) fillBuckets(c *row.Chunk) error {
|
||||
c.Buckets = make([]string, len(c.Rows))
|
||||
for i := range c.Rows {
|
||||
b, err := o.bucket(c.Rows[i])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.Buckets[i] = b
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user