Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
143 lines
3.2 KiB
Go
143 lines
3.2 KiB
Go
// Package scan implements keyset (seek) chunked scanning of a source table.
|
|
package scan
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"time"
|
|
|
|
"archer/internal/row"
|
|
"archer/internal/schema"
|
|
"archer/internal/sqlutil"
|
|
)
|
|
|
|
// CursorState is the resumable position of a scanner.
|
|
type CursorState struct {
|
|
Phase string
|
|
Key row.Key
|
|
DayStart string
|
|
}
|
|
|
|
// Scanner produces chunks in scan order.
|
|
type Scanner interface {
|
|
Next(ctx context.Context) (*row.Chunk, error) // io.EOF when exhausted
|
|
ResumeState() CursorState
|
|
Close() error
|
|
}
|
|
|
|
// Resume is the persisted scan position.
|
|
type Resume struct {
|
|
Phase string // pk|null|value|day
|
|
Key row.Key
|
|
DayStart string
|
|
}
|
|
|
|
// Options configures a Scanner.
|
|
type Options struct {
|
|
DB *sql.DB
|
|
DBName string
|
|
Table string
|
|
Where string
|
|
SelectCols []string
|
|
KeyCols []string
|
|
KeyIdx []int
|
|
ChunkSize int
|
|
|
|
Shard *schema.ShardSpec // nil outside date mode
|
|
ShardIdx int // -1 outside date mode
|
|
ShardLoc *time.Location
|
|
ZeroDate string
|
|
Resume *Resume
|
|
}
|
|
|
|
func (o *Options) qualified() string { return sqlutil.Qualified(o.DBName, o.Table) }
|
|
|
|
func (o *Options) selectSQL() string {
|
|
return "SELECT " + sqlutil.Columns(o.SelectCols) +
|
|
" FROM " + o.qualified() +
|
|
" WHERE (" + o.Where + ")"
|
|
}
|
|
|
|
func (o *Options) keyOf(r row.Row) row.Key {
|
|
k := make(row.Key, len(o.KeyIdx))
|
|
for i, idx := range o.KeyIdx {
|
|
v := r.Values[idx]
|
|
if v.IsNull() {
|
|
k[i] = row.KeyPart{Null: true}
|
|
} else {
|
|
k[i] = row.KeyPart{Val: string(v.Raw)}
|
|
}
|
|
}
|
|
return k
|
|
}
|
|
|
|
func (o *Options) compositeCursor(r row.Row) row.Key {
|
|
k := o.keyOf(r)
|
|
out := make(row.Key, 0, 1+len(k))
|
|
v := r.Values[o.ShardIdx]
|
|
if v.IsNull() {
|
|
out = append(out, row.KeyPart{Null: true})
|
|
} else {
|
|
out = append(out, row.KeyPart{Val: string(v.Raw)})
|
|
}
|
|
return append(out, k...)
|
|
}
|
|
|
|
func (o *Options) bucket(r row.Row) (string, error) {
|
|
return schema.BucketOf(r.Values[o.ShardIdx], o.Shard, o.ShardLoc, o.ZeroDate, o.keyOf(r))
|
|
}
|
|
|
|
// readChunk consumes up to limit rows from an open result set.
|
|
func (o *Options) readChunk(rows *sql.Rows, limit int) (*row.Chunk, bool, error) {
|
|
chunk := &row.Chunk{}
|
|
raw := make([]sql.RawBytes, len(o.SelectCols))
|
|
ptrs := make([]any, len(raw))
|
|
for i := range raw {
|
|
ptrs[i] = &raw[i]
|
|
}
|
|
exhausted := false
|
|
for len(chunk.Rows) < limit {
|
|
if !rows.Next() {
|
|
exhausted = true
|
|
break
|
|
}
|
|
if err := rows.Scan(ptrs...); err != nil {
|
|
return nil, false, fmt.Errorf("scan row: %w", err)
|
|
}
|
|
vals := make([]row.Value, len(raw))
|
|
for i, b := range raw {
|
|
if b == nil {
|
|
continue
|
|
}
|
|
c := make([]byte, len(b))
|
|
copy(c, b)
|
|
vals[i] = row.Value{Raw: c}
|
|
}
|
|
chunk.Rows = append(chunk.Rows, row.Row{Values: vals})
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, false, fmt.Errorf("read rows: %w", err)
|
|
}
|
|
for i := range chunk.Rows {
|
|
chunk.Keys = append(chunk.Keys, o.keyOf(chunk.Rows[i]))
|
|
}
|
|
if len(chunk.Rows) > 0 {
|
|
chunk.FirstKey = chunk.Keys[0]
|
|
chunk.LastKey = chunk.Keys[len(chunk.Keys)-1]
|
|
}
|
|
return chunk, exhausted, nil
|
|
}
|
|
|
|
func (o *Options) fillBuckets(c *row.Chunk) error {
|
|
c.Buckets = make([]string, len(c.Rows))
|
|
for i := range c.Rows {
|
|
b, err := o.bucket(c.Rows[i])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
c.Buckets[i] = b
|
|
}
|
|
return nil
|
|
}
|