// Package scan implements keyset (seek) chunked scanning of a source table. package scan import ( "context" "database/sql" "fmt" "time" "archer/internal/row" "archer/internal/schema" "archer/internal/sqlutil" ) // CursorState is the resumable position of a scanner. type CursorState struct { Phase string Key row.Key DayStart string } // Scanner produces chunks in scan order. type Scanner interface { Next(ctx context.Context) (*row.Chunk, error) // io.EOF when exhausted ResumeState() CursorState Close() error } // Resume is the persisted scan position. type Resume struct { Phase string // pk|null|value|day Key row.Key DayStart string } // Options configures a Scanner. type Options struct { DB *sql.DB DBName string Table string Where string SelectCols []string KeyCols []string KeyIdx []int ChunkSize int Shard *schema.ShardSpec // nil outside date mode ShardIdx int // -1 outside date mode ShardLoc *time.Location ZeroDate string Resume *Resume } func (o *Options) qualified() string { return sqlutil.Qualified(o.DBName, o.Table) } func (o *Options) selectSQL() string { return "SELECT " + sqlutil.Columns(o.SelectCols) + " FROM " + o.qualified() + " WHERE (" + o.Where + ")" } func (o *Options) keyOf(r row.Row) row.Key { k := make(row.Key, len(o.KeyIdx)) for i, idx := range o.KeyIdx { v := r.Values[idx] if v.IsNull() { k[i] = row.KeyPart{Null: true} } else { k[i] = row.KeyPart{Val: string(v.Raw)} } } return k } func (o *Options) compositeCursor(r row.Row) row.Key { k := o.keyOf(r) out := make(row.Key, 0, 1+len(k)) v := r.Values[o.ShardIdx] if v.IsNull() { out = append(out, row.KeyPart{Null: true}) } else { out = append(out, row.KeyPart{Val: string(v.Raw)}) } return append(out, k...) } func (o *Options) bucket(r row.Row) (string, error) { return schema.BucketOf(r.Values[o.ShardIdx], o.Shard, o.ShardLoc, o.ZeroDate, o.keyOf(r)) } // readChunk consumes up to limit rows from an open result set. func (o *Options) readChunk(rows *sql.Rows, limit int) (*row.Chunk, bool, error) { chunk := &row.Chunk{} raw := make([]sql.RawBytes, len(o.SelectCols)) ptrs := make([]any, len(raw)) for i := range raw { ptrs[i] = &raw[i] } exhausted := false for len(chunk.Rows) < limit { if !rows.Next() { exhausted = true break } if err := rows.Scan(ptrs...); err != nil { return nil, false, fmt.Errorf("scan row: %w", err) } vals := make([]row.Value, len(raw)) for i, b := range raw { if b == nil { continue } c := make([]byte, len(b)) copy(c, b) vals[i] = row.Value{Raw: c} } chunk.Rows = append(chunk.Rows, row.Row{Values: vals}) } if err := rows.Err(); err != nil { return nil, false, fmt.Errorf("read rows: %w", err) } for i := range chunk.Rows { chunk.Keys = append(chunk.Keys, o.keyOf(chunk.Rows[i])) } if len(chunk.Rows) > 0 { chunk.FirstKey = chunk.Keys[0] chunk.LastKey = chunk.Keys[len(chunk.Keys)-1] } return chunk, exhausted, nil } func (o *Options) fillBuckets(c *row.Chunk) error { c.Buckets = make([]string, len(c.Rows)) for i := range c.Rows { b, err := o.bucket(c.Rows[i]) if err != nil { return err } c.Buckets[i] = b } return nil }