Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,142 @@
|
||||
// Package scan implements keyset (seek) chunked scanning of a source table.
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"archer/internal/row"
|
||||
"archer/internal/schema"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
// CursorState is the resumable position of a scanner.
|
||||
type CursorState struct {
|
||||
Phase string
|
||||
Key row.Key
|
||||
DayStart string
|
||||
}
|
||||
|
||||
// Scanner produces chunks in scan order.
|
||||
type Scanner interface {
|
||||
Next(ctx context.Context) (*row.Chunk, error) // io.EOF when exhausted
|
||||
ResumeState() CursorState
|
||||
Close() error
|
||||
}
|
||||
|
||||
// Resume is the persisted scan position.
|
||||
type Resume struct {
|
||||
Phase string // pk|null|value|day
|
||||
Key row.Key
|
||||
DayStart string
|
||||
}
|
||||
|
||||
// Options configures a Scanner.
|
||||
type Options struct {
|
||||
DB *sql.DB
|
||||
DBName string
|
||||
Table string
|
||||
Where string
|
||||
SelectCols []string
|
||||
KeyCols []string
|
||||
KeyIdx []int
|
||||
ChunkSize int
|
||||
|
||||
Shard *schema.ShardSpec // nil outside date mode
|
||||
ShardIdx int // -1 outside date mode
|
||||
ShardLoc *time.Location
|
||||
ZeroDate string
|
||||
Resume *Resume
|
||||
}
|
||||
|
||||
func (o *Options) qualified() string { return sqlutil.Qualified(o.DBName, o.Table) }
|
||||
|
||||
func (o *Options) selectSQL() string {
|
||||
return "SELECT " + sqlutil.Columns(o.SelectCols) +
|
||||
" FROM " + o.qualified() +
|
||||
" WHERE (" + o.Where + ")"
|
||||
}
|
||||
|
||||
func (o *Options) keyOf(r row.Row) row.Key {
|
||||
k := make(row.Key, len(o.KeyIdx))
|
||||
for i, idx := range o.KeyIdx {
|
||||
v := r.Values[idx]
|
||||
if v.IsNull() {
|
||||
k[i] = row.KeyPart{Null: true}
|
||||
} else {
|
||||
k[i] = row.KeyPart{Val: string(v.Raw)}
|
||||
}
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
func (o *Options) compositeCursor(r row.Row) row.Key {
|
||||
k := o.keyOf(r)
|
||||
out := make(row.Key, 0, 1+len(k))
|
||||
v := r.Values[o.ShardIdx]
|
||||
if v.IsNull() {
|
||||
out = append(out, row.KeyPart{Null: true})
|
||||
} else {
|
||||
out = append(out, row.KeyPart{Val: string(v.Raw)})
|
||||
}
|
||||
return append(out, k...)
|
||||
}
|
||||
|
||||
func (o *Options) bucket(r row.Row) (string, error) {
|
||||
return schema.BucketOf(r.Values[o.ShardIdx], o.Shard, o.ShardLoc, o.ZeroDate, o.keyOf(r))
|
||||
}
|
||||
|
||||
// readChunk consumes up to limit rows from an open result set.
|
||||
func (o *Options) readChunk(rows *sql.Rows, limit int) (*row.Chunk, bool, error) {
|
||||
chunk := &row.Chunk{}
|
||||
raw := make([]sql.RawBytes, len(o.SelectCols))
|
||||
ptrs := make([]any, len(raw))
|
||||
for i := range raw {
|
||||
ptrs[i] = &raw[i]
|
||||
}
|
||||
exhausted := false
|
||||
for len(chunk.Rows) < limit {
|
||||
if !rows.Next() {
|
||||
exhausted = true
|
||||
break
|
||||
}
|
||||
if err := rows.Scan(ptrs...); err != nil {
|
||||
return nil, false, fmt.Errorf("scan row: %w", err)
|
||||
}
|
||||
vals := make([]row.Value, len(raw))
|
||||
for i, b := range raw {
|
||||
if b == nil {
|
||||
continue
|
||||
}
|
||||
c := make([]byte, len(b))
|
||||
copy(c, b)
|
||||
vals[i] = row.Value{Raw: c}
|
||||
}
|
||||
chunk.Rows = append(chunk.Rows, row.Row{Values: vals})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, false, fmt.Errorf("read rows: %w", err)
|
||||
}
|
||||
for i := range chunk.Rows {
|
||||
chunk.Keys = append(chunk.Keys, o.keyOf(chunk.Rows[i]))
|
||||
}
|
||||
if len(chunk.Rows) > 0 {
|
||||
chunk.FirstKey = chunk.Keys[0]
|
||||
chunk.LastKey = chunk.Keys[len(chunk.Keys)-1]
|
||||
}
|
||||
return chunk, exhausted, nil
|
||||
}
|
||||
|
||||
func (o *Options) fillBuckets(c *row.Chunk) error {
|
||||
c.Buckets = make([]string, len(c.Rows))
|
||||
for i := range c.Rows {
|
||||
b, err := o.bucket(c.Rows[i])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.Buckets[i] = b
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user