Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
427 lines
9.6 KiB
Go
427 lines
9.6 KiB
Go
// Package shard writes row streams into ordered, independently importable
|
|
// shard files and records them in manifest.jsonl.
|
|
package shard
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
"archer/internal/atomicfile"
|
|
"archer/internal/row"
|
|
"archer/internal/schema"
|
|
)
|
|
|
|
// Options configures a Writer.
|
|
type Options struct {
|
|
Dir string
|
|
Mode string // none|count|date
|
|
ShardRows int64
|
|
Format string
|
|
Compressor *Compressor
|
|
Columns []row.Column
|
|
OutIdx []int
|
|
CSVNull string
|
|
StartBucket string
|
|
StartSeq int64
|
|
}
|
|
|
|
// Writer owns the single active shard file.
|
|
type Writer struct {
|
|
opt Options
|
|
man *Manifest
|
|
seq int64
|
|
curBucket string
|
|
cur *shardFile
|
|
|
|
totalRows int64
|
|
shards int64
|
|
bytes int64
|
|
}
|
|
|
|
type shardFile struct {
|
|
bucket string
|
|
seq int64
|
|
dir string
|
|
partial string
|
|
file *os.File
|
|
cw *countWriter
|
|
compW io.WriteCloser
|
|
bw *bufio.Writer
|
|
enc row.Encoder
|
|
buf []byte
|
|
rows int64
|
|
memberRows int64
|
|
firstKey row.Key
|
|
lastKey row.Key
|
|
}
|
|
|
|
type countWriter struct {
|
|
w io.Writer
|
|
n int64
|
|
}
|
|
|
|
func (c *countWriter) Write(p []byte) (int, error) {
|
|
n, err := c.w.Write(p)
|
|
c.n += int64(n)
|
|
return n, err
|
|
}
|
|
|
|
// Open creates a writer. It does not open a shard until the first row arrives.
|
|
func Open(opt Options, man *Manifest) (*Writer, error) {
|
|
w := &Writer{opt: opt, man: man, seq: opt.StartSeq, curBucket: opt.StartBucket}
|
|
return w, nil
|
|
}
|
|
|
|
// Resume reopens an existing partial shard and continues appending to it.
|
|
func (w *Writer) Resume(m *PartialMeta) error {
|
|
dir := BucketDir(w.opt.Dir, m.Bucket)
|
|
path := partialPath(dir, m.Seq)
|
|
f, err := os.OpenFile(path, os.O_RDWR, 0o644)
|
|
if err != nil {
|
|
return fmt.Errorf("resume partial shard: %w", err)
|
|
}
|
|
st, err := f.Stat()
|
|
if err != nil {
|
|
f.Close()
|
|
return err
|
|
}
|
|
if st.Size() < m.Bytes {
|
|
f.Close()
|
|
return fmt.Errorf("partial shard %s is shorter (%d bytes) than its metadata (%d bytes); refusing to resume", path, st.Size(), m.Bytes)
|
|
}
|
|
if st.Size() > m.Bytes {
|
|
if err := f.Truncate(m.Bytes); err != nil {
|
|
f.Close()
|
|
return fmt.Errorf("truncate partial shard: %w", err)
|
|
}
|
|
}
|
|
if _, err := f.Seek(m.Bytes, io.SeekStart); err != nil {
|
|
f.Close()
|
|
return err
|
|
}
|
|
enc, err := row.NewEncoder(w.opt.Format, w.opt.Columns, w.opt.OutIdx, row.Options{CSVNull: w.opt.CSVNull})
|
|
if err != nil {
|
|
f.Close()
|
|
return err
|
|
}
|
|
sf := &shardFile{
|
|
bucket: m.Bucket,
|
|
seq: m.Seq,
|
|
dir: dir,
|
|
partial: path,
|
|
file: f,
|
|
enc: enc,
|
|
rows: m.Rows,
|
|
memberRows: m.Rows,
|
|
firstKey: m.FirstKey,
|
|
lastKey: m.LastKey,
|
|
}
|
|
if err := sf.startChain(m.Bytes); err != nil {
|
|
f.Close()
|
|
return err
|
|
}
|
|
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
|
|
f.Close()
|
|
return err
|
|
}
|
|
w.cur = sf
|
|
w.seq = m.Seq
|
|
w.curBucket = m.Bucket
|
|
w.totalRows = m.Rows
|
|
return nil
|
|
}
|
|
|
|
func (sf *shardFile) startChain(offset int64) error {
|
|
sf.cw = &countWriter{w: sf.file, n: offset}
|
|
return nil
|
|
}
|
|
|
|
func (sf *shardFile) openChain(comp *Compressor, bufSize int) error {
|
|
cw, err := comp.Wrap(sf.cw)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sf.compW = cw
|
|
sf.bw = bufio.NewWriterSize(cw, bufSize)
|
|
sf.memberRows = sf.rows
|
|
return nil
|
|
}
|
|
|
|
func (w *Writer) openShard(bucket string) error {
|
|
if w.cur != nil {
|
|
return fmt.Errorf("internal: shard already open")
|
|
}
|
|
if bucket != w.curBucket {
|
|
w.seq = 0
|
|
w.curBucket = bucket
|
|
}
|
|
w.seq++
|
|
dir := BucketDir(w.opt.Dir, bucket)
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
return fmt.Errorf("create shard directory: %w", err)
|
|
}
|
|
path := partialPath(dir, w.seq)
|
|
f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
|
if err != nil {
|
|
return fmt.Errorf("create partial shard: %w", err)
|
|
}
|
|
enc, err := row.NewEncoder(w.opt.Format, w.opt.Columns, w.opt.OutIdx, row.Options{CSVNull: w.opt.CSVNull})
|
|
if err != nil {
|
|
f.Close()
|
|
return err
|
|
}
|
|
sf := &shardFile{bucket: bucket, seq: w.seq, dir: dir, partial: path, file: f, enc: enc}
|
|
if err := sf.startChain(0); err != nil {
|
|
f.Close()
|
|
return err
|
|
}
|
|
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
|
|
f.Close()
|
|
return err
|
|
}
|
|
if hdr := enc.Header(); len(hdr) > 0 {
|
|
if _, err := sf.bw.Write(hdr); err != nil {
|
|
f.Close()
|
|
return err
|
|
}
|
|
}
|
|
w.cur = sf
|
|
return nil
|
|
}
|
|
|
|
// WriteChunk routes chunk rows into the active shard, sealing on bucket change
|
|
// or when --shard-rows is reached. A chunk may span multiple shards.
|
|
func (w *Writer) WriteChunk(c *row.Chunk) error {
|
|
for i := range c.Rows {
|
|
bucket := ""
|
|
if w.opt.Mode == "date" {
|
|
bucket = c.Buckets[i]
|
|
}
|
|
if w.cur != nil {
|
|
needSeal := false
|
|
if bucket != w.cur.bucket {
|
|
if !schema.BucketLess(w.cur.bucket, bucket) {
|
|
return fmt.Errorf("rows arrived out of shard order: %q after %q", bucket, w.cur.bucket)
|
|
}
|
|
needSeal = true
|
|
} else if w.opt.ShardRows > 0 && w.cur.rows >= w.opt.ShardRows {
|
|
needSeal = true
|
|
}
|
|
if needSeal {
|
|
if err := w.seal(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if w.cur == nil {
|
|
if err := w.openShard(bucket); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := w.cur.writeRow(c.Rows[i]); err != nil {
|
|
return err
|
|
}
|
|
w.cur.trackKey(c.Keys[i])
|
|
w.totalRows++
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (sf *shardFile) writeRow(r row.Row) error {
|
|
var err error
|
|
sf.buf, err = sf.enc.Encode(sf.buf[:0], r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := sf.bw.Write(sf.buf); err != nil {
|
|
return fmt.Errorf("write shard row: %w", err)
|
|
}
|
|
sf.rows++
|
|
return nil
|
|
}
|
|
|
|
func (sf *shardFile) trackKey(k row.Key) {
|
|
if sf.firstKey.IsZero() {
|
|
sf.firstKey = k
|
|
}
|
|
sf.lastKey = k
|
|
}
|
|
|
|
// Checkpoint makes the current shard bytes and their metadata durable. It is
|
|
// called before destructive database work when --purge is enabled.
|
|
func (w *Writer) Checkpoint() error {
|
|
if w.cur == nil {
|
|
return nil
|
|
}
|
|
sf := w.cur
|
|
if sf.rows > sf.memberRows {
|
|
if err := sf.closeMember(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
meta := &PartialMeta{
|
|
Seq: sf.seq,
|
|
Bucket: sf.bucket,
|
|
Rows: sf.rows,
|
|
Bytes: sf.cw.n,
|
|
FirstKey: sf.firstKey,
|
|
LastKey: sf.lastKey,
|
|
}
|
|
if err := savePartialMeta(sf.dir, meta); err != nil {
|
|
return fmt.Errorf("write partial metadata: %w", err)
|
|
}
|
|
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (sf *shardFile) closeMember() error {
|
|
if err := sf.bw.Flush(); err != nil {
|
|
return fmt.Errorf("flush shard: %w", err)
|
|
}
|
|
if err := sf.compW.Close(); err != nil {
|
|
return fmt.Errorf("close shard compressor: %w", err)
|
|
}
|
|
if err := sf.file.Sync(); err != nil {
|
|
return fmt.Errorf("sync shard: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Flush pushes buffered rows to the OS without sealing the shard.
|
|
func (w *Writer) Flush() error {
|
|
if w.cur == nil || w.cur.bw == nil {
|
|
return nil
|
|
}
|
|
return w.cur.bw.Flush()
|
|
}
|
|
|
|
// SealCurrent closes and publishes the active shard even if it is not full.
|
|
func (w *Writer) SealCurrent() error {
|
|
if w.cur == nil {
|
|
return nil
|
|
}
|
|
if w.cur.rows == 0 {
|
|
return w.discardCurrent()
|
|
}
|
|
return w.seal()
|
|
}
|
|
|
|
func (w *Writer) discardCurrent() error {
|
|
sf := w.cur
|
|
sf.file.Close()
|
|
os.Remove(sf.partial)
|
|
removePartialMeta(sf.dir, sf.seq)
|
|
w.cur = nil
|
|
return nil
|
|
}
|
|
|
|
func (w *Writer) seal() error {
|
|
sf := w.cur
|
|
if sf.rows == 0 {
|
|
return w.discardCurrent()
|
|
}
|
|
if sf.rows > sf.memberRows || sf.compW != nil {
|
|
if err := sf.closeMember(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := sf.file.Close(); err != nil {
|
|
return fmt.Errorf("close partial shard: %w", err)
|
|
}
|
|
lo := KeyFilePart(sf.firstKey)
|
|
hi := KeyFilePart(sf.lastKey)
|
|
name := FinalName(sf.seq, lo, hi, w.opt.Format, w.opt.Compressor.Ext)
|
|
finalPath := filepath.Join(sf.dir, name)
|
|
if err := os.Remove(finalPath); err != nil && !os.IsNotExist(err) {
|
|
return fmt.Errorf("remove stale shard: %w", err)
|
|
}
|
|
if err := atomicfile.Replace(sf.partial, finalPath); err != nil {
|
|
return fmt.Errorf("publish shard: %w", err)
|
|
}
|
|
sum, size, err := hashFile(finalPath)
|
|
if err != nil {
|
|
return fmt.Errorf("hash shard: %w", err)
|
|
}
|
|
rel, err := filepath.Rel(w.opt.Dir, finalPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
entry := Entry{
|
|
Seq: sf.seq,
|
|
Path: filepath.ToSlash(rel),
|
|
Date: sf.bucket,
|
|
PKLo: lo,
|
|
PKHi: hi,
|
|
Rows: sf.rows,
|
|
Bytes: size,
|
|
SHA256: sum,
|
|
}
|
|
if err := w.man.Append(entry); err != nil {
|
|
return err
|
|
}
|
|
removePartialMeta(sf.dir, sf.seq)
|
|
w.shards++
|
|
w.bytes += size
|
|
w.cur = nil
|
|
return nil
|
|
}
|
|
|
|
// Close publishes any in-progress shard and closes the manifest.
|
|
func (w *Writer) Close() error {
|
|
if err := w.SealCurrent(); err != nil {
|
|
return err
|
|
}
|
|
return w.man.Close()
|
|
}
|
|
|
|
// StatePosition reports the bucket and the last sealed seq within that bucket,
|
|
// which is what a checkpoint must record for resume.
|
|
func (w *Writer) StatePosition() (bucket string, sealedSeq int64) {
|
|
if w.cur != nil {
|
|
return w.cur.bucket, w.cur.seq - 1
|
|
}
|
|
return w.curBucket, w.seq
|
|
}
|
|
|
|
// Abandon closes file handles without sealing the active shard, leaving the
|
|
// partial and its metadata for a later resume.
|
|
func (w *Writer) Abandon() error {
|
|
if w.cur != nil {
|
|
if w.cur.bw != nil {
|
|
_ = w.cur.bw.Flush()
|
|
}
|
|
_ = w.cur.file.Close()
|
|
w.cur = nil
|
|
}
|
|
return w.man.Close()
|
|
}
|
|
|
|
// Stats returns rows written, shards sealed and bytes sealed.
|
|
func (w *Writer) Stats() (rows, shards, bytes int64) {
|
|
return w.totalRows, w.shards, w.bytes
|
|
}
|
|
|
|
// CurrentShard returns the sequence number of the active shard, or the last
|
|
// one when no shard is currently open.
|
|
func (w *Writer) CurrentShard() int64 {
|
|
if w.cur != nil {
|
|
return w.cur.seq
|
|
}
|
|
return w.seq
|
|
}
|
|
|
|
// WrittenBytes returns sealed bytes plus the bytes already written to the
|
|
// active partial shard.
|
|
func (w *Writer) WrittenBytes() int64 {
|
|
n := w.bytes
|
|
if w.cur != nil && w.cur.cw != nil {
|
|
n += w.cur.cw.n
|
|
}
|
|
return n
|
|
}
|