Files
voidandClaude Sonnet 4.6 bc4a6b504f Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr,
docs, build scripts and ARCHER_* environment variables.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-11-06 22:12:17 +08:00

427 lines
9.6 KiB
Go

// Package shard writes row streams into ordered, independently importable
// shard files and records them in manifest.jsonl.
package shard
import (
"bufio"
"fmt"
"io"
"os"
"path/filepath"
"archer/internal/atomicfile"
"archer/internal/row"
"archer/internal/schema"
)
// Options configures a Writer.
type Options struct {
Dir string
Mode string // none|count|date
ShardRows int64
Format string
Compressor *Compressor
Columns []row.Column
OutIdx []int
CSVNull string
StartBucket string
StartSeq int64
}
// Writer owns the single active shard file.
type Writer struct {
opt Options
man *Manifest
seq int64
curBucket string
cur *shardFile
totalRows int64
shards int64
bytes int64
}
type shardFile struct {
bucket string
seq int64
dir string
partial string
file *os.File
cw *countWriter
compW io.WriteCloser
bw *bufio.Writer
enc row.Encoder
buf []byte
rows int64
memberRows int64
firstKey row.Key
lastKey row.Key
}
type countWriter struct {
w io.Writer
n int64
}
func (c *countWriter) Write(p []byte) (int, error) {
n, err := c.w.Write(p)
c.n += int64(n)
return n, err
}
// Open creates a writer. It does not open a shard until the first row arrives.
func Open(opt Options, man *Manifest) (*Writer, error) {
w := &Writer{opt: opt, man: man, seq: opt.StartSeq, curBucket: opt.StartBucket}
return w, nil
}
// Resume reopens an existing partial shard and continues appending to it.
func (w *Writer) Resume(m *PartialMeta) error {
dir := BucketDir(w.opt.Dir, m.Bucket)
path := partialPath(dir, m.Seq)
f, err := os.OpenFile(path, os.O_RDWR, 0o644)
if err != nil {
return fmt.Errorf("resume partial shard: %w", err)
}
st, err := f.Stat()
if err != nil {
f.Close()
return err
}
if st.Size() < m.Bytes {
f.Close()
return fmt.Errorf("partial shard %s is shorter (%d bytes) than its metadata (%d bytes); refusing to resume", path, st.Size(), m.Bytes)
}
if st.Size() > m.Bytes {
if err := f.Truncate(m.Bytes); err != nil {
f.Close()
return fmt.Errorf("truncate partial shard: %w", err)
}
}
if _, err := f.Seek(m.Bytes, io.SeekStart); err != nil {
f.Close()
return err
}
enc, err := row.NewEncoder(w.opt.Format, w.opt.Columns, w.opt.OutIdx, row.Options{CSVNull: w.opt.CSVNull})
if err != nil {
f.Close()
return err
}
sf := &shardFile{
bucket: m.Bucket,
seq: m.Seq,
dir: dir,
partial: path,
file: f,
enc: enc,
rows: m.Rows,
memberRows: m.Rows,
firstKey: m.FirstKey,
lastKey: m.LastKey,
}
if err := sf.startChain(m.Bytes); err != nil {
f.Close()
return err
}
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
f.Close()
return err
}
w.cur = sf
w.seq = m.Seq
w.curBucket = m.Bucket
w.totalRows = m.Rows
return nil
}
func (sf *shardFile) startChain(offset int64) error {
sf.cw = &countWriter{w: sf.file, n: offset}
return nil
}
func (sf *shardFile) openChain(comp *Compressor, bufSize int) error {
cw, err := comp.Wrap(sf.cw)
if err != nil {
return err
}
sf.compW = cw
sf.bw = bufio.NewWriterSize(cw, bufSize)
sf.memberRows = sf.rows
return nil
}
func (w *Writer) openShard(bucket string) error {
if w.cur != nil {
return fmt.Errorf("internal: shard already open")
}
if bucket != w.curBucket {
w.seq = 0
w.curBucket = bucket
}
w.seq++
dir := BucketDir(w.opt.Dir, bucket)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("create shard directory: %w", err)
}
path := partialPath(dir, w.seq)
f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
if err != nil {
return fmt.Errorf("create partial shard: %w", err)
}
enc, err := row.NewEncoder(w.opt.Format, w.opt.Columns, w.opt.OutIdx, row.Options{CSVNull: w.opt.CSVNull})
if err != nil {
f.Close()
return err
}
sf := &shardFile{bucket: bucket, seq: w.seq, dir: dir, partial: path, file: f, enc: enc}
if err := sf.startChain(0); err != nil {
f.Close()
return err
}
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
f.Close()
return err
}
if hdr := enc.Header(); len(hdr) > 0 {
if _, err := sf.bw.Write(hdr); err != nil {
f.Close()
return err
}
}
w.cur = sf
return nil
}
// WriteChunk routes chunk rows into the active shard, sealing on bucket change
// or when --shard-rows is reached. A chunk may span multiple shards.
func (w *Writer) WriteChunk(c *row.Chunk) error {
for i := range c.Rows {
bucket := ""
if w.opt.Mode == "date" {
bucket = c.Buckets[i]
}
if w.cur != nil {
needSeal := false
if bucket != w.cur.bucket {
if !schema.BucketLess(w.cur.bucket, bucket) {
return fmt.Errorf("rows arrived out of shard order: %q after %q", bucket, w.cur.bucket)
}
needSeal = true
} else if w.opt.ShardRows > 0 && w.cur.rows >= w.opt.ShardRows {
needSeal = true
}
if needSeal {
if err := w.seal(); err != nil {
return err
}
}
}
if w.cur == nil {
if err := w.openShard(bucket); err != nil {
return err
}
}
if err := w.cur.writeRow(c.Rows[i]); err != nil {
return err
}
w.cur.trackKey(c.Keys[i])
w.totalRows++
}
return nil
}
func (sf *shardFile) writeRow(r row.Row) error {
var err error
sf.buf, err = sf.enc.Encode(sf.buf[:0], r)
if err != nil {
return err
}
if _, err := sf.bw.Write(sf.buf); err != nil {
return fmt.Errorf("write shard row: %w", err)
}
sf.rows++
return nil
}
func (sf *shardFile) trackKey(k row.Key) {
if sf.firstKey.IsZero() {
sf.firstKey = k
}
sf.lastKey = k
}
// Checkpoint makes the current shard bytes and their metadata durable. It is
// called before destructive database work when --purge is enabled.
func (w *Writer) Checkpoint() error {
if w.cur == nil {
return nil
}
sf := w.cur
if sf.rows > sf.memberRows {
if err := sf.closeMember(); err != nil {
return err
}
}
meta := &PartialMeta{
Seq: sf.seq,
Bucket: sf.bucket,
Rows: sf.rows,
Bytes: sf.cw.n,
FirstKey: sf.firstKey,
LastKey: sf.lastKey,
}
if err := savePartialMeta(sf.dir, meta); err != nil {
return fmt.Errorf("write partial metadata: %w", err)
}
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
return err
}
return nil
}
func (sf *shardFile) closeMember() error {
if err := sf.bw.Flush(); err != nil {
return fmt.Errorf("flush shard: %w", err)
}
if err := sf.compW.Close(); err != nil {
return fmt.Errorf("close shard compressor: %w", err)
}
if err := sf.file.Sync(); err != nil {
return fmt.Errorf("sync shard: %w", err)
}
return nil
}
// Flush pushes buffered rows to the OS without sealing the shard.
func (w *Writer) Flush() error {
if w.cur == nil || w.cur.bw == nil {
return nil
}
return w.cur.bw.Flush()
}
// SealCurrent closes and publishes the active shard even if it is not full.
func (w *Writer) SealCurrent() error {
if w.cur == nil {
return nil
}
if w.cur.rows == 0 {
return w.discardCurrent()
}
return w.seal()
}
func (w *Writer) discardCurrent() error {
sf := w.cur
sf.file.Close()
os.Remove(sf.partial)
removePartialMeta(sf.dir, sf.seq)
w.cur = nil
return nil
}
func (w *Writer) seal() error {
sf := w.cur
if sf.rows == 0 {
return w.discardCurrent()
}
if sf.rows > sf.memberRows || sf.compW != nil {
if err := sf.closeMember(); err != nil {
return err
}
}
if err := sf.file.Close(); err != nil {
return fmt.Errorf("close partial shard: %w", err)
}
lo := KeyFilePart(sf.firstKey)
hi := KeyFilePart(sf.lastKey)
name := FinalName(sf.seq, lo, hi, w.opt.Format, w.opt.Compressor.Ext)
finalPath := filepath.Join(sf.dir, name)
if err := os.Remove(finalPath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("remove stale shard: %w", err)
}
if err := atomicfile.Replace(sf.partial, finalPath); err != nil {
return fmt.Errorf("publish shard: %w", err)
}
sum, size, err := hashFile(finalPath)
if err != nil {
return fmt.Errorf("hash shard: %w", err)
}
rel, err := filepath.Rel(w.opt.Dir, finalPath)
if err != nil {
return err
}
entry := Entry{
Seq: sf.seq,
Path: filepath.ToSlash(rel),
Date: sf.bucket,
PKLo: lo,
PKHi: hi,
Rows: sf.rows,
Bytes: size,
SHA256: sum,
}
if err := w.man.Append(entry); err != nil {
return err
}
removePartialMeta(sf.dir, sf.seq)
w.shards++
w.bytes += size
w.cur = nil
return nil
}
// Close publishes any in-progress shard and closes the manifest.
func (w *Writer) Close() error {
if err := w.SealCurrent(); err != nil {
return err
}
return w.man.Close()
}
// StatePosition reports the bucket and the last sealed seq within that bucket,
// which is what a checkpoint must record for resume.
func (w *Writer) StatePosition() (bucket string, sealedSeq int64) {
if w.cur != nil {
return w.cur.bucket, w.cur.seq - 1
}
return w.curBucket, w.seq
}
// Abandon closes file handles without sealing the active shard, leaving the
// partial and its metadata for a later resume.
func (w *Writer) Abandon() error {
if w.cur != nil {
if w.cur.bw != nil {
_ = w.cur.bw.Flush()
}
_ = w.cur.file.Close()
w.cur = nil
}
return w.man.Close()
}
// Stats returns rows written, shards sealed and bytes sealed.
func (w *Writer) Stats() (rows, shards, bytes int64) {
return w.totalRows, w.shards, w.bytes
}
// CurrentShard returns the sequence number of the active shard, or the last
// one when no shard is currently open.
func (w *Writer) CurrentShard() int64 {
if w.cur != nil {
return w.cur.seq
}
return w.seq
}
// WrittenBytes returns sealed bytes plus the bytes already written to the
// active partial shard.
func (w *Writer) WrittenBytes() int64 {
n := w.bytes
if w.cur != nil && w.cur.cw != nil {
n += w.cur.cw.n
}
return n
}