Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,426 @@
|
||||
// Package shard writes row streams into ordered, independently importable
|
||||
// shard files and records them in manifest.jsonl.
|
||||
package shard
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"archer/internal/atomicfile"
|
||||
"archer/internal/row"
|
||||
"archer/internal/schema"
|
||||
)
|
||||
|
||||
// Options configures a Writer.
|
||||
type Options struct {
|
||||
Dir string
|
||||
Mode string // none|count|date
|
||||
ShardRows int64
|
||||
Format string
|
||||
Compressor *Compressor
|
||||
Columns []row.Column
|
||||
OutIdx []int
|
||||
CSVNull string
|
||||
StartBucket string
|
||||
StartSeq int64
|
||||
}
|
||||
|
||||
// Writer owns the single active shard file.
|
||||
type Writer struct {
|
||||
opt Options
|
||||
man *Manifest
|
||||
seq int64
|
||||
curBucket string
|
||||
cur *shardFile
|
||||
|
||||
totalRows int64
|
||||
shards int64
|
||||
bytes int64
|
||||
}
|
||||
|
||||
type shardFile struct {
|
||||
bucket string
|
||||
seq int64
|
||||
dir string
|
||||
partial string
|
||||
file *os.File
|
||||
cw *countWriter
|
||||
compW io.WriteCloser
|
||||
bw *bufio.Writer
|
||||
enc row.Encoder
|
||||
buf []byte
|
||||
rows int64
|
||||
memberRows int64
|
||||
firstKey row.Key
|
||||
lastKey row.Key
|
||||
}
|
||||
|
||||
type countWriter struct {
|
||||
w io.Writer
|
||||
n int64
|
||||
}
|
||||
|
||||
func (c *countWriter) Write(p []byte) (int, error) {
|
||||
n, err := c.w.Write(p)
|
||||
c.n += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Open creates a writer. It does not open a shard until the first row arrives.
|
||||
func Open(opt Options, man *Manifest) (*Writer, error) {
|
||||
w := &Writer{opt: opt, man: man, seq: opt.StartSeq, curBucket: opt.StartBucket}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// Resume reopens an existing partial shard and continues appending to it.
|
||||
func (w *Writer) Resume(m *PartialMeta) error {
|
||||
dir := BucketDir(w.opt.Dir, m.Bucket)
|
||||
path := partialPath(dir, m.Seq)
|
||||
f, err := os.OpenFile(path, os.O_RDWR, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resume partial shard: %w", err)
|
||||
}
|
||||
st, err := f.Stat()
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if st.Size() < m.Bytes {
|
||||
f.Close()
|
||||
return fmt.Errorf("partial shard %s is shorter (%d bytes) than its metadata (%d bytes); refusing to resume", path, st.Size(), m.Bytes)
|
||||
}
|
||||
if st.Size() > m.Bytes {
|
||||
if err := f.Truncate(m.Bytes); err != nil {
|
||||
f.Close()
|
||||
return fmt.Errorf("truncate partial shard: %w", err)
|
||||
}
|
||||
}
|
||||
if _, err := f.Seek(m.Bytes, io.SeekStart); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
enc, err := row.NewEncoder(w.opt.Format, w.opt.Columns, w.opt.OutIdx, row.Options{CSVNull: w.opt.CSVNull})
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
sf := &shardFile{
|
||||
bucket: m.Bucket,
|
||||
seq: m.Seq,
|
||||
dir: dir,
|
||||
partial: path,
|
||||
file: f,
|
||||
enc: enc,
|
||||
rows: m.Rows,
|
||||
memberRows: m.Rows,
|
||||
firstKey: m.FirstKey,
|
||||
lastKey: m.LastKey,
|
||||
}
|
||||
if err := sf.startChain(m.Bytes); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
w.cur = sf
|
||||
w.seq = m.Seq
|
||||
w.curBucket = m.Bucket
|
||||
w.totalRows = m.Rows
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sf *shardFile) startChain(offset int64) error {
|
||||
sf.cw = &countWriter{w: sf.file, n: offset}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sf *shardFile) openChain(comp *Compressor, bufSize int) error {
|
||||
cw, err := comp.Wrap(sf.cw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sf.compW = cw
|
||||
sf.bw = bufio.NewWriterSize(cw, bufSize)
|
||||
sf.memberRows = sf.rows
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) openShard(bucket string) error {
|
||||
if w.cur != nil {
|
||||
return fmt.Errorf("internal: shard already open")
|
||||
}
|
||||
if bucket != w.curBucket {
|
||||
w.seq = 0
|
||||
w.curBucket = bucket
|
||||
}
|
||||
w.seq++
|
||||
dir := BucketDir(w.opt.Dir, bucket)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("create shard directory: %w", err)
|
||||
}
|
||||
path := partialPath(dir, w.seq)
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create partial shard: %w", err)
|
||||
}
|
||||
enc, err := row.NewEncoder(w.opt.Format, w.opt.Columns, w.opt.OutIdx, row.Options{CSVNull: w.opt.CSVNull})
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
sf := &shardFile{bucket: bucket, seq: w.seq, dir: dir, partial: path, file: f, enc: enc}
|
||||
if err := sf.startChain(0); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if hdr := enc.Header(); len(hdr) > 0 {
|
||||
if _, err := sf.bw.Write(hdr); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
}
|
||||
w.cur = sf
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteChunk routes chunk rows into the active shard, sealing on bucket change
|
||||
// or when --shard-rows is reached. A chunk may span multiple shards.
|
||||
func (w *Writer) WriteChunk(c *row.Chunk) error {
|
||||
for i := range c.Rows {
|
||||
bucket := ""
|
||||
if w.opt.Mode == "date" {
|
||||
bucket = c.Buckets[i]
|
||||
}
|
||||
if w.cur != nil {
|
||||
needSeal := false
|
||||
if bucket != w.cur.bucket {
|
||||
if !schema.BucketLess(w.cur.bucket, bucket) {
|
||||
return fmt.Errorf("rows arrived out of shard order: %q after %q", bucket, w.cur.bucket)
|
||||
}
|
||||
needSeal = true
|
||||
} else if w.opt.ShardRows > 0 && w.cur.rows >= w.opt.ShardRows {
|
||||
needSeal = true
|
||||
}
|
||||
if needSeal {
|
||||
if err := w.seal(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if w.cur == nil {
|
||||
if err := w.openShard(bucket); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := w.cur.writeRow(c.Rows[i]); err != nil {
|
||||
return err
|
||||
}
|
||||
w.cur.trackKey(c.Keys[i])
|
||||
w.totalRows++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sf *shardFile) writeRow(r row.Row) error {
|
||||
var err error
|
||||
sf.buf, err = sf.enc.Encode(sf.buf[:0], r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := sf.bw.Write(sf.buf); err != nil {
|
||||
return fmt.Errorf("write shard row: %w", err)
|
||||
}
|
||||
sf.rows++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sf *shardFile) trackKey(k row.Key) {
|
||||
if sf.firstKey.IsZero() {
|
||||
sf.firstKey = k
|
||||
}
|
||||
sf.lastKey = k
|
||||
}
|
||||
|
||||
// Checkpoint makes the current shard bytes and their metadata durable. It is
|
||||
// called before destructive database work when --purge is enabled.
|
||||
func (w *Writer) Checkpoint() error {
|
||||
if w.cur == nil {
|
||||
return nil
|
||||
}
|
||||
sf := w.cur
|
||||
if sf.rows > sf.memberRows {
|
||||
if err := sf.closeMember(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
meta := &PartialMeta{
|
||||
Seq: sf.seq,
|
||||
Bucket: sf.bucket,
|
||||
Rows: sf.rows,
|
||||
Bytes: sf.cw.n,
|
||||
FirstKey: sf.firstKey,
|
||||
LastKey: sf.lastKey,
|
||||
}
|
||||
if err := savePartialMeta(sf.dir, meta); err != nil {
|
||||
return fmt.Errorf("write partial metadata: %w", err)
|
||||
}
|
||||
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sf *shardFile) closeMember() error {
|
||||
if err := sf.bw.Flush(); err != nil {
|
||||
return fmt.Errorf("flush shard: %w", err)
|
||||
}
|
||||
if err := sf.compW.Close(); err != nil {
|
||||
return fmt.Errorf("close shard compressor: %w", err)
|
||||
}
|
||||
if err := sf.file.Sync(); err != nil {
|
||||
return fmt.Errorf("sync shard: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Flush pushes buffered rows to the OS without sealing the shard.
|
||||
func (w *Writer) Flush() error {
|
||||
if w.cur == nil || w.cur.bw == nil {
|
||||
return nil
|
||||
}
|
||||
return w.cur.bw.Flush()
|
||||
}
|
||||
|
||||
// SealCurrent closes and publishes the active shard even if it is not full.
|
||||
func (w *Writer) SealCurrent() error {
|
||||
if w.cur == nil {
|
||||
return nil
|
||||
}
|
||||
if w.cur.rows == 0 {
|
||||
return w.discardCurrent()
|
||||
}
|
||||
return w.seal()
|
||||
}
|
||||
|
||||
func (w *Writer) discardCurrent() error {
|
||||
sf := w.cur
|
||||
sf.file.Close()
|
||||
os.Remove(sf.partial)
|
||||
removePartialMeta(sf.dir, sf.seq)
|
||||
w.cur = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) seal() error {
|
||||
sf := w.cur
|
||||
if sf.rows == 0 {
|
||||
return w.discardCurrent()
|
||||
}
|
||||
if sf.rows > sf.memberRows || sf.compW != nil {
|
||||
if err := sf.closeMember(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := sf.file.Close(); err != nil {
|
||||
return fmt.Errorf("close partial shard: %w", err)
|
||||
}
|
||||
lo := KeyFilePart(sf.firstKey)
|
||||
hi := KeyFilePart(sf.lastKey)
|
||||
name := FinalName(sf.seq, lo, hi, w.opt.Format, w.opt.Compressor.Ext)
|
||||
finalPath := filepath.Join(sf.dir, name)
|
||||
if err := os.Remove(finalPath); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("remove stale shard: %w", err)
|
||||
}
|
||||
if err := atomicfile.Replace(sf.partial, finalPath); err != nil {
|
||||
return fmt.Errorf("publish shard: %w", err)
|
||||
}
|
||||
sum, size, err := hashFile(finalPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash shard: %w", err)
|
||||
}
|
||||
rel, err := filepath.Rel(w.opt.Dir, finalPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entry := Entry{
|
||||
Seq: sf.seq,
|
||||
Path: filepath.ToSlash(rel),
|
||||
Date: sf.bucket,
|
||||
PKLo: lo,
|
||||
PKHi: hi,
|
||||
Rows: sf.rows,
|
||||
Bytes: size,
|
||||
SHA256: sum,
|
||||
}
|
||||
if err := w.man.Append(entry); err != nil {
|
||||
return err
|
||||
}
|
||||
removePartialMeta(sf.dir, sf.seq)
|
||||
w.shards++
|
||||
w.bytes += size
|
||||
w.cur = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close publishes any in-progress shard and closes the manifest.
|
||||
func (w *Writer) Close() error {
|
||||
if err := w.SealCurrent(); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.man.Close()
|
||||
}
|
||||
|
||||
// StatePosition reports the bucket and the last sealed seq within that bucket,
|
||||
// which is what a checkpoint must record for resume.
|
||||
func (w *Writer) StatePosition() (bucket string, sealedSeq int64) {
|
||||
if w.cur != nil {
|
||||
return w.cur.bucket, w.cur.seq - 1
|
||||
}
|
||||
return w.curBucket, w.seq
|
||||
}
|
||||
|
||||
// Abandon closes file handles without sealing the active shard, leaving the
|
||||
// partial and its metadata for a later resume.
|
||||
func (w *Writer) Abandon() error {
|
||||
if w.cur != nil {
|
||||
if w.cur.bw != nil {
|
||||
_ = w.cur.bw.Flush()
|
||||
}
|
||||
_ = w.cur.file.Close()
|
||||
w.cur = nil
|
||||
}
|
||||
return w.man.Close()
|
||||
}
|
||||
|
||||
// Stats returns rows written, shards sealed and bytes sealed.
|
||||
func (w *Writer) Stats() (rows, shards, bytes int64) {
|
||||
return w.totalRows, w.shards, w.bytes
|
||||
}
|
||||
|
||||
// CurrentShard returns the sequence number of the active shard, or the last
|
||||
// one when no shard is currently open.
|
||||
func (w *Writer) CurrentShard() int64 {
|
||||
if w.cur != nil {
|
||||
return w.cur.seq
|
||||
}
|
||||
return w.seq
|
||||
}
|
||||
|
||||
// WrittenBytes returns sealed bytes plus the bytes already written to the
|
||||
// active partial shard.
|
||||
func (w *Writer) WrittenBytes() int64 {
|
||||
n := w.bytes
|
||||
if w.cur != nil && w.cur.cw != nil {
|
||||
n += w.cur.cw.n
|
||||
}
|
||||
return n
|
||||
}
|
||||
Reference in new issue
Block a user