// Package shard writes row streams into ordered, independently importable // shard files and records them in manifest.jsonl. package shard import ( "bufio" "fmt" "io" "os" "path/filepath" "archer/internal/atomicfile" "archer/internal/row" "archer/internal/schema" ) // Options configures a Writer. type Options struct { Dir string Mode string // none|count|date ShardRows int64 Format string Compressor *Compressor Columns []row.Column OutIdx []int CSVNull string StartBucket string StartSeq int64 } // Writer owns the single active shard file. type Writer struct { opt Options man *Manifest seq int64 curBucket string cur *shardFile totalRows int64 shards int64 bytes int64 } type shardFile struct { bucket string seq int64 dir string partial string file *os.File cw *countWriter compW io.WriteCloser bw *bufio.Writer enc row.Encoder buf []byte rows int64 memberRows int64 firstKey row.Key lastKey row.Key } type countWriter struct { w io.Writer n int64 } func (c *countWriter) Write(p []byte) (int, error) { n, err := c.w.Write(p) c.n += int64(n) return n, err } // Open creates a writer. It does not open a shard until the first row arrives. func Open(opt Options, man *Manifest) (*Writer, error) { w := &Writer{opt: opt, man: man, seq: opt.StartSeq, curBucket: opt.StartBucket} return w, nil } // Resume reopens an existing partial shard and continues appending to it. func (w *Writer) Resume(m *PartialMeta) error { dir := BucketDir(w.opt.Dir, m.Bucket) path := partialPath(dir, m.Seq) f, err := os.OpenFile(path, os.O_RDWR, 0o644) if err != nil { return fmt.Errorf("resume partial shard: %w", err) } st, err := f.Stat() if err != nil { f.Close() return err } if st.Size() < m.Bytes { f.Close() return fmt.Errorf("partial shard %s is shorter (%d bytes) than its metadata (%d bytes); refusing to resume", path, st.Size(), m.Bytes) } if st.Size() > m.Bytes { if err := f.Truncate(m.Bytes); err != nil { f.Close() return fmt.Errorf("truncate partial shard: %w", err) } } if _, err := f.Seek(m.Bytes, io.SeekStart); err != nil { f.Close() return err } enc, err := row.NewEncoder(w.opt.Format, w.opt.Columns, w.opt.OutIdx, row.Options{CSVNull: w.opt.CSVNull}) if err != nil { f.Close() return err } sf := &shardFile{ bucket: m.Bucket, seq: m.Seq, dir: dir, partial: path, file: f, enc: enc, rows: m.Rows, memberRows: m.Rows, firstKey: m.FirstKey, lastKey: m.LastKey, } if err := sf.startChain(m.Bytes); err != nil { f.Close() return err } if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil { f.Close() return err } w.cur = sf w.seq = m.Seq w.curBucket = m.Bucket w.totalRows = m.Rows return nil } func (sf *shardFile) startChain(offset int64) error { sf.cw = &countWriter{w: sf.file, n: offset} return nil } func (sf *shardFile) openChain(comp *Compressor, bufSize int) error { cw, err := comp.Wrap(sf.cw) if err != nil { return err } sf.compW = cw sf.bw = bufio.NewWriterSize(cw, bufSize) sf.memberRows = sf.rows return nil } func (w *Writer) openShard(bucket string) error { if w.cur != nil { return fmt.Errorf("internal: shard already open") } if bucket != w.curBucket { w.seq = 0 w.curBucket = bucket } w.seq++ dir := BucketDir(w.opt.Dir, bucket) if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("create shard directory: %w", err) } path := partialPath(dir, w.seq) f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) if err != nil { return fmt.Errorf("create partial shard: %w", err) } enc, err := row.NewEncoder(w.opt.Format, w.opt.Columns, w.opt.OutIdx, row.Options{CSVNull: w.opt.CSVNull}) if err != nil { f.Close() return err } sf := &shardFile{bucket: bucket, seq: w.seq, dir: dir, partial: path, file: f, enc: enc} if err := sf.startChain(0); err != nil { f.Close() return err } if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil { f.Close() return err } if hdr := enc.Header(); len(hdr) > 0 { if _, err := sf.bw.Write(hdr); err != nil { f.Close() return err } } w.cur = sf return nil } // WriteChunk routes chunk rows into the active shard, sealing on bucket change // or when --shard-rows is reached. A chunk may span multiple shards. func (w *Writer) WriteChunk(c *row.Chunk) error { for i := range c.Rows { bucket := "" if w.opt.Mode == "date" { bucket = c.Buckets[i] } if w.cur != nil { needSeal := false if bucket != w.cur.bucket { if !schema.BucketLess(w.cur.bucket, bucket) { return fmt.Errorf("rows arrived out of shard order: %q after %q", bucket, w.cur.bucket) } needSeal = true } else if w.opt.ShardRows > 0 && w.cur.rows >= w.opt.ShardRows { needSeal = true } if needSeal { if err := w.seal(); err != nil { return err } } } if w.cur == nil { if err := w.openShard(bucket); err != nil { return err } } if err := w.cur.writeRow(c.Rows[i]); err != nil { return err } w.cur.trackKey(c.Keys[i]) w.totalRows++ } return nil } func (sf *shardFile) writeRow(r row.Row) error { var err error sf.buf, err = sf.enc.Encode(sf.buf[:0], r) if err != nil { return err } if _, err := sf.bw.Write(sf.buf); err != nil { return fmt.Errorf("write shard row: %w", err) } sf.rows++ return nil } func (sf *shardFile) trackKey(k row.Key) { if sf.firstKey.IsZero() { sf.firstKey = k } sf.lastKey = k } // Checkpoint makes the current shard bytes and their metadata durable. It is // called before destructive database work when --purge is enabled. func (w *Writer) Checkpoint() error { if w.cur == nil { return nil } sf := w.cur if sf.rows > sf.memberRows { if err := sf.closeMember(); err != nil { return err } } meta := &PartialMeta{ Seq: sf.seq, Bucket: sf.bucket, Rows: sf.rows, Bytes: sf.cw.n, FirstKey: sf.firstKey, LastKey: sf.lastKey, } if err := savePartialMeta(sf.dir, meta); err != nil { return fmt.Errorf("write partial metadata: %w", err) } if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil { return err } return nil } func (sf *shardFile) closeMember() error { if err := sf.bw.Flush(); err != nil { return fmt.Errorf("flush shard: %w", err) } if err := sf.compW.Close(); err != nil { return fmt.Errorf("close shard compressor: %w", err) } if err := sf.file.Sync(); err != nil { return fmt.Errorf("sync shard: %w", err) } return nil } // Flush pushes buffered rows to the OS without sealing the shard. func (w *Writer) Flush() error { if w.cur == nil || w.cur.bw == nil { return nil } return w.cur.bw.Flush() } // SealCurrent closes and publishes the active shard even if it is not full. func (w *Writer) SealCurrent() error { if w.cur == nil { return nil } if w.cur.rows == 0 { return w.discardCurrent() } return w.seal() } func (w *Writer) discardCurrent() error { sf := w.cur sf.file.Close() os.Remove(sf.partial) removePartialMeta(sf.dir, sf.seq) w.cur = nil return nil } func (w *Writer) seal() error { sf := w.cur if sf.rows == 0 { return w.discardCurrent() } if sf.rows > sf.memberRows || sf.compW != nil { if err := sf.closeMember(); err != nil { return err } } if err := sf.file.Close(); err != nil { return fmt.Errorf("close partial shard: %w", err) } lo := KeyFilePart(sf.firstKey) hi := KeyFilePart(sf.lastKey) name := FinalName(sf.seq, lo, hi, w.opt.Format, w.opt.Compressor.Ext) finalPath := filepath.Join(sf.dir, name) if err := os.Remove(finalPath); err != nil && !os.IsNotExist(err) { return fmt.Errorf("remove stale shard: %w", err) } if err := atomicfile.Replace(sf.partial, finalPath); err != nil { return fmt.Errorf("publish shard: %w", err) } sum, size, err := hashFile(finalPath) if err != nil { return fmt.Errorf("hash shard: %w", err) } rel, err := filepath.Rel(w.opt.Dir, finalPath) if err != nil { return err } entry := Entry{ Seq: sf.seq, Path: filepath.ToSlash(rel), Date: sf.bucket, PKLo: lo, PKHi: hi, Rows: sf.rows, Bytes: size, SHA256: sum, } if err := w.man.Append(entry); err != nil { return err } removePartialMeta(sf.dir, sf.seq) w.shards++ w.bytes += size w.cur = nil return nil } // Close publishes any in-progress shard and closes the manifest. func (w *Writer) Close() error { if err := w.SealCurrent(); err != nil { return err } return w.man.Close() } // StatePosition reports the bucket and the last sealed seq within that bucket, // which is what a checkpoint must record for resume. func (w *Writer) StatePosition() (bucket string, sealedSeq int64) { if w.cur != nil { return w.cur.bucket, w.cur.seq - 1 } return w.curBucket, w.seq } // Abandon closes file handles without sealing the active shard, leaving the // partial and its metadata for a later resume. func (w *Writer) Abandon() error { if w.cur != nil { if w.cur.bw != nil { _ = w.cur.bw.Flush() } _ = w.cur.file.Close() w.cur = nil } return w.man.Close() } // Stats returns rows written, shards sealed and bytes sealed. func (w *Writer) Stats() (rows, shards, bytes int64) { return w.totalRows, w.shards, w.bytes } // CurrentShard returns the sequence number of the active shard, or the last // one when no shard is currently open. func (w *Writer) CurrentShard() int64 { if w.cur != nil { return w.cur.seq } return w.seq } // WrittenBytes returns sealed bytes plus the bytes already written to the // active partial shard. func (w *Writer) WrittenBytes() int64 { n := w.bytes if w.cur != nil && w.cur.cw != nil { n += w.cur.cw.n } return n }