Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
280 lines
6.0 KiB
Go
280 lines
6.0 KiB
Go
package shard
|
|
|
|
import (
|
|
"bufio"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Entry is one line of manifest.jsonl.
|
|
type Entry struct {
|
|
Seq int64 `json:"seq"`
|
|
Path string `json:"path"`
|
|
Date string `json:"date,omitempty"`
|
|
PKLo string `json:"pk_lo"`
|
|
PKHi string `json:"pk_hi"`
|
|
Rows int64 `json:"rows"`
|
|
Bytes int64 `json:"bytes"`
|
|
SHA256 string `json:"sha256"`
|
|
ClosedAt string `json:"closed_at"`
|
|
}
|
|
|
|
// Manifest appends sealed-shard records to manifest.jsonl and knows which
|
|
// shards already exist.
|
|
type Manifest struct {
|
|
path string
|
|
f *os.File
|
|
seen map[string]bool
|
|
entries []Entry
|
|
}
|
|
|
|
const manifestName = "manifest.jsonl"
|
|
|
|
func OpenManifest(root string) (*Manifest, error) {
|
|
path := filepath.Join(root, manifestName)
|
|
m := &Manifest{path: path, seen: map[string]bool{}}
|
|
if err := m.readExisting(); err != nil {
|
|
return nil, err
|
|
}
|
|
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open manifest: %w", err)
|
|
}
|
|
m.f = f
|
|
return m, nil
|
|
}
|
|
|
|
func (m *Manifest) readExisting() error {
|
|
f, err := os.Open(m.path)
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("read manifest: %w", err)
|
|
}
|
|
defer f.Close()
|
|
sc := bufio.NewScanner(f)
|
|
sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
|
|
for sc.Scan() {
|
|
line := strings.TrimSpace(sc.Text())
|
|
if line == "" {
|
|
continue
|
|
}
|
|
var e Entry
|
|
if err := json.Unmarshal([]byte(line), &e); err != nil {
|
|
continue
|
|
}
|
|
m.entries = append(m.entries, e)
|
|
m.seen[e.Path] = true
|
|
}
|
|
return sc.Err()
|
|
}
|
|
|
|
func (m *Manifest) Has(rel string) bool { return m.seen[rel] }
|
|
|
|
func (m *Manifest) Entries() []Entry { return m.entries }
|
|
|
|
// Append writes one record and fsyncs it. A sealed shard is only considered
|
|
// complete once its manifest line is durable.
|
|
func (m *Manifest) Append(e Entry) error {
|
|
if e.ClosedAt == "" {
|
|
e.ClosedAt = time.Now().UTC().Format(time.RFC3339)
|
|
}
|
|
b, err := json.Marshal(e)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
b = append(b, '\n')
|
|
if _, err := m.f.Write(b); err != nil {
|
|
return fmt.Errorf("append manifest: %w", err)
|
|
}
|
|
if err := m.f.Sync(); err != nil {
|
|
return fmt.Errorf("sync manifest: %w", err)
|
|
}
|
|
m.seen[e.Path] = true
|
|
m.entries = append(m.entries, e)
|
|
return nil
|
|
}
|
|
|
|
func (m *Manifest) Close() error {
|
|
if m.f == nil {
|
|
return nil
|
|
}
|
|
return m.f.Close()
|
|
}
|
|
|
|
var shardNameRE = regexp.MustCompile(`^(\d+)\..*?\.([A-Za-z0-9]+?)(\.gz|\.zst|\.xz)?$`)
|
|
|
|
// Adopt scans the output tree for sealed shard files that have no manifest
|
|
// record (a crash after rename but before the manifest append) and records
|
|
// them, so resume never overwrites durable data. It returns the maximum seq
|
|
// seen per bucket.
|
|
func (m *Manifest) Adopt(root, format string, comp *Compressor) (map[string]int64, error) {
|
|
maxSeq := map[string]int64{}
|
|
root = filepath.Clean(root)
|
|
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if d.IsDir() {
|
|
return nil
|
|
}
|
|
name := d.Name()
|
|
if name == manifestName || name == "state.json" || name == "state.json.tmp" ||
|
|
strings.HasSuffix(name, ".partial") || strings.HasSuffix(name, ".partial.meta") {
|
|
return nil
|
|
}
|
|
mt := shardNameRE.FindStringSubmatch(name)
|
|
if mt == nil {
|
|
return nil
|
|
}
|
|
if mt[2] != format || mt[3] != comp.Ext {
|
|
return nil
|
|
}
|
|
rel, err := filepath.Rel(root, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rel = filepath.ToSlash(rel)
|
|
bucket := bucketOfRel(rel)
|
|
var seq int64
|
|
fmt.Sscanf(mt[1], "%d", &seq)
|
|
if seq > maxSeq[bucket] {
|
|
maxSeq[bucket] = seq
|
|
}
|
|
if m.seen[rel] {
|
|
return nil
|
|
}
|
|
rows, err := countRows(path, comp, format)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sum, size, err := hashFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
entry := Entry{
|
|
Seq: seq,
|
|
Path: rel,
|
|
Date: bucket,
|
|
Rows: rows,
|
|
Bytes: size,
|
|
SHA256: sum,
|
|
}
|
|
return m.Append(entry)
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("adopt sealed shards: %w", err)
|
|
}
|
|
return maxSeq, nil
|
|
}
|
|
|
|
func bucketOfRel(rel string) string {
|
|
dir := filepath.ToSlash(filepath.Dir(rel))
|
|
if dir == "." {
|
|
return ""
|
|
}
|
|
parts := strings.Split(dir, "/")
|
|
for _, p := range parts {
|
|
if strings.HasPrefix(p, "date=") {
|
|
return strings.TrimPrefix(p, "date=")
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// MaxSeqByBucket returns the highest seq recorded per bucket.
|
|
func (m *Manifest) MaxSeqByBucket() map[string]int64 {
|
|
out := map[string]int64{}
|
|
for _, e := range m.entries {
|
|
if e.Seq > out[e.Date] {
|
|
out[e.Date] = e.Seq
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func hashFile(path string) (string, int64, error) {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
defer f.Close()
|
|
h := sha256.New()
|
|
n, err := io.Copy(h, f)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
return hex.EncodeToString(h.Sum(nil)), n, nil
|
|
}
|
|
|
|
func countRows(path string, comp *Compressor, format string) (int64, error) {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer f.Close()
|
|
r, err := comp.Reader(f)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer r.Close()
|
|
br := bufio.NewReaderSize(r, 256*1024)
|
|
var n int64
|
|
for {
|
|
_, err := br.ReadBytes('\n')
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
n++
|
|
}
|
|
if format == "csv" && n > 0 {
|
|
n--
|
|
}
|
|
return n, nil
|
|
}
|
|
|
|
// CleanupPartials removes leftover .partial files and their metadata, keeping
|
|
// keepPartial (if non-empty) and its sidecar.
|
|
func CleanupPartials(root, keepPartial string) error {
|
|
var paths []string
|
|
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if d.IsDir() {
|
|
return nil
|
|
}
|
|
if keepPartial != "" && (path == keepPartial || path == keepPartial+".meta") {
|
|
return nil
|
|
}
|
|
name := d.Name()
|
|
if strings.HasSuffix(name, ".partial") || strings.HasSuffix(name, ".partial.meta") {
|
|
paths = append(paths, path)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sort.Strings(paths)
|
|
for _, p := range paths {
|
|
if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|