package shard import ( "bufio" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "io" "os" "path/filepath" "regexp" "sort" "strings" "time" ) // Entry is one line of manifest.jsonl. type Entry struct { Seq int64 `json:"seq"` Path string `json:"path"` Date string `json:"date,omitempty"` PKLo string `json:"pk_lo"` PKHi string `json:"pk_hi"` Rows int64 `json:"rows"` Bytes int64 `json:"bytes"` SHA256 string `json:"sha256"` ClosedAt string `json:"closed_at"` } // Manifest appends sealed-shard records to manifest.jsonl and knows which // shards already exist. type Manifest struct { path string f *os.File seen map[string]bool entries []Entry } const manifestName = "manifest.jsonl" func OpenManifest(root string) (*Manifest, error) { path := filepath.Join(root, manifestName) m := &Manifest{path: path, seen: map[string]bool{}} if err := m.readExisting(); err != nil { return nil, err } f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) if err != nil { return nil, fmt.Errorf("open manifest: %w", err) } m.f = f return m, nil } func (m *Manifest) readExisting() error { f, err := os.Open(m.path) if os.IsNotExist(err) { return nil } if err != nil { return fmt.Errorf("read manifest: %w", err) } defer f.Close() sc := bufio.NewScanner(f) sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) for sc.Scan() { line := strings.TrimSpace(sc.Text()) if line == "" { continue } var e Entry if err := json.Unmarshal([]byte(line), &e); err != nil { continue } m.entries = append(m.entries, e) m.seen[e.Path] = true } return sc.Err() } func (m *Manifest) Has(rel string) bool { return m.seen[rel] } func (m *Manifest) Entries() []Entry { return m.entries } // Append writes one record and fsyncs it. A sealed shard is only considered // complete once its manifest line is durable. func (m *Manifest) Append(e Entry) error { if e.ClosedAt == "" { e.ClosedAt = time.Now().UTC().Format(time.RFC3339) } b, err := json.Marshal(e) if err != nil { return err } b = append(b, '\n') if _, err := m.f.Write(b); err != nil { return fmt.Errorf("append manifest: %w", err) } if err := m.f.Sync(); err != nil { return fmt.Errorf("sync manifest: %w", err) } m.seen[e.Path] = true m.entries = append(m.entries, e) return nil } func (m *Manifest) Close() error { if m.f == nil { return nil } return m.f.Close() } var shardNameRE = regexp.MustCompile(`^(\d+)\..*?\.([A-Za-z0-9]+?)(\.gz|\.zst|\.xz)?$`) // Adopt scans the output tree for sealed shard files that have no manifest // record (a crash after rename but before the manifest append) and records // them, so resume never overwrites durable data. It returns the maximum seq // seen per bucket. func (m *Manifest) Adopt(root, format string, comp *Compressor) (map[string]int64, error) { maxSeq := map[string]int64{} root = filepath.Clean(root) err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { if err != nil { return err } if d.IsDir() { return nil } name := d.Name() if name == manifestName || name == "state.json" || name == "state.json.tmp" || strings.HasSuffix(name, ".partial") || strings.HasSuffix(name, ".partial.meta") { return nil } mt := shardNameRE.FindStringSubmatch(name) if mt == nil { return nil } if mt[2] != format || mt[3] != comp.Ext { return nil } rel, err := filepath.Rel(root, path) if err != nil { return err } rel = filepath.ToSlash(rel) bucket := bucketOfRel(rel) var seq int64 fmt.Sscanf(mt[1], "%d", &seq) if seq > maxSeq[bucket] { maxSeq[bucket] = seq } if m.seen[rel] { return nil } rows, err := countRows(path, comp, format) if err != nil { return err } sum, size, err := hashFile(path) if err != nil { return err } entry := Entry{ Seq: seq, Path: rel, Date: bucket, Rows: rows, Bytes: size, SHA256: sum, } return m.Append(entry) }) if err != nil { return nil, fmt.Errorf("adopt sealed shards: %w", err) } return maxSeq, nil } func bucketOfRel(rel string) string { dir := filepath.ToSlash(filepath.Dir(rel)) if dir == "." { return "" } parts := strings.Split(dir, "/") for _, p := range parts { if strings.HasPrefix(p, "date=") { return strings.TrimPrefix(p, "date=") } } return "" } // MaxSeqByBucket returns the highest seq recorded per bucket. func (m *Manifest) MaxSeqByBucket() map[string]int64 { out := map[string]int64{} for _, e := range m.entries { if e.Seq > out[e.Date] { out[e.Date] = e.Seq } } return out } func hashFile(path string) (string, int64, error) { f, err := os.Open(path) if err != nil { return "", 0, err } defer f.Close() h := sha256.New() n, err := io.Copy(h, f) if err != nil { return "", 0, err } return hex.EncodeToString(h.Sum(nil)), n, nil } func countRows(path string, comp *Compressor, format string) (int64, error) { f, err := os.Open(path) if err != nil { return 0, err } defer f.Close() r, err := comp.Reader(f) if err != nil { return 0, err } defer r.Close() br := bufio.NewReaderSize(r, 256*1024) var n int64 for { _, err := br.ReadBytes('\n') if err == io.EOF { break } if err != nil { return 0, err } n++ } if format == "csv" && n > 0 { n-- } return n, nil } // CleanupPartials removes leftover .partial files and their metadata, keeping // keepPartial (if non-empty) and its sidecar. func CleanupPartials(root, keepPartial string) error { var paths []string err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { if err != nil { return err } if d.IsDir() { return nil } if keepPartial != "" && (path == keepPartial || path == keepPartial+".meta") { return nil } name := d.Name() if strings.HasSuffix(name, ".partial") || strings.HasSuffix(name, ".partial.meta") { paths = append(paths, path) } return nil }) if err != nil { return err } sort.Strings(paths) for _, p := range paths { if err := os.Remove(p); err != nil && !os.IsNotExist(err) { return err } } return nil }