Files
archer/internal/shard/manifest.go
T
voidandClaude Sonnet 4.6 366d61ca25 Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-11-08 21:14:23 +08:00

317 lines
7.1 KiB
Go

package shard
import (
"bufio"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
)
// Entry is one line of manifest.jsonl.
type Entry struct {
Seq int64 `json:"seq"`
Path string `json:"path"`
Date string `json:"date,omitempty"`
PKLo string `json:"pk_lo"`
PKHi string `json:"pk_hi"`
Rows int64 `json:"rows"`
Bytes int64 `json:"bytes"`
SHA256 string `json:"sha256"`
ClosedAt string `json:"closed_at"`
}
// LoadManifest reads manifest.jsonl strictly for replay. Unlike OpenManifest it
// never creates or appends to the archive, and a malformed line is an error
// rather than a line to skip: an importer must not mistake corruption for a
// missing shard.
func LoadManifest(root string) ([]Entry, error) {
path := filepath.Join(root, manifestName)
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("open manifest: %w", err)
}
defer f.Close()
var entries []Entry
sc := bufio.NewScanner(f)
sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
line := 0
for sc.Scan() {
line++
text := strings.TrimSpace(sc.Text())
if text == "" {
continue
}
var e Entry
if err := json.Unmarshal([]byte(text), &e); err != nil {
return nil, fmt.Errorf("parse manifest.jsonl line %d: %w", line, err)
}
if e.Path == "" {
return nil, fmt.Errorf("manifest.jsonl line %d: missing path", line)
}
entries = append(entries, e)
}
if err := sc.Err(); err != nil {
return nil, fmt.Errorf("read manifest: %w", err)
}
return entries, nil
}
// Manifest appends sealed-shard records to manifest.jsonl and knows which
// shards already exist.
type Manifest struct {
path string
f *os.File
seen map[string]bool
entries []Entry
}
const manifestName = "manifest.jsonl"
func OpenManifest(root string) (*Manifest, error) {
path := filepath.Join(root, manifestName)
m := &Manifest{path: path, seen: map[string]bool{}}
if err := m.readExisting(); err != nil {
return nil, err
}
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
if err != nil {
return nil, fmt.Errorf("open manifest: %w", err)
}
m.f = f
return m, nil
}
func (m *Manifest) readExisting() error {
f, err := os.Open(m.path)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return fmt.Errorf("read manifest: %w", err)
}
defer f.Close()
sc := bufio.NewScanner(f)
sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
for sc.Scan() {
line := strings.TrimSpace(sc.Text())
if line == "" {
continue
}
var e Entry
if err := json.Unmarshal([]byte(line), &e); err != nil {
continue
}
m.entries = append(m.entries, e)
m.seen[e.Path] = true
}
return sc.Err()
}
func (m *Manifest) Has(rel string) bool { return m.seen[rel] }
func (m *Manifest) Entries() []Entry { return m.entries }
// Append writes one record and fsyncs it. A sealed shard is only considered
// complete once its manifest line is durable.
func (m *Manifest) Append(e Entry) error {
if e.ClosedAt == "" {
e.ClosedAt = time.Now().UTC().Format(time.RFC3339)
}
b, err := json.Marshal(e)
if err != nil {
return err
}
b = append(b, '\n')
if _, err := m.f.Write(b); err != nil {
return fmt.Errorf("append manifest: %w", err)
}
if err := m.f.Sync(); err != nil {
return fmt.Errorf("sync manifest: %w", err)
}
m.seen[e.Path] = true
m.entries = append(m.entries, e)
return nil
}
func (m *Manifest) Close() error {
if m.f == nil {
return nil
}
return m.f.Close()
}
var shardNameRE = regexp.MustCompile(`^(\d+)\..*?\.([A-Za-z0-9]+?)(\.gz|\.zst|\.xz)?$`)
// Adopt scans the output tree for sealed shard files that have no manifest
// record (a crash after rename but before the manifest append) and records
// them, so resume never overwrites durable data. It returns the maximum seq
// seen per bucket.
func (m *Manifest) Adopt(root, format string, comp *Compressor) (map[string]int64, error) {
maxSeq := map[string]int64{}
root = filepath.Clean(root)
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
name := d.Name()
if name == manifestName || name == "state.json" || name == "state.json.tmp" ||
strings.HasSuffix(name, ".partial") || strings.HasSuffix(name, ".partial.meta") {
return nil
}
mt := shardNameRE.FindStringSubmatch(name)
if mt == nil {
return nil
}
if mt[2] != format || mt[3] != comp.Ext {
return nil
}
rel, err := filepath.Rel(root, path)
if err != nil {
return err
}
rel = filepath.ToSlash(rel)
bucket := bucketOfRel(rel)
var seq int64
fmt.Sscanf(mt[1], "%d", &seq)
if seq > maxSeq[bucket] {
maxSeq[bucket] = seq
}
if m.seen[rel] {
return nil
}
rows, err := countRows(path, comp, format)
if err != nil {
return err
}
sum, size, err := hashFile(path)
if err != nil {
return err
}
entry := Entry{
Seq: seq,
Path: rel,
Date: bucket,
Rows: rows,
Bytes: size,
SHA256: sum,
}
return m.Append(entry)
})
if err != nil {
return nil, fmt.Errorf("adopt sealed shards: %w", err)
}
return maxSeq, nil
}
func bucketOfRel(rel string) string {
dir := filepath.ToSlash(filepath.Dir(rel))
if dir == "." {
return ""
}
parts := strings.Split(dir, "/")
for _, p := range parts {
if strings.HasPrefix(p, "date=") {
return strings.TrimPrefix(p, "date=")
}
}
return ""
}
// MaxSeqByBucket returns the highest seq recorded per bucket.
func (m *Manifest) MaxSeqByBucket() map[string]int64 {
out := map[string]int64{}
for _, e := range m.entries {
if e.Seq > out[e.Date] {
out[e.Date] = e.Seq
}
}
return out
}
func hashFile(path string) (string, int64, error) {
f, err := os.Open(path)
if err != nil {
return "", 0, err
}
defer f.Close()
h := sha256.New()
n, err := io.Copy(h, f)
if err != nil {
return "", 0, err
}
return hex.EncodeToString(h.Sum(nil)), n, nil
}
func countRows(path string, comp *Compressor, format string) (int64, error) {
f, err := os.Open(path)
if err != nil {
return 0, err
}
defer f.Close()
r, err := comp.Reader(f)
if err != nil {
return 0, err
}
defer r.Close()
br := bufio.NewReaderSize(r, 256*1024)
var n int64
for {
_, err := br.ReadBytes('\n')
if err == io.EOF {
break
}
if err != nil {
return 0, err
}
n++
}
if format == "csv" && n > 0 {
n--
}
return n, nil
}
// CleanupPartials removes leftover .partial files and their metadata, keeping
// keepPartial (if non-empty) and its sidecar.
func CleanupPartials(root, keepPartial string) error {
var paths []string
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
if keepPartial != "" && (path == keepPartial || path == keepPartial+".meta") {
return nil
}
name := d.Name()
if strings.HasSuffix(name, ".partial") || strings.HasSuffix(name, ".partial.meta") {
paths = append(paths, path)
}
return nil
})
if err != nil {
return err
}
sort.Strings(paths)
for _, p := range paths {
if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
return err
}
}
return nil
}