Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,279 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Entry is one line of manifest.jsonl.
|
||||
type Entry struct {
|
||||
Seq int64 `json:"seq"`
|
||||
Path string `json:"path"`
|
||||
Date string `json:"date,omitempty"`
|
||||
PKLo string `json:"pk_lo"`
|
||||
PKHi string `json:"pk_hi"`
|
||||
Rows int64 `json:"rows"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
ClosedAt string `json:"closed_at"`
|
||||
}
|
||||
|
||||
// Manifest appends sealed-shard records to manifest.jsonl and knows which
|
||||
// shards already exist.
|
||||
type Manifest struct {
|
||||
path string
|
||||
f *os.File
|
||||
seen map[string]bool
|
||||
entries []Entry
|
||||
}
|
||||
|
||||
const manifestName = "manifest.jsonl"
|
||||
|
||||
func OpenManifest(root string) (*Manifest, error) {
|
||||
path := filepath.Join(root, manifestName)
|
||||
m := &Manifest{path: path, seen: map[string]bool{}}
|
||||
if err := m.readExisting(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open manifest: %w", err)
|
||||
}
|
||||
m.f = f
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *Manifest) readExisting() error {
|
||||
f, err := os.Open(m.path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read manifest: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
sc := bufio.NewScanner(f)
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var e Entry
|
||||
if err := json.Unmarshal([]byte(line), &e); err != nil {
|
||||
continue
|
||||
}
|
||||
m.entries = append(m.entries, e)
|
||||
m.seen[e.Path] = true
|
||||
}
|
||||
return sc.Err()
|
||||
}
|
||||
|
||||
func (m *Manifest) Has(rel string) bool { return m.seen[rel] }
|
||||
|
||||
func (m *Manifest) Entries() []Entry { return m.entries }
|
||||
|
||||
// Append writes one record and fsyncs it. A sealed shard is only considered
|
||||
// complete once its manifest line is durable.
|
||||
func (m *Manifest) Append(e Entry) error {
|
||||
if e.ClosedAt == "" {
|
||||
e.ClosedAt = time.Now().UTC().Format(time.RFC3339)
|
||||
}
|
||||
b, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b = append(b, '\n')
|
||||
if _, err := m.f.Write(b); err != nil {
|
||||
return fmt.Errorf("append manifest: %w", err)
|
||||
}
|
||||
if err := m.f.Sync(); err != nil {
|
||||
return fmt.Errorf("sync manifest: %w", err)
|
||||
}
|
||||
m.seen[e.Path] = true
|
||||
m.entries = append(m.entries, e)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manifest) Close() error {
|
||||
if m.f == nil {
|
||||
return nil
|
||||
}
|
||||
return m.f.Close()
|
||||
}
|
||||
|
||||
var shardNameRE = regexp.MustCompile(`^(\d+)\..*?\.([A-Za-z0-9]+?)(\.gz|\.zst|\.xz)?$`)
|
||||
|
||||
// Adopt scans the output tree for sealed shard files that have no manifest
|
||||
// record (a crash after rename but before the manifest append) and records
|
||||
// them, so resume never overwrites durable data. It returns the maximum seq
|
||||
// seen per bucket.
|
||||
func (m *Manifest) Adopt(root, format string, comp *Compressor) (map[string]int64, error) {
|
||||
maxSeq := map[string]int64{}
|
||||
root = filepath.Clean(root)
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
if name == manifestName || name == "state.json" || name == "state.json.tmp" ||
|
||||
strings.HasSuffix(name, ".partial") || strings.HasSuffix(name, ".partial.meta") {
|
||||
return nil
|
||||
}
|
||||
mt := shardNameRE.FindStringSubmatch(name)
|
||||
if mt == nil {
|
||||
return nil
|
||||
}
|
||||
if mt[2] != format || mt[3] != comp.Ext {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
bucket := bucketOfRel(rel)
|
||||
var seq int64
|
||||
fmt.Sscanf(mt[1], "%d", &seq)
|
||||
if seq > maxSeq[bucket] {
|
||||
maxSeq[bucket] = seq
|
||||
}
|
||||
if m.seen[rel] {
|
||||
return nil
|
||||
}
|
||||
rows, err := countRows(path, comp, format)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sum, size, err := hashFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entry := Entry{
|
||||
Seq: seq,
|
||||
Path: rel,
|
||||
Date: bucket,
|
||||
Rows: rows,
|
||||
Bytes: size,
|
||||
SHA256: sum,
|
||||
}
|
||||
return m.Append(entry)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("adopt sealed shards: %w", err)
|
||||
}
|
||||
return maxSeq, nil
|
||||
}
|
||||
|
||||
func bucketOfRel(rel string) string {
|
||||
dir := filepath.ToSlash(filepath.Dir(rel))
|
||||
if dir == "." {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(dir, "/")
|
||||
for _, p := range parts {
|
||||
if strings.HasPrefix(p, "date=") {
|
||||
return strings.TrimPrefix(p, "date=")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// MaxSeqByBucket returns the highest seq recorded per bucket.
|
||||
func (m *Manifest) MaxSeqByBucket() map[string]int64 {
|
||||
out := map[string]int64{}
|
||||
for _, e := range m.entries {
|
||||
if e.Seq > out[e.Date] {
|
||||
out[e.Date] = e.Seq
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func hashFile(path string) (string, int64, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(h, f)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), n, nil
|
||||
}
|
||||
|
||||
func countRows(path string, comp *Compressor, format string) (int64, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
r, err := comp.Reader(f)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer r.Close()
|
||||
br := bufio.NewReaderSize(r, 256*1024)
|
||||
var n int64
|
||||
for {
|
||||
_, err := br.ReadBytes('\n')
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n++
|
||||
}
|
||||
if format == "csv" && n > 0 {
|
||||
n--
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// CleanupPartials removes leftover .partial files and their metadata, keeping
|
||||
// keepPartial (if non-empty) and its sidecar.
|
||||
func CleanupPartials(root, keepPartial string) error {
|
||||
var paths []string
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if keepPartial != "" && (path == keepPartial || path == keepPartial+".meta") {
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
if strings.HasSuffix(name, ".partial") || strings.HasSuffix(name, ".partial.meta") {
|
||||
paths = append(paths, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Strings(paths)
|
||||
for _, p := range paths {
|
||||
if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user