Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,132 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
// DefaultLevel is the compression level used when none is specified. It
|
||||
// matches the byte output of archer before --compress-level existed.
|
||||
const DefaultLevel = 3
|
||||
|
||||
// xzReadDictCap is the largest dictionary xz can produce here (level 9).
|
||||
// Readers always allow it so lowering the level with --force-resume does not
|
||||
// make previously written shards unreadable.
|
||||
const xzReadDictCap = 64 << 20
|
||||
|
||||
// Compressor wraps shard byte streams. Every shard is compressed on its own
|
||||
// so a single file can be decompressed and imported without its siblings.
|
||||
// Level is a format-neutral 1..9 scale where 9 is the strongest setting.
|
||||
type Compressor struct {
|
||||
Name string // "none", "gzip", "zstd", "xz"
|
||||
Ext string // "", ".gz", ".zst", ".xz"
|
||||
Level int
|
||||
}
|
||||
|
||||
func NewCompressor(name string, level int) (*Compressor, error) {
|
||||
if level < 1 || level > 9 {
|
||||
return nil, fmt.Errorf("compression level %d out of range 1-9", level)
|
||||
}
|
||||
switch name {
|
||||
case "", "none":
|
||||
return &Compressor{Name: "none", Level: level}, nil
|
||||
case "gzip":
|
||||
return &Compressor{Name: "gzip", Ext: ".gz", Level: level}, nil
|
||||
case "zstd":
|
||||
return &Compressor{Name: "zstd", Ext: ".zst", Level: level}, nil
|
||||
case "xz":
|
||||
return &Compressor{Name: "xz", Ext: ".xz", Level: level}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported compression %q", name)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Compressor) Wrap(w io.Writer) (io.WriteCloser, error) {
|
||||
switch c.Name {
|
||||
case "gzip":
|
||||
return gzip.NewWriterLevel(w, gzipLevel(c.Level))
|
||||
case "zstd":
|
||||
return zstd.NewWriter(w, zstd.WithEncoderLevel(zstdLevel(c.Level)))
|
||||
case "xz":
|
||||
xw, err := xz.WriterConfig{DictCap: xzDictCap(c.Level)}.NewWriter(w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return xw, nil
|
||||
default:
|
||||
return nopCloser{w}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Reader returns a decompressing reader for an existing shard.
|
||||
func (c *Compressor) Reader(r io.Reader) (io.ReadCloser, error) {
|
||||
switch c.Name {
|
||||
case "gzip":
|
||||
return gzip.NewReader(r)
|
||||
case "zstd":
|
||||
zr, err := zstd.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return zr.IOReadCloser(), nil
|
||||
case "xz":
|
||||
xr, err := xz.ReaderConfig{DictCap: xzReadDictCap}.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.NopCloser(xr), nil
|
||||
default:
|
||||
return io.NopCloser(r), nil
|
||||
}
|
||||
}
|
||||
|
||||
// gzipLevel passes the neutral level straight through.
|
||||
func gzipLevel(level int) int {
|
||||
return level
|
||||
}
|
||||
|
||||
func zstdLevel(level int) zstd.EncoderLevel {
|
||||
switch {
|
||||
case level <= 2:
|
||||
return zstd.SpeedFastest
|
||||
case level <= 5:
|
||||
return zstd.SpeedDefault
|
||||
case level <= 8:
|
||||
return zstd.SpeedBetterCompression
|
||||
default:
|
||||
return zstd.SpeedBestCompression
|
||||
}
|
||||
}
|
||||
|
||||
// xzDictCap maps the neutral level onto xz dictionary sizes; the library has
|
||||
// no preset parameter, so the dictionary is the only strength knob.
|
||||
func xzDictCap(level int) int {
|
||||
switch level {
|
||||
case 1:
|
||||
return 1 << 20
|
||||
case 2:
|
||||
return 2 << 20
|
||||
case 3, 4:
|
||||
return 4 << 20
|
||||
case 5, 6:
|
||||
return 8 << 20
|
||||
case 7:
|
||||
return 16 << 20
|
||||
case 8:
|
||||
return 32 << 20
|
||||
default:
|
||||
return 64 << 20
|
||||
}
|
||||
}
|
||||
|
||||
type nopCloser struct{ io.Writer }
|
||||
|
||||
func (nopCloser) Close() error { return nil }
|
||||
|
||||
type flushCloser interface {
|
||||
Flush() error
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Entry is one line of manifest.jsonl.
|
||||
type Entry struct {
|
||||
Seq int64 `json:"seq"`
|
||||
Path string `json:"path"`
|
||||
Date string `json:"date,omitempty"`
|
||||
PKLo string `json:"pk_lo"`
|
||||
PKHi string `json:"pk_hi"`
|
||||
Rows int64 `json:"rows"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
ClosedAt string `json:"closed_at"`
|
||||
}
|
||||
|
||||
// Manifest appends sealed-shard records to manifest.jsonl and knows which
|
||||
// shards already exist.
|
||||
type Manifest struct {
|
||||
path string
|
||||
f *os.File
|
||||
seen map[string]bool
|
||||
entries []Entry
|
||||
}
|
||||
|
||||
const manifestName = "manifest.jsonl"
|
||||
|
||||
func OpenManifest(root string) (*Manifest, error) {
|
||||
path := filepath.Join(root, manifestName)
|
||||
m := &Manifest{path: path, seen: map[string]bool{}}
|
||||
if err := m.readExisting(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open manifest: %w", err)
|
||||
}
|
||||
m.f = f
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *Manifest) readExisting() error {
|
||||
f, err := os.Open(m.path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read manifest: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
sc := bufio.NewScanner(f)
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var e Entry
|
||||
if err := json.Unmarshal([]byte(line), &e); err != nil {
|
||||
continue
|
||||
}
|
||||
m.entries = append(m.entries, e)
|
||||
m.seen[e.Path] = true
|
||||
}
|
||||
return sc.Err()
|
||||
}
|
||||
|
||||
func (m *Manifest) Has(rel string) bool { return m.seen[rel] }
|
||||
|
||||
func (m *Manifest) Entries() []Entry { return m.entries }
|
||||
|
||||
// Append writes one record and fsyncs it. A sealed shard is only considered
|
||||
// complete once its manifest line is durable.
|
||||
func (m *Manifest) Append(e Entry) error {
|
||||
if e.ClosedAt == "" {
|
||||
e.ClosedAt = time.Now().UTC().Format(time.RFC3339)
|
||||
}
|
||||
b, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b = append(b, '\n')
|
||||
if _, err := m.f.Write(b); err != nil {
|
||||
return fmt.Errorf("append manifest: %w", err)
|
||||
}
|
||||
if err := m.f.Sync(); err != nil {
|
||||
return fmt.Errorf("sync manifest: %w", err)
|
||||
}
|
||||
m.seen[e.Path] = true
|
||||
m.entries = append(m.entries, e)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manifest) Close() error {
|
||||
if m.f == nil {
|
||||
return nil
|
||||
}
|
||||
return m.f.Close()
|
||||
}
|
||||
|
||||
var shardNameRE = regexp.MustCompile(`^(\d+)\..*?\.([A-Za-z0-9]+?)(\.gz|\.zst|\.xz)?$`)
|
||||
|
||||
// Adopt scans the output tree for sealed shard files that have no manifest
|
||||
// record (a crash after rename but before the manifest append) and records
|
||||
// them, so resume never overwrites durable data. It returns the maximum seq
|
||||
// seen per bucket.
|
||||
func (m *Manifest) Adopt(root, format string, comp *Compressor) (map[string]int64, error) {
|
||||
maxSeq := map[string]int64{}
|
||||
root = filepath.Clean(root)
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
if name == manifestName || name == "state.json" || name == "state.json.tmp" ||
|
||||
strings.HasSuffix(name, ".partial") || strings.HasSuffix(name, ".partial.meta") {
|
||||
return nil
|
||||
}
|
||||
mt := shardNameRE.FindStringSubmatch(name)
|
||||
if mt == nil {
|
||||
return nil
|
||||
}
|
||||
if mt[2] != format || mt[3] != comp.Ext {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
bucket := bucketOfRel(rel)
|
||||
var seq int64
|
||||
fmt.Sscanf(mt[1], "%d", &seq)
|
||||
if seq > maxSeq[bucket] {
|
||||
maxSeq[bucket] = seq
|
||||
}
|
||||
if m.seen[rel] {
|
||||
return nil
|
||||
}
|
||||
rows, err := countRows(path, comp, format)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sum, size, err := hashFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entry := Entry{
|
||||
Seq: seq,
|
||||
Path: rel,
|
||||
Date: bucket,
|
||||
Rows: rows,
|
||||
Bytes: size,
|
||||
SHA256: sum,
|
||||
}
|
||||
return m.Append(entry)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("adopt sealed shards: %w", err)
|
||||
}
|
||||
return maxSeq, nil
|
||||
}
|
||||
|
||||
func bucketOfRel(rel string) string {
|
||||
dir := filepath.ToSlash(filepath.Dir(rel))
|
||||
if dir == "." {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(dir, "/")
|
||||
for _, p := range parts {
|
||||
if strings.HasPrefix(p, "date=") {
|
||||
return strings.TrimPrefix(p, "date=")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// MaxSeqByBucket returns the highest seq recorded per bucket.
|
||||
func (m *Manifest) MaxSeqByBucket() map[string]int64 {
|
||||
out := map[string]int64{}
|
||||
for _, e := range m.entries {
|
||||
if e.Seq > out[e.Date] {
|
||||
out[e.Date] = e.Seq
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func hashFile(path string) (string, int64, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(h, f)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), n, nil
|
||||
}
|
||||
|
||||
func countRows(path string, comp *Compressor, format string) (int64, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
r, err := comp.Reader(f)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer r.Close()
|
||||
br := bufio.NewReaderSize(r, 256*1024)
|
||||
var n int64
|
||||
for {
|
||||
_, err := br.ReadBytes('\n')
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n++
|
||||
}
|
||||
if format == "csv" && n > 0 {
|
||||
n--
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// CleanupPartials removes leftover .partial files and their metadata, keeping
|
||||
// keepPartial (if non-empty) and its sidecar.
|
||||
func CleanupPartials(root, keepPartial string) error {
|
||||
var paths []string
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if keepPartial != "" && (path == keepPartial || path == keepPartial+".meta") {
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
if strings.HasSuffix(name, ".partial") || strings.HasSuffix(name, ".partial.meta") {
|
||||
paths = append(paths, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Strings(paths)
|
||||
for _, p := range paths {
|
||||
if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
// FinalName builds the sealed shard filename:
|
||||
// <seq:06d>.<lo>-<hi>.<format>[.<comp>]
|
||||
func FinalName(seq int64, lo, hi, format, compExt string) string {
|
||||
return fmt.Sprintf("%06d.%s-%s.%s%s", seq, safeNamePart(lo), safeNamePart(hi), format, compExt)
|
||||
}
|
||||
|
||||
// KeyFilePart renders a key tuple into a filename-safe fragment.
|
||||
func KeyFilePart(k row.Key) string {
|
||||
if len(k) == 0 {
|
||||
return "NULL"
|
||||
}
|
||||
parts := make([]string, len(k))
|
||||
for i, p := range k {
|
||||
if p.Null {
|
||||
parts[i] = "NULL"
|
||||
} else {
|
||||
parts[i] = p.Val
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
func safeNamePart(s string) string {
|
||||
if s == "" {
|
||||
return "_"
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == '.', r == '_', r == '-':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteByte('_')
|
||||
}
|
||||
if b.Len() >= 96 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if b.Len() == 0 {
|
||||
return "_"
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// BucketDir returns the directory that holds shards for a bucket.
|
||||
func BucketDir(root, bucket string) string {
|
||||
if bucket == "" {
|
||||
return root
|
||||
}
|
||||
return filepath.Join(root, "date="+safeNamePart(bucket))
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"archer/internal/atomicfile"
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
// PartialMeta describes the durable prefix of an in-progress shard. It is
|
||||
// written before each chunk is committed, so a resume can truncate the file
|
||||
// back to a consistent offset and continue appending.
|
||||
type PartialMeta struct {
|
||||
Seq int64 `json:"seq"`
|
||||
Bucket string `json:"bucket,omitempty"`
|
||||
Rows int64 `json:"rows"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
FirstKey row.Key `json:"first_key"`
|
||||
LastKey row.Key `json:"last_key"`
|
||||
}
|
||||
|
||||
func partialPath(dir string, seq int64) string {
|
||||
return filepath.Join(dir, fmt.Sprintf("%06d.partial", seq))
|
||||
}
|
||||
|
||||
func partialMetaPath(dir string, seq int64) string {
|
||||
return partialPath(dir, seq) + ".meta"
|
||||
}
|
||||
|
||||
// PartialPath exposes the partial path for tests and diagnostics.
|
||||
func PartialPath(root, bucket string, seq int64) string {
|
||||
return partialPath(BucketDir(root, bucket), seq)
|
||||
}
|
||||
|
||||
// LoadPartialMeta reads the sidecar for an in-progress shard. A missing file
|
||||
// returns (nil, nil).
|
||||
func LoadPartialMeta(root, bucket string, seq int64) (*PartialMeta, error) {
|
||||
path := partialMetaPath(BucketDir(root, bucket), seq)
|
||||
b, err := os.ReadFile(path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read partial meta: %w", err)
|
||||
}
|
||||
var m PartialMeta
|
||||
if err := json.Unmarshal(b, &m); err != nil {
|
||||
return nil, fmt.Errorf("parse partial meta: %w", err)
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
func savePartialMeta(dir string, m *PartialMeta) error {
|
||||
path := partialMetaPath(dir, m.Seq)
|
||||
b, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := f.Write(b); err != nil {
|
||||
f.Close()
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
f.Close()
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := atomicfile.Replace(tmp, path); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func removePartialMeta(dir string, seq int64) {
|
||||
os.Remove(partialMetaPath(dir, seq))
|
||||
}
|
||||
@@ -0,0 +1,426 @@
|
||||
// Package shard writes row streams into ordered, independently importable
|
||||
// shard files and records them in manifest.jsonl.
|
||||
package shard
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"archer/internal/atomicfile"
|
||||
"archer/internal/row"
|
||||
"archer/internal/schema"
|
||||
)
|
||||
|
||||
// Options configures a Writer.
|
||||
type Options struct {
|
||||
Dir string
|
||||
Mode string // none|count|date
|
||||
ShardRows int64
|
||||
Format string
|
||||
Compressor *Compressor
|
||||
Columns []row.Column
|
||||
OutIdx []int
|
||||
CSVNull string
|
||||
StartBucket string
|
||||
StartSeq int64
|
||||
}
|
||||
|
||||
// Writer owns the single active shard file.
|
||||
type Writer struct {
|
||||
opt Options
|
||||
man *Manifest
|
||||
seq int64
|
||||
curBucket string
|
||||
cur *shardFile
|
||||
|
||||
totalRows int64
|
||||
shards int64
|
||||
bytes int64
|
||||
}
|
||||
|
||||
type shardFile struct {
|
||||
bucket string
|
||||
seq int64
|
||||
dir string
|
||||
partial string
|
||||
file *os.File
|
||||
cw *countWriter
|
||||
compW io.WriteCloser
|
||||
bw *bufio.Writer
|
||||
enc row.Encoder
|
||||
buf []byte
|
||||
rows int64
|
||||
memberRows int64
|
||||
firstKey row.Key
|
||||
lastKey row.Key
|
||||
}
|
||||
|
||||
type countWriter struct {
|
||||
w io.Writer
|
||||
n int64
|
||||
}
|
||||
|
||||
func (c *countWriter) Write(p []byte) (int, error) {
|
||||
n, err := c.w.Write(p)
|
||||
c.n += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Open creates a writer. It does not open a shard until the first row arrives.
|
||||
func Open(opt Options, man *Manifest) (*Writer, error) {
|
||||
w := &Writer{opt: opt, man: man, seq: opt.StartSeq, curBucket: opt.StartBucket}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// Resume reopens an existing partial shard and continues appending to it.
|
||||
func (w *Writer) Resume(m *PartialMeta) error {
|
||||
dir := BucketDir(w.opt.Dir, m.Bucket)
|
||||
path := partialPath(dir, m.Seq)
|
||||
f, err := os.OpenFile(path, os.O_RDWR, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resume partial shard: %w", err)
|
||||
}
|
||||
st, err := f.Stat()
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if st.Size() < m.Bytes {
|
||||
f.Close()
|
||||
return fmt.Errorf("partial shard %s is shorter (%d bytes) than its metadata (%d bytes); refusing to resume", path, st.Size(), m.Bytes)
|
||||
}
|
||||
if st.Size() > m.Bytes {
|
||||
if err := f.Truncate(m.Bytes); err != nil {
|
||||
f.Close()
|
||||
return fmt.Errorf("truncate partial shard: %w", err)
|
||||
}
|
||||
}
|
||||
if _, err := f.Seek(m.Bytes, io.SeekStart); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
enc, err := row.NewEncoder(w.opt.Format, w.opt.Columns, w.opt.OutIdx, row.Options{CSVNull: w.opt.CSVNull})
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
sf := &shardFile{
|
||||
bucket: m.Bucket,
|
||||
seq: m.Seq,
|
||||
dir: dir,
|
||||
partial: path,
|
||||
file: f,
|
||||
enc: enc,
|
||||
rows: m.Rows,
|
||||
memberRows: m.Rows,
|
||||
firstKey: m.FirstKey,
|
||||
lastKey: m.LastKey,
|
||||
}
|
||||
if err := sf.startChain(m.Bytes); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
w.cur = sf
|
||||
w.seq = m.Seq
|
||||
w.curBucket = m.Bucket
|
||||
w.totalRows = m.Rows
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sf *shardFile) startChain(offset int64) error {
|
||||
sf.cw = &countWriter{w: sf.file, n: offset}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sf *shardFile) openChain(comp *Compressor, bufSize int) error {
|
||||
cw, err := comp.Wrap(sf.cw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sf.compW = cw
|
||||
sf.bw = bufio.NewWriterSize(cw, bufSize)
|
||||
sf.memberRows = sf.rows
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) openShard(bucket string) error {
|
||||
if w.cur != nil {
|
||||
return fmt.Errorf("internal: shard already open")
|
||||
}
|
||||
if bucket != w.curBucket {
|
||||
w.seq = 0
|
||||
w.curBucket = bucket
|
||||
}
|
||||
w.seq++
|
||||
dir := BucketDir(w.opt.Dir, bucket)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("create shard directory: %w", err)
|
||||
}
|
||||
path := partialPath(dir, w.seq)
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create partial shard: %w", err)
|
||||
}
|
||||
enc, err := row.NewEncoder(w.opt.Format, w.opt.Columns, w.opt.OutIdx, row.Options{CSVNull: w.opt.CSVNull})
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
sf := &shardFile{bucket: bucket, seq: w.seq, dir: dir, partial: path, file: f, enc: enc}
|
||||
if err := sf.startChain(0); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if hdr := enc.Header(); len(hdr) > 0 {
|
||||
if _, err := sf.bw.Write(hdr); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
}
|
||||
w.cur = sf
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteChunk routes chunk rows into the active shard, sealing on bucket change
|
||||
// or when --shard-rows is reached. A chunk may span multiple shards.
|
||||
func (w *Writer) WriteChunk(c *row.Chunk) error {
|
||||
for i := range c.Rows {
|
||||
bucket := ""
|
||||
if w.opt.Mode == "date" {
|
||||
bucket = c.Buckets[i]
|
||||
}
|
||||
if w.cur != nil {
|
||||
needSeal := false
|
||||
if bucket != w.cur.bucket {
|
||||
if !schema.BucketLess(w.cur.bucket, bucket) {
|
||||
return fmt.Errorf("rows arrived out of shard order: %q after %q", bucket, w.cur.bucket)
|
||||
}
|
||||
needSeal = true
|
||||
} else if w.opt.ShardRows > 0 && w.cur.rows >= w.opt.ShardRows {
|
||||
needSeal = true
|
||||
}
|
||||
if needSeal {
|
||||
if err := w.seal(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if w.cur == nil {
|
||||
if err := w.openShard(bucket); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := w.cur.writeRow(c.Rows[i]); err != nil {
|
||||
return err
|
||||
}
|
||||
w.cur.trackKey(c.Keys[i])
|
||||
w.totalRows++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sf *shardFile) writeRow(r row.Row) error {
|
||||
var err error
|
||||
sf.buf, err = sf.enc.Encode(sf.buf[:0], r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := sf.bw.Write(sf.buf); err != nil {
|
||||
return fmt.Errorf("write shard row: %w", err)
|
||||
}
|
||||
sf.rows++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sf *shardFile) trackKey(k row.Key) {
|
||||
if sf.firstKey.IsZero() {
|
||||
sf.firstKey = k
|
||||
}
|
||||
sf.lastKey = k
|
||||
}
|
||||
|
||||
// Checkpoint makes the current shard bytes and their metadata durable. It is
|
||||
// called before destructive database work when --purge is enabled.
|
||||
func (w *Writer) Checkpoint() error {
|
||||
if w.cur == nil {
|
||||
return nil
|
||||
}
|
||||
sf := w.cur
|
||||
if sf.rows > sf.memberRows {
|
||||
if err := sf.closeMember(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
meta := &PartialMeta{
|
||||
Seq: sf.seq,
|
||||
Bucket: sf.bucket,
|
||||
Rows: sf.rows,
|
||||
Bytes: sf.cw.n,
|
||||
FirstKey: sf.firstKey,
|
||||
LastKey: sf.lastKey,
|
||||
}
|
||||
if err := savePartialMeta(sf.dir, meta); err != nil {
|
||||
return fmt.Errorf("write partial metadata: %w", err)
|
||||
}
|
||||
if err := sf.openChain(w.opt.Compressor, 256*1024); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sf *shardFile) closeMember() error {
|
||||
if err := sf.bw.Flush(); err != nil {
|
||||
return fmt.Errorf("flush shard: %w", err)
|
||||
}
|
||||
if err := sf.compW.Close(); err != nil {
|
||||
return fmt.Errorf("close shard compressor: %w", err)
|
||||
}
|
||||
if err := sf.file.Sync(); err != nil {
|
||||
return fmt.Errorf("sync shard: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Flush pushes buffered rows to the OS without sealing the shard.
|
||||
func (w *Writer) Flush() error {
|
||||
if w.cur == nil || w.cur.bw == nil {
|
||||
return nil
|
||||
}
|
||||
return w.cur.bw.Flush()
|
||||
}
|
||||
|
||||
// SealCurrent closes and publishes the active shard even if it is not full.
|
||||
func (w *Writer) SealCurrent() error {
|
||||
if w.cur == nil {
|
||||
return nil
|
||||
}
|
||||
if w.cur.rows == 0 {
|
||||
return w.discardCurrent()
|
||||
}
|
||||
return w.seal()
|
||||
}
|
||||
|
||||
func (w *Writer) discardCurrent() error {
|
||||
sf := w.cur
|
||||
sf.file.Close()
|
||||
os.Remove(sf.partial)
|
||||
removePartialMeta(sf.dir, sf.seq)
|
||||
w.cur = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) seal() error {
|
||||
sf := w.cur
|
||||
if sf.rows == 0 {
|
||||
return w.discardCurrent()
|
||||
}
|
||||
if sf.rows > sf.memberRows || sf.compW != nil {
|
||||
if err := sf.closeMember(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := sf.file.Close(); err != nil {
|
||||
return fmt.Errorf("close partial shard: %w", err)
|
||||
}
|
||||
lo := KeyFilePart(sf.firstKey)
|
||||
hi := KeyFilePart(sf.lastKey)
|
||||
name := FinalName(sf.seq, lo, hi, w.opt.Format, w.opt.Compressor.Ext)
|
||||
finalPath := filepath.Join(sf.dir, name)
|
||||
if err := os.Remove(finalPath); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("remove stale shard: %w", err)
|
||||
}
|
||||
if err := atomicfile.Replace(sf.partial, finalPath); err != nil {
|
||||
return fmt.Errorf("publish shard: %w", err)
|
||||
}
|
||||
sum, size, err := hashFile(finalPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash shard: %w", err)
|
||||
}
|
||||
rel, err := filepath.Rel(w.opt.Dir, finalPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entry := Entry{
|
||||
Seq: sf.seq,
|
||||
Path: filepath.ToSlash(rel),
|
||||
Date: sf.bucket,
|
||||
PKLo: lo,
|
||||
PKHi: hi,
|
||||
Rows: sf.rows,
|
||||
Bytes: size,
|
||||
SHA256: sum,
|
||||
}
|
||||
if err := w.man.Append(entry); err != nil {
|
||||
return err
|
||||
}
|
||||
removePartialMeta(sf.dir, sf.seq)
|
||||
w.shards++
|
||||
w.bytes += size
|
||||
w.cur = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close publishes any in-progress shard and closes the manifest.
|
||||
func (w *Writer) Close() error {
|
||||
if err := w.SealCurrent(); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.man.Close()
|
||||
}
|
||||
|
||||
// StatePosition reports the bucket and the last sealed seq within that bucket,
|
||||
// which is what a checkpoint must record for resume.
|
||||
func (w *Writer) StatePosition() (bucket string, sealedSeq int64) {
|
||||
if w.cur != nil {
|
||||
return w.cur.bucket, w.cur.seq - 1
|
||||
}
|
||||
return w.curBucket, w.seq
|
||||
}
|
||||
|
||||
// Abandon closes file handles without sealing the active shard, leaving the
|
||||
// partial and its metadata for a later resume.
|
||||
func (w *Writer) Abandon() error {
|
||||
if w.cur != nil {
|
||||
if w.cur.bw != nil {
|
||||
_ = w.cur.bw.Flush()
|
||||
}
|
||||
_ = w.cur.file.Close()
|
||||
w.cur = nil
|
||||
}
|
||||
return w.man.Close()
|
||||
}
|
||||
|
||||
// Stats returns rows written, shards sealed and bytes sealed.
|
||||
func (w *Writer) Stats() (rows, shards, bytes int64) {
|
||||
return w.totalRows, w.shards, w.bytes
|
||||
}
|
||||
|
||||
// CurrentShard returns the sequence number of the active shard, or the last
|
||||
// one when no shard is currently open.
|
||||
func (w *Writer) CurrentShard() int64 {
|
||||
if w.cur != nil {
|
||||
return w.cur.seq
|
||||
}
|
||||
return w.seq
|
||||
}
|
||||
|
||||
// WrittenBytes returns sealed bytes plus the bytes already written to the
|
||||
// active partial shard.
|
||||
func (w *Writer) WrittenBytes() int64 {
|
||||
n := w.bytes
|
||||
if w.cur != nil && w.cur.cw != nil {
|
||||
n += w.cur.cw.n
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,459 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
func testOpts(dir, mode string) Options {
|
||||
return Options{
|
||||
Dir: dir,
|
||||
Mode: mode,
|
||||
Format: "dump",
|
||||
Compressor: &Compressor{Name: "none"},
|
||||
Columns: []row.Column{{Name: "id", Type: "int"}, {Name: "v", Type: "varchar"}},
|
||||
}
|
||||
}
|
||||
|
||||
func chunk(start int, values ...string) *row.Chunk {
|
||||
c := &row.Chunk{}
|
||||
for i, v := range values {
|
||||
id := start + i
|
||||
c.Rows = append(c.Rows, row.Row{Values: []row.Value{
|
||||
{Raw: []byte(itoa(id))},
|
||||
{Raw: []byte(v)},
|
||||
}})
|
||||
c.Keys = append(c.Keys, row.KeyOf(itoa(id)))
|
||||
}
|
||||
if len(c.Keys) > 0 {
|
||||
c.FirstKey = c.Keys[0]
|
||||
c.LastKey = c.Keys[len(c.Keys)-1]
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func itoa(i int) string {
|
||||
if i == 0 {
|
||||
return "0"
|
||||
}
|
||||
var b [20]byte
|
||||
p := len(b)
|
||||
neg := i < 0
|
||||
if neg {
|
||||
i = -i
|
||||
}
|
||||
for i > 0 {
|
||||
p--
|
||||
b[p] = byte('0' + i%10)
|
||||
i /= 10
|
||||
}
|
||||
if neg {
|
||||
p--
|
||||
b[p] = '-'
|
||||
}
|
||||
return string(b[p:])
|
||||
}
|
||||
|
||||
func TestCountShardsOrderAndNames(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, err := OpenManifest(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opt := testOpts(dir, "count")
|
||||
opt.ShardRows = 2
|
||||
w, err := Open(opt, man)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A chunk of 3 rows spans two shards.
|
||||
if err := w.WriteChunk(chunk(1, "a", "b", "c")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.WriteChunk(chunk(4, "d", "e")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
want := map[string]string{
|
||||
"000001.1-2.dump": "1\ta\n2\tb\n",
|
||||
"000002.3-4.dump": "3\tc\n4\td\n",
|
||||
"000003.5-5.dump": "5\te\n",
|
||||
}
|
||||
for name, content := range want {
|
||||
b, err := os.ReadFile(filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if string(b) != content {
|
||||
t.Fatalf("%s: got %q want %q", name, b, content)
|
||||
}
|
||||
}
|
||||
entries := man.Entries()
|
||||
if len(entries) != 3 {
|
||||
t.Fatalf("manifest entries: %d", len(entries))
|
||||
}
|
||||
if entries[0].PKLo != "1" || entries[0].PKHi != "2" || entries[0].Rows != 2 {
|
||||
t.Fatalf("entry0 %+v", entries[0])
|
||||
}
|
||||
if entries[2].Path != "000003.5-5.dump" || entries[2].SHA256 == "" {
|
||||
t.Fatalf("entry2 %+v", entries[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDateShardsDirectories(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
opt := testOpts(dir, "date")
|
||||
w, _ := Open(opt, man)
|
||||
c := chunk(1, "a", "b", "c")
|
||||
c.Buckets = []string{"2025-06-01", "2025-06-01", "2025-06-02"}
|
||||
if err := w.WriteChunk(c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "date=2025-06-01", "000001.1-2.dump")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "date=2025-06-02", "000001.3-3.dump")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDateOrderViolation(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
w, _ := Open(testOpts(dir, "date"), man)
|
||||
c := chunk(1, "a", "b")
|
||||
c.Buckets = []string{"2025-06-02", "2025-06-01"}
|
||||
if err := w.WriteChunk(c); err == nil {
|
||||
t.Fatal("expected ordering error")
|
||||
}
|
||||
_ = w.Abandon()
|
||||
}
|
||||
|
||||
func TestResumeAppendsAfterTruncatingGarbage(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
opt := testOpts(dir, "count")
|
||||
opt.ShardRows = 10
|
||||
w, _ := Open(opt, man)
|
||||
if err := w.WriteChunk(chunk(1, "a", "b")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Checkpoint(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := LoadPartialMeta(dir, "", 1)
|
||||
if err != nil || meta == nil {
|
||||
t.Fatalf("meta %v %v", meta, err)
|
||||
}
|
||||
if err := w.Abandon(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Simulate bytes from an uncommitted chunk that were flushed but never
|
||||
// checkpointed: resume must truncate them.
|
||||
partial := PartialPath(dir, "", 1)
|
||||
f, err := os.OpenFile(partial, os.O_APPEND|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.WriteString("garbage\n")
|
||||
f.Close()
|
||||
|
||||
man2, _ := OpenManifest(dir)
|
||||
w2, err := Open(opt, man2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w2.Resume(meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w2.WriteChunk(chunk(3, "c", "d")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w2.Checkpoint(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w2.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(dir, "000001.1-4.dump"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(b) != "1\ta\n2\tb\n3\tc\n4\td\n" {
|
||||
t.Fatalf("got %q", b)
|
||||
}
|
||||
if _, err := os.Stat(partial); !os.IsNotExist(err) {
|
||||
t.Fatal("partial should be gone after seal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestAdoptOrphan(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
orphan := filepath.Join(dir, "000007.9-12.dump")
|
||||
if err := os.WriteFile(orphan, []byte("9\tx\n10\ty\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
comp := &Compressor{Name: "none"}
|
||||
maxSeq, err := man.Adopt(dir, "dump", comp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if maxSeq[""] != 7 {
|
||||
t.Fatalf("maxSeq %+v", maxSeq)
|
||||
}
|
||||
if len(man.Entries()) != 1 || man.Entries()[0].Rows != 2 {
|
||||
t.Fatalf("entries %+v", man.Entries())
|
||||
}
|
||||
// A second adopt must not duplicate the entry.
|
||||
if _, err := man.Adopt(dir, "dump", comp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(man.Entries()) != 1 {
|
||||
t.Fatalf("entries %+v", man.Entries())
|
||||
}
|
||||
man.Close()
|
||||
}
|
||||
|
||||
func TestSafeNamePart(t *testing.T) {
|
||||
got := safeNamePart("2024-01-01 00:00:00")
|
||||
if strings.ContainsAny(got, `:/\*?"<>|`) {
|
||||
t.Fatalf("unsafe characters kept: %q", got)
|
||||
}
|
||||
if got != "2024-01-01_00_00_00" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if safeNamePart("") != "_" {
|
||||
t.Fatal("empty name part")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressedShardIsIndependent(t *testing.T) {
|
||||
for _, compName := range []string{"gzip", "zstd", "xz"} {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
comp, err := NewCompressor(compName, DefaultLevel)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opt := testOpts(dir, "none")
|
||||
opt.Compressor = comp
|
||||
w, _ := Open(opt, man)
|
||||
if err := w.WriteChunk(chunk(1, "a", "b")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var path string
|
||||
entries := man.Entries()
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("%s: entries %d", compName, len(entries))
|
||||
}
|
||||
path = filepath.Join(dir, entries[0].Path)
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := comp.Reader(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var b strings.Builder
|
||||
buf := make([]byte, 1024)
|
||||
for {
|
||||
n, err := r.Read(buf)
|
||||
b.Write(buf[:n])
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
r.Close()
|
||||
f.Close()
|
||||
if b.String() != "1\ta\n2\tb\n" {
|
||||
t.Fatalf("%s: got %q", compName, b.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressionLevelMapping(t *testing.T) {
|
||||
xzDictMiB := []int{1, 2, 4, 4, 8, 8, 16, 32, 64}
|
||||
for level := 1; level <= 9; level++ {
|
||||
c, err := NewCompressor("gzip", level)
|
||||
if err != nil {
|
||||
t.Fatalf("level %d: %v", level, err)
|
||||
}
|
||||
if c.Level != level || c.Name != "gzip" || c.Ext != ".gz" {
|
||||
t.Fatalf("level %d: %+v", level, c)
|
||||
}
|
||||
if got := gzipLevel(level); got != level {
|
||||
t.Fatalf("gzip level %d -> %d", level, got)
|
||||
}
|
||||
|
||||
var wantZstd zstd.EncoderLevel
|
||||
switch {
|
||||
case level <= 2:
|
||||
wantZstd = zstd.SpeedFastest
|
||||
case level <= 5:
|
||||
wantZstd = zstd.SpeedDefault
|
||||
case level <= 8:
|
||||
wantZstd = zstd.SpeedBetterCompression
|
||||
default:
|
||||
wantZstd = zstd.SpeedBestCompression
|
||||
}
|
||||
if got := zstdLevel(level); got != wantZstd {
|
||||
t.Fatalf("zstd level %d -> %v want %v", level, got, wantZstd)
|
||||
}
|
||||
if got, want := xzDictCap(level), xzDictMiB[level-1]<<20; got != want {
|
||||
t.Fatalf("xz level %d -> %d want %d", level, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range []string{"none", "gzip", "zstd", "xz"} {
|
||||
for _, level := range []int{-1, 0, 10} {
|
||||
if _, err := NewCompressor(name, level); err == nil {
|
||||
t.Fatalf("%s level %d: expected error", name, level)
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := NewCompressor("brotli", DefaultLevel); err == nil {
|
||||
t.Fatal("unsupported compressor must fail")
|
||||
}
|
||||
|
||||
for name, ext := range map[string]string{"none": "", "gzip": ".gz", "zstd": ".zst", "xz": ".xz"} {
|
||||
c, err := NewCompressor(name, DefaultLevel)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if c.Ext != ext {
|
||||
t.Fatalf("%s: ext %q want %q", name, c.Ext, ext)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestXZCheckpointKeepsAllStreams(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
defer man.Close()
|
||||
comp, err := NewCompressor("xz", 9)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opt := testOpts(dir, "none")
|
||||
opt.Compressor = comp
|
||||
w, _ := Open(opt, man)
|
||||
if err := w.WriteChunk(chunk(1, "a", "b")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Checkpoint(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.WriteChunk(chunk(3, "c", "d")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
entries := man.Entries()
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("entries %d", len(entries))
|
||||
}
|
||||
if entries[0].Rows != 4 {
|
||||
t.Fatalf("rows %d", entries[0].Rows)
|
||||
}
|
||||
if !strings.HasSuffix(entries[0].Path, ".dump.xz") {
|
||||
t.Fatalf("path %q", entries[0].Path)
|
||||
}
|
||||
|
||||
f, err := os.Open(filepath.Join(dir, entries[0].Path))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
r, err := comp.Reader(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
got, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != "1\ta\n2\tb\n3\tc\n4\td\n" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestAdoptXZOrphan(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
defer man.Close()
|
||||
comp, err := NewCompressor("xz", DefaultLevel)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
xw, err := comp.Wrap(&buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := xw.Write([]byte("9\tx\n10\ty\n")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := xw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
orphan := filepath.Join(dir, "000007.9-12.dump.xz")
|
||||
if err := os.WriteFile(orphan, buf.Bytes(), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
maxSeq, err := man.Adopt(dir, "dump", comp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if maxSeq[""] != 7 {
|
||||
t.Fatalf("maxSeq %+v", maxSeq)
|
||||
}
|
||||
entries := man.Entries()
|
||||
if len(entries) != 1 || entries[0].Seq != 7 || entries[0].Rows != 2 {
|
||||
t.Fatalf("entries %+v", entries)
|
||||
}
|
||||
if entries[0].Path != "000007.9-12.dump.xz" {
|
||||
t.Fatalf("path %q", entries[0].Path)
|
||||
}
|
||||
|
||||
// A resumed writer must continue after the adopted seq, not overwrite it.
|
||||
opt := testOpts(dir, "none")
|
||||
opt.Compressor = comp
|
||||
opt.StartSeq = maxSeq[""]
|
||||
w, _ := Open(opt, man)
|
||||
if err := w.WriteChunk(chunk(1, "a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "000008.1-1.dump.xz")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(orphan); err != nil {
|
||||
t.Fatalf("adopted shard was overwritten: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user