Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
133 lines
3.2 KiB
Go
133 lines
3.2 KiB
Go
package shard
|
|
|
|
import (
|
|
"compress/gzip"
|
|
"fmt"
|
|
"io"
|
|
|
|
"github.com/klauspost/compress/zstd"
|
|
"github.com/ulikunitz/xz"
|
|
)
|
|
|
|
// DefaultLevel is the compression level used when none is specified. It
|
|
// matches the byte output of archer before --compress-level existed.
|
|
const DefaultLevel = 3
|
|
|
|
// xzReadDictCap is the largest dictionary xz can produce here (level 9).
|
|
// Readers always allow it so lowering the level with --force-resume does not
|
|
// make previously written shards unreadable.
|
|
const xzReadDictCap = 64 << 20
|
|
|
|
// Compressor wraps shard byte streams. Every shard is compressed on its own
|
|
// so a single file can be decompressed and imported without its siblings.
|
|
// Level is a format-neutral 1..9 scale where 9 is the strongest setting.
|
|
type Compressor struct {
|
|
Name string // "none", "gzip", "zstd", "xz"
|
|
Ext string // "", ".gz", ".zst", ".xz"
|
|
Level int
|
|
}
|
|
|
|
func NewCompressor(name string, level int) (*Compressor, error) {
|
|
if level < 1 || level > 9 {
|
|
return nil, fmt.Errorf("compression level %d out of range 1-9", level)
|
|
}
|
|
switch name {
|
|
case "", "none":
|
|
return &Compressor{Name: "none", Level: level}, nil
|
|
case "gzip":
|
|
return &Compressor{Name: "gzip", Ext: ".gz", Level: level}, nil
|
|
case "zstd":
|
|
return &Compressor{Name: "zstd", Ext: ".zst", Level: level}, nil
|
|
case "xz":
|
|
return &Compressor{Name: "xz", Ext: ".xz", Level: level}, nil
|
|
default:
|
|
return nil, fmt.Errorf("unsupported compression %q", name)
|
|
}
|
|
}
|
|
|
|
func (c *Compressor) Wrap(w io.Writer) (io.WriteCloser, error) {
|
|
switch c.Name {
|
|
case "gzip":
|
|
return gzip.NewWriterLevel(w, gzipLevel(c.Level))
|
|
case "zstd":
|
|
return zstd.NewWriter(w, zstd.WithEncoderLevel(zstdLevel(c.Level)))
|
|
case "xz":
|
|
xw, err := xz.WriterConfig{DictCap: xzDictCap(c.Level)}.NewWriter(w)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return xw, nil
|
|
default:
|
|
return nopCloser{w}, nil
|
|
}
|
|
}
|
|
|
|
// Reader returns a decompressing reader for an existing shard.
|
|
func (c *Compressor) Reader(r io.Reader) (io.ReadCloser, error) {
|
|
switch c.Name {
|
|
case "gzip":
|
|
return gzip.NewReader(r)
|
|
case "zstd":
|
|
zr, err := zstd.NewReader(r)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return zr.IOReadCloser(), nil
|
|
case "xz":
|
|
xr, err := xz.ReaderConfig{DictCap: xzReadDictCap}.NewReader(r)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return io.NopCloser(xr), nil
|
|
default:
|
|
return io.NopCloser(r), nil
|
|
}
|
|
}
|
|
|
|
// gzipLevel passes the neutral level straight through.
|
|
func gzipLevel(level int) int {
|
|
return level
|
|
}
|
|
|
|
func zstdLevel(level int) zstd.EncoderLevel {
|
|
switch {
|
|
case level <= 2:
|
|
return zstd.SpeedFastest
|
|
case level <= 5:
|
|
return zstd.SpeedDefault
|
|
case level <= 8:
|
|
return zstd.SpeedBetterCompression
|
|
default:
|
|
return zstd.SpeedBestCompression
|
|
}
|
|
}
|
|
|
|
// xzDictCap maps the neutral level onto xz dictionary sizes; the library has
|
|
// no preset parameter, so the dictionary is the only strength knob.
|
|
func xzDictCap(level int) int {
|
|
switch level {
|
|
case 1:
|
|
return 1 << 20
|
|
case 2:
|
|
return 2 << 20
|
|
case 3, 4:
|
|
return 4 << 20
|
|
case 5, 6:
|
|
return 8 << 20
|
|
case 7:
|
|
return 16 << 20
|
|
case 8:
|
|
return 32 << 20
|
|
default:
|
|
return 64 << 20
|
|
}
|
|
}
|
|
|
|
type nopCloser struct{ io.Writer }
|
|
|
|
func (nopCloser) Close() error { return nil }
|
|
|
|
type flushCloser interface {
|
|
Flush() error
|
|
}
|