Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,132 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
// DefaultLevel is the compression level used when none is specified. It
|
||||
// matches the byte output of archer before --compress-level existed.
|
||||
const DefaultLevel = 3
|
||||
|
||||
// xzReadDictCap is the largest dictionary xz can produce here (level 9).
|
||||
// Readers always allow it so lowering the level with --force-resume does not
|
||||
// make previously written shards unreadable.
|
||||
const xzReadDictCap = 64 << 20
|
||||
|
||||
// Compressor wraps shard byte streams. Every shard is compressed on its own
|
||||
// so a single file can be decompressed and imported without its siblings.
|
||||
// Level is a format-neutral 1..9 scale where 9 is the strongest setting.
|
||||
type Compressor struct {
|
||||
Name string // "none", "gzip", "zstd", "xz"
|
||||
Ext string // "", ".gz", ".zst", ".xz"
|
||||
Level int
|
||||
}
|
||||
|
||||
func NewCompressor(name string, level int) (*Compressor, error) {
|
||||
if level < 1 || level > 9 {
|
||||
return nil, fmt.Errorf("compression level %d out of range 1-9", level)
|
||||
}
|
||||
switch name {
|
||||
case "", "none":
|
||||
return &Compressor{Name: "none", Level: level}, nil
|
||||
case "gzip":
|
||||
return &Compressor{Name: "gzip", Ext: ".gz", Level: level}, nil
|
||||
case "zstd":
|
||||
return &Compressor{Name: "zstd", Ext: ".zst", Level: level}, nil
|
||||
case "xz":
|
||||
return &Compressor{Name: "xz", Ext: ".xz", Level: level}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported compression %q", name)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Compressor) Wrap(w io.Writer) (io.WriteCloser, error) {
|
||||
switch c.Name {
|
||||
case "gzip":
|
||||
return gzip.NewWriterLevel(w, gzipLevel(c.Level))
|
||||
case "zstd":
|
||||
return zstd.NewWriter(w, zstd.WithEncoderLevel(zstdLevel(c.Level)))
|
||||
case "xz":
|
||||
xw, err := xz.WriterConfig{DictCap: xzDictCap(c.Level)}.NewWriter(w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return xw, nil
|
||||
default:
|
||||
return nopCloser{w}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Reader returns a decompressing reader for an existing shard.
|
||||
func (c *Compressor) Reader(r io.Reader) (io.ReadCloser, error) {
|
||||
switch c.Name {
|
||||
case "gzip":
|
||||
return gzip.NewReader(r)
|
||||
case "zstd":
|
||||
zr, err := zstd.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return zr.IOReadCloser(), nil
|
||||
case "xz":
|
||||
xr, err := xz.ReaderConfig{DictCap: xzReadDictCap}.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.NopCloser(xr), nil
|
||||
default:
|
||||
return io.NopCloser(r), nil
|
||||
}
|
||||
}
|
||||
|
||||
// gzipLevel passes the neutral level straight through.
|
||||
func gzipLevel(level int) int {
|
||||
return level
|
||||
}
|
||||
|
||||
func zstdLevel(level int) zstd.EncoderLevel {
|
||||
switch {
|
||||
case level <= 2:
|
||||
return zstd.SpeedFastest
|
||||
case level <= 5:
|
||||
return zstd.SpeedDefault
|
||||
case level <= 8:
|
||||
return zstd.SpeedBetterCompression
|
||||
default:
|
||||
return zstd.SpeedBestCompression
|
||||
}
|
||||
}
|
||||
|
||||
// xzDictCap maps the neutral level onto xz dictionary sizes; the library has
|
||||
// no preset parameter, so the dictionary is the only strength knob.
|
||||
func xzDictCap(level int) int {
|
||||
switch level {
|
||||
case 1:
|
||||
return 1 << 20
|
||||
case 2:
|
||||
return 2 << 20
|
||||
case 3, 4:
|
||||
return 4 << 20
|
||||
case 5, 6:
|
||||
return 8 << 20
|
||||
case 7:
|
||||
return 16 << 20
|
||||
case 8:
|
||||
return 32 << 20
|
||||
default:
|
||||
return 64 << 20
|
||||
}
|
||||
}
|
||||
|
||||
type nopCloser struct{ io.Writer }
|
||||
|
||||
func (nopCloser) Close() error { return nil }
|
||||
|
||||
type flushCloser interface {
|
||||
Flush() error
|
||||
}
|
||||
Reference in new issue
Block a user