package shard import ( "compress/gzip" "fmt" "io" "github.com/klauspost/compress/zstd" "github.com/ulikunitz/xz" ) // DefaultLevel is the compression level used when none is specified. It // matches the byte output of archer before --compress-level existed. const DefaultLevel = 3 // xzReadDictCap is the largest dictionary xz can produce here (level 9). // Readers always allow it so lowering the level with --force-resume does not // make previously written shards unreadable. const xzReadDictCap = 64 << 20 // Compressor wraps shard byte streams. Every shard is compressed on its own // so a single file can be decompressed and imported without its siblings. // Level is a format-neutral 1..9 scale where 9 is the strongest setting. type Compressor struct { Name string // "none", "gzip", "zstd", "xz" Ext string // "", ".gz", ".zst", ".xz" Level int } func NewCompressor(name string, level int) (*Compressor, error) { if level < 1 || level > 9 { return nil, fmt.Errorf("compression level %d out of range 1-9", level) } switch name { case "", "none": return &Compressor{Name: "none", Level: level}, nil case "gzip": return &Compressor{Name: "gzip", Ext: ".gz", Level: level}, nil case "zstd": return &Compressor{Name: "zstd", Ext: ".zst", Level: level}, nil case "xz": return &Compressor{Name: "xz", Ext: ".xz", Level: level}, nil default: return nil, fmt.Errorf("unsupported compression %q", name) } } func (c *Compressor) Wrap(w io.Writer) (io.WriteCloser, error) { switch c.Name { case "gzip": return gzip.NewWriterLevel(w, gzipLevel(c.Level)) case "zstd": return zstd.NewWriter(w, zstd.WithEncoderLevel(zstdLevel(c.Level))) case "xz": xw, err := xz.WriterConfig{DictCap: xzDictCap(c.Level)}.NewWriter(w) if err != nil { return nil, err } return xw, nil default: return nopCloser{w}, nil } } // Reader returns a decompressing reader for an existing shard. func (c *Compressor) Reader(r io.Reader) (io.ReadCloser, error) { switch c.Name { case "gzip": return gzip.NewReader(r) case "zstd": zr, err := zstd.NewReader(r) if err != nil { return nil, err } return zr.IOReadCloser(), nil case "xz": xr, err := xz.ReaderConfig{DictCap: xzReadDictCap}.NewReader(r) if err != nil { return nil, err } return io.NopCloser(xr), nil default: return io.NopCloser(r), nil } } // gzipLevel passes the neutral level straight through. func gzipLevel(level int) int { return level } func zstdLevel(level int) zstd.EncoderLevel { switch { case level <= 2: return zstd.SpeedFastest case level <= 5: return zstd.SpeedDefault case level <= 8: return zstd.SpeedBetterCompression default: return zstd.SpeedBestCompression } } // xzDictCap maps the neutral level onto xz dictionary sizes; the library has // no preset parameter, so the dictionary is the only strength knob. func xzDictCap(level int) int { switch level { case 1: return 1 << 20 case 2: return 2 << 20 case 3, 4: return 4 << 20 case 5, 6: return 8 << 20 case 7: return 16 << 20 case 8: return 32 << 20 default: return 64 << 20 } } type nopCloser struct{ io.Writer } func (nopCloser) Close() error { return nil } type flushCloser interface { Flush() error }