Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,459 @@
|
||||
package shard
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
func testOpts(dir, mode string) Options {
|
||||
return Options{
|
||||
Dir: dir,
|
||||
Mode: mode,
|
||||
Format: "dump",
|
||||
Compressor: &Compressor{Name: "none"},
|
||||
Columns: []row.Column{{Name: "id", Type: "int"}, {Name: "v", Type: "varchar"}},
|
||||
}
|
||||
}
|
||||
|
||||
func chunk(start int, values ...string) *row.Chunk {
|
||||
c := &row.Chunk{}
|
||||
for i, v := range values {
|
||||
id := start + i
|
||||
c.Rows = append(c.Rows, row.Row{Values: []row.Value{
|
||||
{Raw: []byte(itoa(id))},
|
||||
{Raw: []byte(v)},
|
||||
}})
|
||||
c.Keys = append(c.Keys, row.KeyOf(itoa(id)))
|
||||
}
|
||||
if len(c.Keys) > 0 {
|
||||
c.FirstKey = c.Keys[0]
|
||||
c.LastKey = c.Keys[len(c.Keys)-1]
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func itoa(i int) string {
|
||||
if i == 0 {
|
||||
return "0"
|
||||
}
|
||||
var b [20]byte
|
||||
p := len(b)
|
||||
neg := i < 0
|
||||
if neg {
|
||||
i = -i
|
||||
}
|
||||
for i > 0 {
|
||||
p--
|
||||
b[p] = byte('0' + i%10)
|
||||
i /= 10
|
||||
}
|
||||
if neg {
|
||||
p--
|
||||
b[p] = '-'
|
||||
}
|
||||
return string(b[p:])
|
||||
}
|
||||
|
||||
func TestCountShardsOrderAndNames(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, err := OpenManifest(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opt := testOpts(dir, "count")
|
||||
opt.ShardRows = 2
|
||||
w, err := Open(opt, man)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A chunk of 3 rows spans two shards.
|
||||
if err := w.WriteChunk(chunk(1, "a", "b", "c")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.WriteChunk(chunk(4, "d", "e")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
want := map[string]string{
|
||||
"000001.1-2.dump": "1\ta\n2\tb\n",
|
||||
"000002.3-4.dump": "3\tc\n4\td\n",
|
||||
"000003.5-5.dump": "5\te\n",
|
||||
}
|
||||
for name, content := range want {
|
||||
b, err := os.ReadFile(filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if string(b) != content {
|
||||
t.Fatalf("%s: got %q want %q", name, b, content)
|
||||
}
|
||||
}
|
||||
entries := man.Entries()
|
||||
if len(entries) != 3 {
|
||||
t.Fatalf("manifest entries: %d", len(entries))
|
||||
}
|
||||
if entries[0].PKLo != "1" || entries[0].PKHi != "2" || entries[0].Rows != 2 {
|
||||
t.Fatalf("entry0 %+v", entries[0])
|
||||
}
|
||||
if entries[2].Path != "000003.5-5.dump" || entries[2].SHA256 == "" {
|
||||
t.Fatalf("entry2 %+v", entries[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDateShardsDirectories(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
opt := testOpts(dir, "date")
|
||||
w, _ := Open(opt, man)
|
||||
c := chunk(1, "a", "b", "c")
|
||||
c.Buckets = []string{"2025-06-01", "2025-06-01", "2025-06-02"}
|
||||
if err := w.WriteChunk(c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "date=2025-06-01", "000001.1-2.dump")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "date=2025-06-02", "000001.3-3.dump")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDateOrderViolation(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
w, _ := Open(testOpts(dir, "date"), man)
|
||||
c := chunk(1, "a", "b")
|
||||
c.Buckets = []string{"2025-06-02", "2025-06-01"}
|
||||
if err := w.WriteChunk(c); err == nil {
|
||||
t.Fatal("expected ordering error")
|
||||
}
|
||||
_ = w.Abandon()
|
||||
}
|
||||
|
||||
func TestResumeAppendsAfterTruncatingGarbage(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
opt := testOpts(dir, "count")
|
||||
opt.ShardRows = 10
|
||||
w, _ := Open(opt, man)
|
||||
if err := w.WriteChunk(chunk(1, "a", "b")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Checkpoint(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := LoadPartialMeta(dir, "", 1)
|
||||
if err != nil || meta == nil {
|
||||
t.Fatalf("meta %v %v", meta, err)
|
||||
}
|
||||
if err := w.Abandon(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Simulate bytes from an uncommitted chunk that were flushed but never
|
||||
// checkpointed: resume must truncate them.
|
||||
partial := PartialPath(dir, "", 1)
|
||||
f, err := os.OpenFile(partial, os.O_APPEND|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.WriteString("garbage\n")
|
||||
f.Close()
|
||||
|
||||
man2, _ := OpenManifest(dir)
|
||||
w2, err := Open(opt, man2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w2.Resume(meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w2.WriteChunk(chunk(3, "c", "d")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w2.Checkpoint(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w2.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(dir, "000001.1-4.dump"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(b) != "1\ta\n2\tb\n3\tc\n4\td\n" {
|
||||
t.Fatalf("got %q", b)
|
||||
}
|
||||
if _, err := os.Stat(partial); !os.IsNotExist(err) {
|
||||
t.Fatal("partial should be gone after seal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestAdoptOrphan(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
orphan := filepath.Join(dir, "000007.9-12.dump")
|
||||
if err := os.WriteFile(orphan, []byte("9\tx\n10\ty\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
comp := &Compressor{Name: "none"}
|
||||
maxSeq, err := man.Adopt(dir, "dump", comp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if maxSeq[""] != 7 {
|
||||
t.Fatalf("maxSeq %+v", maxSeq)
|
||||
}
|
||||
if len(man.Entries()) != 1 || man.Entries()[0].Rows != 2 {
|
||||
t.Fatalf("entries %+v", man.Entries())
|
||||
}
|
||||
// A second adopt must not duplicate the entry.
|
||||
if _, err := man.Adopt(dir, "dump", comp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(man.Entries()) != 1 {
|
||||
t.Fatalf("entries %+v", man.Entries())
|
||||
}
|
||||
man.Close()
|
||||
}
|
||||
|
||||
func TestSafeNamePart(t *testing.T) {
|
||||
got := safeNamePart("2024-01-01 00:00:00")
|
||||
if strings.ContainsAny(got, `:/\*?"<>|`) {
|
||||
t.Fatalf("unsafe characters kept: %q", got)
|
||||
}
|
||||
if got != "2024-01-01_00_00_00" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if safeNamePart("") != "_" {
|
||||
t.Fatal("empty name part")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressedShardIsIndependent(t *testing.T) {
|
||||
for _, compName := range []string{"gzip", "zstd", "xz"} {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
comp, err := NewCompressor(compName, DefaultLevel)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opt := testOpts(dir, "none")
|
||||
opt.Compressor = comp
|
||||
w, _ := Open(opt, man)
|
||||
if err := w.WriteChunk(chunk(1, "a", "b")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var path string
|
||||
entries := man.Entries()
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("%s: entries %d", compName, len(entries))
|
||||
}
|
||||
path = filepath.Join(dir, entries[0].Path)
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := comp.Reader(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var b strings.Builder
|
||||
buf := make([]byte, 1024)
|
||||
for {
|
||||
n, err := r.Read(buf)
|
||||
b.Write(buf[:n])
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
r.Close()
|
||||
f.Close()
|
||||
if b.String() != "1\ta\n2\tb\n" {
|
||||
t.Fatalf("%s: got %q", compName, b.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressionLevelMapping(t *testing.T) {
|
||||
xzDictMiB := []int{1, 2, 4, 4, 8, 8, 16, 32, 64}
|
||||
for level := 1; level <= 9; level++ {
|
||||
c, err := NewCompressor("gzip", level)
|
||||
if err != nil {
|
||||
t.Fatalf("level %d: %v", level, err)
|
||||
}
|
||||
if c.Level != level || c.Name != "gzip" || c.Ext != ".gz" {
|
||||
t.Fatalf("level %d: %+v", level, c)
|
||||
}
|
||||
if got := gzipLevel(level); got != level {
|
||||
t.Fatalf("gzip level %d -> %d", level, got)
|
||||
}
|
||||
|
||||
var wantZstd zstd.EncoderLevel
|
||||
switch {
|
||||
case level <= 2:
|
||||
wantZstd = zstd.SpeedFastest
|
||||
case level <= 5:
|
||||
wantZstd = zstd.SpeedDefault
|
||||
case level <= 8:
|
||||
wantZstd = zstd.SpeedBetterCompression
|
||||
default:
|
||||
wantZstd = zstd.SpeedBestCompression
|
||||
}
|
||||
if got := zstdLevel(level); got != wantZstd {
|
||||
t.Fatalf("zstd level %d -> %v want %v", level, got, wantZstd)
|
||||
}
|
||||
if got, want := xzDictCap(level), xzDictMiB[level-1]<<20; got != want {
|
||||
t.Fatalf("xz level %d -> %d want %d", level, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range []string{"none", "gzip", "zstd", "xz"} {
|
||||
for _, level := range []int{-1, 0, 10} {
|
||||
if _, err := NewCompressor(name, level); err == nil {
|
||||
t.Fatalf("%s level %d: expected error", name, level)
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := NewCompressor("brotli", DefaultLevel); err == nil {
|
||||
t.Fatal("unsupported compressor must fail")
|
||||
}
|
||||
|
||||
for name, ext := range map[string]string{"none": "", "gzip": ".gz", "zstd": ".zst", "xz": ".xz"} {
|
||||
c, err := NewCompressor(name, DefaultLevel)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if c.Ext != ext {
|
||||
t.Fatalf("%s: ext %q want %q", name, c.Ext, ext)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestXZCheckpointKeepsAllStreams(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
defer man.Close()
|
||||
comp, err := NewCompressor("xz", 9)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opt := testOpts(dir, "none")
|
||||
opt.Compressor = comp
|
||||
w, _ := Open(opt, man)
|
||||
if err := w.WriteChunk(chunk(1, "a", "b")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Checkpoint(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.WriteChunk(chunk(3, "c", "d")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
entries := man.Entries()
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("entries %d", len(entries))
|
||||
}
|
||||
if entries[0].Rows != 4 {
|
||||
t.Fatalf("rows %d", entries[0].Rows)
|
||||
}
|
||||
if !strings.HasSuffix(entries[0].Path, ".dump.xz") {
|
||||
t.Fatalf("path %q", entries[0].Path)
|
||||
}
|
||||
|
||||
f, err := os.Open(filepath.Join(dir, entries[0].Path))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
r, err := comp.Reader(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
got, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != "1\ta\n2\tb\n3\tc\n4\td\n" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestAdoptXZOrphan(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
man, _ := OpenManifest(dir)
|
||||
defer man.Close()
|
||||
comp, err := NewCompressor("xz", DefaultLevel)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
xw, err := comp.Wrap(&buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := xw.Write([]byte("9\tx\n10\ty\n")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := xw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
orphan := filepath.Join(dir, "000007.9-12.dump.xz")
|
||||
if err := os.WriteFile(orphan, buf.Bytes(), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
maxSeq, err := man.Adopt(dir, "dump", comp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if maxSeq[""] != 7 {
|
||||
t.Fatalf("maxSeq %+v", maxSeq)
|
||||
}
|
||||
entries := man.Entries()
|
||||
if len(entries) != 1 || entries[0].Seq != 7 || entries[0].Rows != 2 {
|
||||
t.Fatalf("entries %+v", entries)
|
||||
}
|
||||
if entries[0].Path != "000007.9-12.dump.xz" {
|
||||
t.Fatalf("path %q", entries[0].Path)
|
||||
}
|
||||
|
||||
// A resumed writer must continue after the adopted seq, not overwrite it.
|
||||
opt := testOpts(dir, "none")
|
||||
opt.Compressor = comp
|
||||
opt.StartSeq = maxSeq[""]
|
||||
w, _ := Open(opt, man)
|
||||
if err := w.WriteChunk(chunk(1, "a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "000008.1-1.dump.xz")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(orphan); err != nil {
|
||||
t.Fatalf("adopted shard was overwritten: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user