package shard import ( "bytes" "io" "os" "path/filepath" "strings" "testing" "github.com/klauspost/compress/zstd" "archer/internal/row" ) func testOpts(dir, mode string) Options { return Options{ Dir: dir, Mode: mode, Format: "dump", Compressor: &Compressor{Name: "none"}, Columns: []row.Column{{Name: "id", Type: "int"}, {Name: "v", Type: "varchar"}}, } } func chunk(start int, values ...string) *row.Chunk { c := &row.Chunk{} for i, v := range values { id := start + i c.Rows = append(c.Rows, row.Row{Values: []row.Value{ {Raw: []byte(itoa(id))}, {Raw: []byte(v)}, }}) c.Keys = append(c.Keys, row.KeyOf(itoa(id))) } if len(c.Keys) > 0 { c.FirstKey = c.Keys[0] c.LastKey = c.Keys[len(c.Keys)-1] } return c } func itoa(i int) string { if i == 0 { return "0" } var b [20]byte p := len(b) neg := i < 0 if neg { i = -i } for i > 0 { p-- b[p] = byte('0' + i%10) i /= 10 } if neg { p-- b[p] = '-' } return string(b[p:]) } func TestCountShardsOrderAndNames(t *testing.T) { dir := t.TempDir() man, err := OpenManifest(dir) if err != nil { t.Fatal(err) } opt := testOpts(dir, "count") opt.ShardRows = 2 w, err := Open(opt, man) if err != nil { t.Fatal(err) } // A chunk of 3 rows spans two shards. if err := w.WriteChunk(chunk(1, "a", "b", "c")); err != nil { t.Fatal(err) } if err := w.WriteChunk(chunk(4, "d", "e")); err != nil { t.Fatal(err) } if err := w.Close(); err != nil { t.Fatal(err) } want := map[string]string{ "000001.1-2.dump": "1\ta\n2\tb\n", "000002.3-4.dump": "3\tc\n4\td\n", "000003.5-5.dump": "5\te\n", } for name, content := range want { b, err := os.ReadFile(filepath.Join(dir, name)) if err != nil { t.Fatalf("%s: %v", name, err) } if string(b) != content { t.Fatalf("%s: got %q want %q", name, b, content) } } entries := man.Entries() if len(entries) != 3 { t.Fatalf("manifest entries: %d", len(entries)) } if entries[0].PKLo != "1" || entries[0].PKHi != "2" || entries[0].Rows != 2 { t.Fatalf("entry0 %+v", entries[0]) } if entries[2].Path != "000003.5-5.dump" || entries[2].SHA256 == "" { t.Fatalf("entry2 %+v", entries[2]) } } func TestDateShardsDirectories(t *testing.T) { dir := t.TempDir() man, _ := OpenManifest(dir) opt := testOpts(dir, "date") w, _ := Open(opt, man) c := chunk(1, "a", "b", "c") c.Buckets = []string{"2025-06-01", "2025-06-01", "2025-06-02"} if err := w.WriteChunk(c); err != nil { t.Fatal(err) } if err := w.Close(); err != nil { t.Fatal(err) } if _, err := os.Stat(filepath.Join(dir, "date=2025-06-01", "000001.1-2.dump")); err != nil { t.Fatal(err) } if _, err := os.Stat(filepath.Join(dir, "date=2025-06-02", "000001.3-3.dump")); err != nil { t.Fatal(err) } } func TestDateOrderViolation(t *testing.T) { dir := t.TempDir() man, _ := OpenManifest(dir) w, _ := Open(testOpts(dir, "date"), man) c := chunk(1, "a", "b") c.Buckets = []string{"2025-06-02", "2025-06-01"} if err := w.WriteChunk(c); err == nil { t.Fatal("expected ordering error") } _ = w.Abandon() } func TestResumeAppendsAfterTruncatingGarbage(t *testing.T) { dir := t.TempDir() man, _ := OpenManifest(dir) opt := testOpts(dir, "count") opt.ShardRows = 10 w, _ := Open(opt, man) if err := w.WriteChunk(chunk(1, "a", "b")); err != nil { t.Fatal(err) } if err := w.Checkpoint(); err != nil { t.Fatal(err) } meta, err := LoadPartialMeta(dir, "", 1) if err != nil || meta == nil { t.Fatalf("meta %v %v", meta, err) } if err := w.Abandon(); err != nil { t.Fatal(err) } // Simulate bytes from an uncommitted chunk that were flushed but never // checkpointed: resume must truncate them. partial := PartialPath(dir, "", 1) f, err := os.OpenFile(partial, os.O_APPEND|os.O_WRONLY, 0o644) if err != nil { t.Fatal(err) } f.WriteString("garbage\n") f.Close() man2, _ := OpenManifest(dir) w2, err := Open(opt, man2) if err != nil { t.Fatal(err) } if err := w2.Resume(meta); err != nil { t.Fatal(err) } if err := w2.WriteChunk(chunk(3, "c", "d")); err != nil { t.Fatal(err) } if err := w2.Checkpoint(); err != nil { t.Fatal(err) } if err := w2.Close(); err != nil { t.Fatal(err) } b, err := os.ReadFile(filepath.Join(dir, "000001.1-4.dump")) if err != nil { t.Fatal(err) } if string(b) != "1\ta\n2\tb\n3\tc\n4\td\n" { t.Fatalf("got %q", b) } if _, err := os.Stat(partial); !os.IsNotExist(err) { t.Fatal("partial should be gone after seal") } } func TestManifestAdoptOrphan(t *testing.T) { dir := t.TempDir() man, _ := OpenManifest(dir) orphan := filepath.Join(dir, "000007.9-12.dump") if err := os.WriteFile(orphan, []byte("9\tx\n10\ty\n"), 0o644); err != nil { t.Fatal(err) } comp := &Compressor{Name: "none"} maxSeq, err := man.Adopt(dir, "dump", comp) if err != nil { t.Fatal(err) } if maxSeq[""] != 7 { t.Fatalf("maxSeq %+v", maxSeq) } if len(man.Entries()) != 1 || man.Entries()[0].Rows != 2 { t.Fatalf("entries %+v", man.Entries()) } // A second adopt must not duplicate the entry. if _, err := man.Adopt(dir, "dump", comp); err != nil { t.Fatal(err) } if len(man.Entries()) != 1 { t.Fatalf("entries %+v", man.Entries()) } man.Close() } func TestSafeNamePart(t *testing.T) { got := safeNamePart("2024-01-01 00:00:00") if strings.ContainsAny(got, `:/\*?"<>|`) { t.Fatalf("unsafe characters kept: %q", got) } if got != "2024-01-01_00_00_00" { t.Fatalf("got %q", got) } if safeNamePart("") != "_" { t.Fatal("empty name part") } } func TestCompressedShardIsIndependent(t *testing.T) { for _, compName := range []string{"gzip", "zstd", "xz"} { dir := t.TempDir() man, _ := OpenManifest(dir) comp, err := NewCompressor(compName, DefaultLevel) if err != nil { t.Fatal(err) } opt := testOpts(dir, "none") opt.Compressor = comp w, _ := Open(opt, man) if err := w.WriteChunk(chunk(1, "a", "b")); err != nil { t.Fatal(err) } if err := w.Close(); err != nil { t.Fatal(err) } var path string entries := man.Entries() if len(entries) != 1 { t.Fatalf("%s: entries %d", compName, len(entries)) } path = filepath.Join(dir, entries[0].Path) f, err := os.Open(path) if err != nil { t.Fatal(err) } r, err := comp.Reader(f) if err != nil { t.Fatal(err) } var b strings.Builder buf := make([]byte, 1024) for { n, err := r.Read(buf) b.Write(buf[:n]) if err != nil { break } } r.Close() f.Close() if b.String() != "1\ta\n2\tb\n" { t.Fatalf("%s: got %q", compName, b.String()) } } } func TestCompressionLevelMapping(t *testing.T) { xzDictMiB := []int{1, 2, 4, 4, 8, 8, 16, 32, 64} for level := 1; level <= 9; level++ { c, err := NewCompressor("gzip", level) if err != nil { t.Fatalf("level %d: %v", level, err) } if c.Level != level || c.Name != "gzip" || c.Ext != ".gz" { t.Fatalf("level %d: %+v", level, c) } if got := gzipLevel(level); got != level { t.Fatalf("gzip level %d -> %d", level, got) } var wantZstd zstd.EncoderLevel switch { case level <= 2: wantZstd = zstd.SpeedFastest case level <= 5: wantZstd = zstd.SpeedDefault case level <= 8: wantZstd = zstd.SpeedBetterCompression default: wantZstd = zstd.SpeedBestCompression } if got := zstdLevel(level); got != wantZstd { t.Fatalf("zstd level %d -> %v want %v", level, got, wantZstd) } if got, want := xzDictCap(level), xzDictMiB[level-1]<<20; got != want { t.Fatalf("xz level %d -> %d want %d", level, got, want) } } for _, name := range []string{"none", "gzip", "zstd", "xz"} { for _, level := range []int{-1, 0, 10} { if _, err := NewCompressor(name, level); err == nil { t.Fatalf("%s level %d: expected error", name, level) } } } if _, err := NewCompressor("brotli", DefaultLevel); err == nil { t.Fatal("unsupported compressor must fail") } for name, ext := range map[string]string{"none": "", "gzip": ".gz", "zstd": ".zst", "xz": ".xz"} { c, err := NewCompressor(name, DefaultLevel) if err != nil { t.Fatalf("%s: %v", name, err) } if c.Ext != ext { t.Fatalf("%s: ext %q want %q", name, c.Ext, ext) } } } func TestXZCheckpointKeepsAllStreams(t *testing.T) { dir := t.TempDir() man, _ := OpenManifest(dir) defer man.Close() comp, err := NewCompressor("xz", 9) if err != nil { t.Fatal(err) } opt := testOpts(dir, "none") opt.Compressor = comp w, _ := Open(opt, man) if err := w.WriteChunk(chunk(1, "a", "b")); err != nil { t.Fatal(err) } if err := w.Checkpoint(); err != nil { t.Fatal(err) } if err := w.WriteChunk(chunk(3, "c", "d")); err != nil { t.Fatal(err) } if err := w.Close(); err != nil { t.Fatal(err) } entries := man.Entries() if len(entries) != 1 { t.Fatalf("entries %d", len(entries)) } if entries[0].Rows != 4 { t.Fatalf("rows %d", entries[0].Rows) } if !strings.HasSuffix(entries[0].Path, ".dump.xz") { t.Fatalf("path %q", entries[0].Path) } f, err := os.Open(filepath.Join(dir, entries[0].Path)) if err != nil { t.Fatal(err) } defer f.Close() r, err := comp.Reader(f) if err != nil { t.Fatal(err) } defer r.Close() got, err := io.ReadAll(r) if err != nil { t.Fatal(err) } if string(got) != "1\ta\n2\tb\n3\tc\n4\td\n" { t.Fatalf("got %q", got) } } func TestManifestAdoptXZOrphan(t *testing.T) { dir := t.TempDir() man, _ := OpenManifest(dir) defer man.Close() comp, err := NewCompressor("xz", DefaultLevel) if err != nil { t.Fatal(err) } var buf bytes.Buffer xw, err := comp.Wrap(&buf) if err != nil { t.Fatal(err) } if _, err := xw.Write([]byte("9\tx\n10\ty\n")); err != nil { t.Fatal(err) } if err := xw.Close(); err != nil { t.Fatal(err) } orphan := filepath.Join(dir, "000007.9-12.dump.xz") if err := os.WriteFile(orphan, buf.Bytes(), 0o644); err != nil { t.Fatal(err) } maxSeq, err := man.Adopt(dir, "dump", comp) if err != nil { t.Fatal(err) } if maxSeq[""] != 7 { t.Fatalf("maxSeq %+v", maxSeq) } entries := man.Entries() if len(entries) != 1 || entries[0].Seq != 7 || entries[0].Rows != 2 { t.Fatalf("entries %+v", entries) } if entries[0].Path != "000007.9-12.dump.xz" { t.Fatalf("path %q", entries[0].Path) } // A resumed writer must continue after the adopted seq, not overwrite it. opt := testOpts(dir, "none") opt.Compressor = comp opt.StartSeq = maxSeq[""] w, _ := Open(opt, man) if err := w.WriteChunk(chunk(1, "a")); err != nil { t.Fatal(err) } if err := w.Close(); err != nil { t.Fatal(err) } if _, err := os.Stat(filepath.Join(dir, "000008.1-1.dump.xz")); err != nil { t.Fatal(err) } if _, err := os.Stat(orphan); err != nil { t.Fatalf("adopted shard was overwritten: %v", err) } }