Initial commit: archer, a Go-native MySQL/MariaDB table archiver

Renamed from arcdb: module path, cmd/archer, internal/archererr,
docs, build scripts and ARCHER_* environment variables.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-06 22:12:17 +08:00
commit bc4a6b504f
99 files changed
+12752

No files matched your search

+68
View File
@@ -0,0 +1,68 @@
package row
import "bytes"
// csvEncoder emits RFC4180 CSV with a column-name header and CRLF line
// endings. NULL is written as the configured token and empty strings as empty
// fields.
type csvEncoder struct {
cols []Column
idx []int
null string
}
func (e *csvEncoder) Header() []byte {
var b bytes.Buffer
for n, i := range e.idx {
if n > 0 {
b.WriteByte(',')
}
writeCSVField(&b, []byte(e.cols[i].Name))
}
b.WriteString("\r\n")
return b.Bytes()
}
func (e *csvEncoder) Encode(dst []byte, r Row) ([]byte, error) {
for n, i := range e.idx {
if n > 0 {
dst = append(dst, ',')
}
v := r.Values[i]
if v.IsNull() {
dst = append(dst, e.null...)
continue
}
dst = appendCSVField(dst, v.Raw)
}
return append(dst, '\r', '\n'), nil
}
func needsCSVQuote(b []byte) bool {
for _, c := range b {
switch c {
case ',', '"', '\r', '\n':
return true
}
}
return false
}
func appendCSVField(dst, field []byte) []byte {
if !needsCSVQuote(field) {
return append(dst, field...)
}
dst = append(dst, '"')
for _, c := range field {
if c == '"' {
dst = append(dst, '"', '"')
continue
}
dst = append(dst, c)
}
return append(dst, '"')
}
func writeCSVField(b *bytes.Buffer, field []byte) {
b.Write(appendCSVField(nil, field))
}
+44
View File
@@ -0,0 +1,44 @@
package row
// dumpEncoder emits MySQL LOAD DATA compatible TSV. Fields are separated by
// TAB, rows by LF, NULL is \N and the escape table matches
// SELECT ... INTO OUTFILE with the default FIELDS ESCAPED BY '\\'.
type dumpEncoder struct {
idx []int
}
func (e *dumpEncoder) Header() []byte { return nil }
func (e *dumpEncoder) Encode(dst []byte, r Row) ([]byte, error) {
for n, i := range e.idx {
if n > 0 {
dst = append(dst, '\t')
}
v := r.Values[i]
if v.IsNull() {
dst = append(dst, '\\', 'N')
continue
}
for _, b := range v.Raw {
switch b {
case '\\':
dst = append(dst, '\\', '\\')
case '\t':
dst = append(dst, '\\', 't')
case '\n':
dst = append(dst, '\\', 'n')
case '\r':
dst = append(dst, '\\', 'r')
case 0:
dst = append(dst, '\\', '0')
case '\b':
dst = append(dst, '\\', 'b')
case 0x1a:
dst = append(dst, '\\', 'Z')
default:
dst = append(dst, b)
}
}
}
return append(dst, '\n'), nil
}
+52
View File
@@ -0,0 +1,52 @@
package row
import "fmt"
// Options carries encoder-level knobs shared by all formats.
type Options struct {
CSVNull string
}
// Encoder appends the textual representation of a row to dst. It is not safe
// for concurrent use; each shard owns its own encoder.
type Encoder interface {
// Header returns bytes that must be written at the start of every shard.
Header() []byte
// Encode appends one row to dst.
Encode(dst []byte, r Row) ([]byte, error)
}
// NewEncoder builds an encoder for the given format. cols describes every
// column in the row (select order) and idx selects which of them are emitted,
// in output order.
func NewEncoder(format string, cols []Column, idx []int, opts Options) (Encoder, error) {
if len(idx) == 0 {
idx = make([]int, len(cols))
for i := range cols {
idx[i] = i
}
}
switch format {
case "dump":
return &dumpEncoder{idx: idx}, nil
case "csv":
return &csvEncoder{cols: cols, idx: idx, null: opts.CSVNull}, nil
case "jsonl":
return &jsonlEncoder{cols: cols, idx: idx}, nil
default:
return nil, fmt.Errorf("unsupported format %q", format)
}
}
// FormatExt returns the file extension for a format.
func FormatExt(format string) string {
switch format {
case "dump":
return "dump"
case "csv":
return "csv"
case "jsonl":
return "jsonl"
}
return format
}
+105
View File
@@ -0,0 +1,105 @@
package row
import (
"strings"
"testing"
)
func vals(raw ...any) Row {
r := Row{}
for _, v := range raw {
switch x := v.(type) {
case nil:
r.Values = append(r.Values, Value{})
case string:
r.Values = append(r.Values, Value{Raw: []byte(x)})
}
}
return r
}
func cols(names ...string) []Column {
out := make([]Column, len(names))
for i, n := range names {
out[i] = Column{Name: n, Type: "varchar"}
}
return out
}
func TestDumpEncoding(t *testing.T) {
enc, err := NewEncoder("dump", cols("a", "b", "c"), nil, Options{})
if err != nil {
t.Fatal(err)
}
r := vals("x\ty", nil, `a\nb`)
got, err := enc.Encode(nil, r)
if err != nil {
t.Fatal(err)
}
want := "x\\ty\t\\N\ta\\\\nb\n"
if string(got) != want {
t.Fatalf("got %q want %q", got, want)
}
}
func TestDumpEscapesControlBytes(t *testing.T) {
enc, _ := NewEncoder("dump", cols("a"), nil, Options{})
r := Row{Values: []Value{{Raw: []byte{'a', 0, '\b', 0x1a, '\r'}}}}
got, _ := enc.Encode(nil, r)
want := "a\\0\\b\\Z\\r\n"
if string(got) != want {
t.Fatalf("got %q want %q", got, want)
}
}
func TestCSVEncoding(t *testing.T) {
enc, err := NewEncoder("csv", cols("a", "b", "c", "d"), nil, Options{CSVNull: `\N`})
if err != nil {
t.Fatal(err)
}
if h := string(enc.Header()); h != "a,b,c,d\r\n" {
t.Fatalf("header %q", h)
}
r := vals(`a,"b"`, "", nil, "plain")
got, _ := enc.Encode(nil, r)
want := "\"a,\"\"b\"\"\",,\\N,plain\r\n"
if string(got) != want {
t.Fatalf("got %q want %q", got, want)
}
}
func TestCSVNullToken(t *testing.T) {
enc, _ := NewEncoder("csv", cols("a"), nil, Options{CSVNull: "NULL"})
got, _ := enc.Encode(nil, vals(nil))
if string(got) != "NULL\r\n" {
t.Fatalf("got %q", got)
}
}
func TestJSONLTypes(t *testing.T) {
columns := []Column{
{Name: "id", Type: "bigint"},
{Name: "price", Type: "decimal"},
{Name: "created", Type: "datetime"},
{Name: "day", Type: "date"},
{Name: "blob", Type: "blob"},
{Name: "note", Type: "varchar"},
{Name: "missing", Type: "varchar"},
}
enc, _ := NewEncoder("jsonl", columns, nil, Options{})
r := vals("42", "12.30", "2024-01-02 03:04:05", "2024-01-02", "hi", `line "one"`, nil)
got, _ := enc.Encode(nil, r)
want := `{"id":42,"price":"12.30","created":"2024-01-02T03:04:05Z","day":"2024-01-02","blob":"aGk=","note":"line \"one\"","missing":null}` + "\n"
if string(got) != want {
t.Fatalf("got %s\nwant %s", got, want)
}
}
func TestJSONLColumnProjection(t *testing.T) {
columns := []Column{{Name: "a", Type: "int"}, {Name: "b", Type: "int"}, {Name: "c", Type: "int"}}
enc, _ := NewEncoder("jsonl", columns, []int{2, 0}, Options{})
got, _ := enc.Encode(nil, vals("1", "2", "3"))
if strings.TrimSpace(string(got)) != `{"c":3,"a":1}` {
t.Fatalf("got %s", got)
}
}
+125
View File
@@ -0,0 +1,125 @@
package row
import (
"encoding/base64"
"time"
)
// jsonlEncoder emits one JSON object per line. Types are mapped deliberately
// so downstream tools see useful values: decimals become strings to preserve
// precision, binary columns become base64, date/time columns become strings
// and NULL becomes JSON null.
type jsonlEncoder struct {
cols []Column
idx []int
}
func (e *jsonlEncoder) Header() []byte { return nil }
func (e *jsonlEncoder) Encode(dst []byte, r Row) ([]byte, error) {
dst = append(dst, '{')
for n, i := range e.idx {
if n > 0 {
dst = append(dst, ',')
}
dst = appendJSONString(dst, []byte(e.cols[i].Name))
dst = append(dst, ':')
dst = appendJSONValue(dst, e.cols[i], r.Values[i])
}
return append(dst, '}', '\n'), nil
}
func appendJSONValue(dst []byte, col Column, v Value) []byte {
if v.IsNull() {
return append(dst, "null"...)
}
switch {
case IsInteger(col.Type), IsFloat(col.Type):
if looksNumeric(v.Raw) {
return append(dst, v.Raw...)
}
return appendJSONString(dst, v.Raw)
case IsDecimal(col.Type):
return appendJSONString(dst, v.Raw)
case IsBinary(col.Type):
dst = append(dst, '"')
dst = append(dst, base64.StdEncoding.EncodeToString(v.Raw)...)
return append(dst, '"')
case IsDate(col.Type):
return appendJSONString(dst, v.Raw)
case IsDateTime(col.Type):
if s, ok := formatRFC3339(v.Raw); ok {
return appendJSONString(dst, []byte(s))
}
return appendJSONString(dst, v.Raw)
default:
return appendJSONString(dst, v.Raw)
}
}
func looksNumeric(b []byte) bool {
if len(b) == 0 {
return false
}
for i, c := range b {
if (c >= '0' && c <= '9') || c == '-' || c == '+' || c == '.' || c == 'e' || c == 'E' {
continue
}
if i == 0 && c == ' ' {
continue
}
return false
}
return true
}
var dateTimeLayouts = []string{
"2006-01-02 15:04:05.999999999",
"2006-01-02 15:04:05",
"2006-01-02T15:04:05Z07:00",
"2006-01-02T15:04:05.999999999Z07:00",
}
func formatRFC3339(raw []byte) (string, bool) {
s := string(raw)
for _, layout := range dateTimeLayouts {
t, err := time.ParseInLocation(layout, s, time.UTC)
if err != nil {
continue
}
if t.Nanosecond() != 0 {
return t.UTC().Format(time.RFC3339Nano), true
}
return t.UTC().Format(time.RFC3339), true
}
return "", false
}
const hexDigits = "0123456789abcdef"
func appendJSONString(dst, s []byte) []byte {
dst = append(dst, '"')
for _, c := range s {
switch c {
case '"', '\\':
dst = append(dst, '\\', c)
case '\n':
dst = append(dst, '\\', 'n')
case '\r':
dst = append(dst, '\\', 'r')
case '\t':
dst = append(dst, '\\', 't')
case '\b':
dst = append(dst, '\\', 'b')
case '\f':
dst = append(dst, '\\', 'f')
default:
if c < 0x20 {
dst = append(dst, '\\', 'u', '0', '0', hexDigits[c>>4], hexDigits[c&0xf])
} else {
dst = append(dst, c)
}
}
}
return append(dst, '"')
}
+152
View File
@@ -0,0 +1,152 @@
// Package row defines scanned values, key tuples and the line encoders used
// to serialize rows into dump, csv and jsonl shards.
package row
import (
"strings"
)
// Value is one column value as returned by the MySQL driver with
// parseTime disabled. A nil Raw means SQL NULL; an empty non-nil Raw means
// the empty string.
type Value struct {
Raw []byte
}
func (v Value) IsNull() bool { return v.Raw == nil }
func (v Value) Bytes() []byte { return v.Raw }
func (v Value) String() string { return string(v.Raw) }
// DriverValue returns the value as a bind parameter.
func (v Value) DriverValue() any {
if v.Raw == nil {
return nil
}
return v.Raw
}
// KeyPart is a single component of a key tuple. Values are held as text so a
// key can be serialized into state.json and rebound to the driver without
// carrying driver-specific types.
type KeyPart struct {
Null bool `json:"null,omitempty"`
Val string `json:"val,omitempty"`
}
// DriverValue converts the part into a value accepted by database/sql.
func (p KeyPart) DriverValue() any {
if p.Null {
return nil
}
return p.Val
}
// Key is a tuple of values identifying a row position in a unique index.
type Key []KeyPart
func KeyOf(values ...string) Key {
parts := make(Key, len(values))
for i, v := range values {
parts[i] = KeyPart{Val: v}
}
return parts
}
// Args returns the key as driver bind parameters.
func (k Key) Args() []any {
out := make([]any, len(k))
for i, p := range k {
out[i] = p.DriverValue()
}
return out
}
func (k Key) IsZero() bool { return len(k) == 0 }
func (k Key) Equal(o Key) bool {
if len(k) != len(o) {
return false
}
for i := range k {
if k[i] != o[i] {
return false
}
}
return true
}
func (k Key) String() string {
if len(k) == 0 {
return "()"
}
parts := make([]string, len(k))
for i, p := range k {
if p.Null {
parts[i] = "NULL"
} else {
parts[i] = p.Val
}
}
return "(" + strings.Join(parts, ",") + ")"
}
// FirstLast returns the two parts of a composite (or single) key.
func (k Key) FirstLast() (KeyPart, KeyPart) {
if len(k) == 0 {
return KeyPart{Null: true}, KeyPart{Null: true}
}
return k[0], k[len(k)-1]
}
// Row holds a scanned row in select-list order.
type Row struct {
Values []Value
}
// Chunk is a batch of rows produced by one scan step. Keys is parallel to
// Rows and holds the unique-index key of each row; Buckets is parallel to
// Rows and is populated only in date sharding mode.
type Chunk struct {
Rows []Row
Keys []Key
Buckets []string
FirstKey Key
LastKey Key
}
func (c *Chunk) Len() int { return len(c.Rows) }
// Column describes one exported column for encoding purposes.
type Column struct {
Name string
Type string // lowercase information_schema DATA_TYPE
}
func isOneOf(t string, set ...string) bool {
for _, s := range set {
if t == s {
return true
}
}
return false
}
func IsInteger(t string) bool {
return isOneOf(t, "tinyint", "smallint", "mediumint", "int", "integer", "bigint", "year")
}
func IsFloat(t string) bool {
return isOneOf(t, "float", "double", "real")
}
func IsDecimal(t string) bool { return isOneOf(t, "decimal", "numeric") }
func IsBinary(t string) bool {
return isOneOf(t, "binary", "varbinary", "tinyblob", "blob", "mediumblob", "longblob", "bit", "geometry", "geometrycollection", "point", "linestring", "polygon", "multipoint", "multilinestring", "multipolygon")
}
func IsDate(t string) bool { return t == "date" }
func IsDateTime(t string) bool { return isOneOf(t, "datetime", "timestamp") }
func IsTimeType(t string) bool { return isOneOf(t, "date", "datetime", "timestamp") }