Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,68 @@
|
||||
package row
|
||||
|
||||
import "bytes"
|
||||
|
||||
// csvEncoder emits RFC4180 CSV with a column-name header and CRLF line
|
||||
// endings. NULL is written as the configured token and empty strings as empty
|
||||
// fields.
|
||||
type csvEncoder struct {
|
||||
cols []Column
|
||||
idx []int
|
||||
null string
|
||||
}
|
||||
|
||||
func (e *csvEncoder) Header() []byte {
|
||||
var b bytes.Buffer
|
||||
for n, i := range e.idx {
|
||||
if n > 0 {
|
||||
b.WriteByte(',')
|
||||
}
|
||||
writeCSVField(&b, []byte(e.cols[i].Name))
|
||||
}
|
||||
b.WriteString("\r\n")
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func (e *csvEncoder) Encode(dst []byte, r Row) ([]byte, error) {
|
||||
for n, i := range e.idx {
|
||||
if n > 0 {
|
||||
dst = append(dst, ',')
|
||||
}
|
||||
v := r.Values[i]
|
||||
if v.IsNull() {
|
||||
dst = append(dst, e.null...)
|
||||
continue
|
||||
}
|
||||
dst = appendCSVField(dst, v.Raw)
|
||||
}
|
||||
return append(dst, '\r', '\n'), nil
|
||||
}
|
||||
|
||||
func needsCSVQuote(b []byte) bool {
|
||||
for _, c := range b {
|
||||
switch c {
|
||||
case ',', '"', '\r', '\n':
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func appendCSVField(dst, field []byte) []byte {
|
||||
if !needsCSVQuote(field) {
|
||||
return append(dst, field...)
|
||||
}
|
||||
dst = append(dst, '"')
|
||||
for _, c := range field {
|
||||
if c == '"' {
|
||||
dst = append(dst, '"', '"')
|
||||
continue
|
||||
}
|
||||
dst = append(dst, c)
|
||||
}
|
||||
return append(dst, '"')
|
||||
}
|
||||
|
||||
func writeCSVField(b *bytes.Buffer, field []byte) {
|
||||
b.Write(appendCSVField(nil, field))
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package row
|
||||
|
||||
// dumpEncoder emits MySQL LOAD DATA compatible TSV. Fields are separated by
|
||||
// TAB, rows by LF, NULL is \N and the escape table matches
|
||||
// SELECT ... INTO OUTFILE with the default FIELDS ESCAPED BY '\\'.
|
||||
type dumpEncoder struct {
|
||||
idx []int
|
||||
}
|
||||
|
||||
func (e *dumpEncoder) Header() []byte { return nil }
|
||||
|
||||
func (e *dumpEncoder) Encode(dst []byte, r Row) ([]byte, error) {
|
||||
for n, i := range e.idx {
|
||||
if n > 0 {
|
||||
dst = append(dst, '\t')
|
||||
}
|
||||
v := r.Values[i]
|
||||
if v.IsNull() {
|
||||
dst = append(dst, '\\', 'N')
|
||||
continue
|
||||
}
|
||||
for _, b := range v.Raw {
|
||||
switch b {
|
||||
case '\\':
|
||||
dst = append(dst, '\\', '\\')
|
||||
case '\t':
|
||||
dst = append(dst, '\\', 't')
|
||||
case '\n':
|
||||
dst = append(dst, '\\', 'n')
|
||||
case '\r':
|
||||
dst = append(dst, '\\', 'r')
|
||||
case 0:
|
||||
dst = append(dst, '\\', '0')
|
||||
case '\b':
|
||||
dst = append(dst, '\\', 'b')
|
||||
case 0x1a:
|
||||
dst = append(dst, '\\', 'Z')
|
||||
default:
|
||||
dst = append(dst, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
return append(dst, '\n'), nil
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package row
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Options carries encoder-level knobs shared by all formats.
|
||||
type Options struct {
|
||||
CSVNull string
|
||||
}
|
||||
|
||||
// Encoder appends the textual representation of a row to dst. It is not safe
|
||||
// for concurrent use; each shard owns its own encoder.
|
||||
type Encoder interface {
|
||||
// Header returns bytes that must be written at the start of every shard.
|
||||
Header() []byte
|
||||
// Encode appends one row to dst.
|
||||
Encode(dst []byte, r Row) ([]byte, error)
|
||||
}
|
||||
|
||||
// NewEncoder builds an encoder for the given format. cols describes every
|
||||
// column in the row (select order) and idx selects which of them are emitted,
|
||||
// in output order.
|
||||
func NewEncoder(format string, cols []Column, idx []int, opts Options) (Encoder, error) {
|
||||
if len(idx) == 0 {
|
||||
idx = make([]int, len(cols))
|
||||
for i := range cols {
|
||||
idx[i] = i
|
||||
}
|
||||
}
|
||||
switch format {
|
||||
case "dump":
|
||||
return &dumpEncoder{idx: idx}, nil
|
||||
case "csv":
|
||||
return &csvEncoder{cols: cols, idx: idx, null: opts.CSVNull}, nil
|
||||
case "jsonl":
|
||||
return &jsonlEncoder{cols: cols, idx: idx}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported format %q", format)
|
||||
}
|
||||
}
|
||||
|
||||
// FormatExt returns the file extension for a format.
|
||||
func FormatExt(format string) string {
|
||||
switch format {
|
||||
case "dump":
|
||||
return "dump"
|
||||
case "csv":
|
||||
return "csv"
|
||||
case "jsonl":
|
||||
return "jsonl"
|
||||
}
|
||||
return format
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package row
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func vals(raw ...any) Row {
|
||||
r := Row{}
|
||||
for _, v := range raw {
|
||||
switch x := v.(type) {
|
||||
case nil:
|
||||
r.Values = append(r.Values, Value{})
|
||||
case string:
|
||||
r.Values = append(r.Values, Value{Raw: []byte(x)})
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func cols(names ...string) []Column {
|
||||
out := make([]Column, len(names))
|
||||
for i, n := range names {
|
||||
out[i] = Column{Name: n, Type: "varchar"}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestDumpEncoding(t *testing.T) {
|
||||
enc, err := NewEncoder("dump", cols("a", "b", "c"), nil, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := vals("x\ty", nil, `a\nb`)
|
||||
got, err := enc.Encode(nil, r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "x\\ty\t\\N\ta\\\\nb\n"
|
||||
if string(got) != want {
|
||||
t.Fatalf("got %q want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDumpEscapesControlBytes(t *testing.T) {
|
||||
enc, _ := NewEncoder("dump", cols("a"), nil, Options{})
|
||||
r := Row{Values: []Value{{Raw: []byte{'a', 0, '\b', 0x1a, '\r'}}}}
|
||||
got, _ := enc.Encode(nil, r)
|
||||
want := "a\\0\\b\\Z\\r\n"
|
||||
if string(got) != want {
|
||||
t.Fatalf("got %q want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVEncoding(t *testing.T) {
|
||||
enc, err := NewEncoder("csv", cols("a", "b", "c", "d"), nil, Options{CSVNull: `\N`})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h := string(enc.Header()); h != "a,b,c,d\r\n" {
|
||||
t.Fatalf("header %q", h)
|
||||
}
|
||||
r := vals(`a,"b"`, "", nil, "plain")
|
||||
got, _ := enc.Encode(nil, r)
|
||||
want := "\"a,\"\"b\"\"\",,\\N,plain\r\n"
|
||||
if string(got) != want {
|
||||
t.Fatalf("got %q want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVNullToken(t *testing.T) {
|
||||
enc, _ := NewEncoder("csv", cols("a"), nil, Options{CSVNull: "NULL"})
|
||||
got, _ := enc.Encode(nil, vals(nil))
|
||||
if string(got) != "NULL\r\n" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONLTypes(t *testing.T) {
|
||||
columns := []Column{
|
||||
{Name: "id", Type: "bigint"},
|
||||
{Name: "price", Type: "decimal"},
|
||||
{Name: "created", Type: "datetime"},
|
||||
{Name: "day", Type: "date"},
|
||||
{Name: "blob", Type: "blob"},
|
||||
{Name: "note", Type: "varchar"},
|
||||
{Name: "missing", Type: "varchar"},
|
||||
}
|
||||
enc, _ := NewEncoder("jsonl", columns, nil, Options{})
|
||||
r := vals("42", "12.30", "2024-01-02 03:04:05", "2024-01-02", "hi", `line "one"`, nil)
|
||||
got, _ := enc.Encode(nil, r)
|
||||
want := `{"id":42,"price":"12.30","created":"2024-01-02T03:04:05Z","day":"2024-01-02","blob":"aGk=","note":"line \"one\"","missing":null}` + "\n"
|
||||
if string(got) != want {
|
||||
t.Fatalf("got %s\nwant %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONLColumnProjection(t *testing.T) {
|
||||
columns := []Column{{Name: "a", Type: "int"}, {Name: "b", Type: "int"}, {Name: "c", Type: "int"}}
|
||||
enc, _ := NewEncoder("jsonl", columns, []int{2, 0}, Options{})
|
||||
got, _ := enc.Encode(nil, vals("1", "2", "3"))
|
||||
if strings.TrimSpace(string(got)) != `{"c":3,"a":1}` {
|
||||
t.Fatalf("got %s", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package row
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"time"
|
||||
)
|
||||
|
||||
// jsonlEncoder emits one JSON object per line. Types are mapped deliberately
|
||||
// so downstream tools see useful values: decimals become strings to preserve
|
||||
// precision, binary columns become base64, date/time columns become strings
|
||||
// and NULL becomes JSON null.
|
||||
type jsonlEncoder struct {
|
||||
cols []Column
|
||||
idx []int
|
||||
}
|
||||
|
||||
func (e *jsonlEncoder) Header() []byte { return nil }
|
||||
|
||||
func (e *jsonlEncoder) Encode(dst []byte, r Row) ([]byte, error) {
|
||||
dst = append(dst, '{')
|
||||
for n, i := range e.idx {
|
||||
if n > 0 {
|
||||
dst = append(dst, ',')
|
||||
}
|
||||
dst = appendJSONString(dst, []byte(e.cols[i].Name))
|
||||
dst = append(dst, ':')
|
||||
dst = appendJSONValue(dst, e.cols[i], r.Values[i])
|
||||
}
|
||||
return append(dst, '}', '\n'), nil
|
||||
}
|
||||
|
||||
func appendJSONValue(dst []byte, col Column, v Value) []byte {
|
||||
if v.IsNull() {
|
||||
return append(dst, "null"...)
|
||||
}
|
||||
switch {
|
||||
case IsInteger(col.Type), IsFloat(col.Type):
|
||||
if looksNumeric(v.Raw) {
|
||||
return append(dst, v.Raw...)
|
||||
}
|
||||
return appendJSONString(dst, v.Raw)
|
||||
case IsDecimal(col.Type):
|
||||
return appendJSONString(dst, v.Raw)
|
||||
case IsBinary(col.Type):
|
||||
dst = append(dst, '"')
|
||||
dst = append(dst, base64.StdEncoding.EncodeToString(v.Raw)...)
|
||||
return append(dst, '"')
|
||||
case IsDate(col.Type):
|
||||
return appendJSONString(dst, v.Raw)
|
||||
case IsDateTime(col.Type):
|
||||
if s, ok := formatRFC3339(v.Raw); ok {
|
||||
return appendJSONString(dst, []byte(s))
|
||||
}
|
||||
return appendJSONString(dst, v.Raw)
|
||||
default:
|
||||
return appendJSONString(dst, v.Raw)
|
||||
}
|
||||
}
|
||||
|
||||
func looksNumeric(b []byte) bool {
|
||||
if len(b) == 0 {
|
||||
return false
|
||||
}
|
||||
for i, c := range b {
|
||||
if (c >= '0' && c <= '9') || c == '-' || c == '+' || c == '.' || c == 'e' || c == 'E' {
|
||||
continue
|
||||
}
|
||||
if i == 0 && c == ' ' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var dateTimeLayouts = []string{
|
||||
"2006-01-02 15:04:05.999999999",
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02T15:04:05Z07:00",
|
||||
"2006-01-02T15:04:05.999999999Z07:00",
|
||||
}
|
||||
|
||||
func formatRFC3339(raw []byte) (string, bool) {
|
||||
s := string(raw)
|
||||
for _, layout := range dateTimeLayouts {
|
||||
t, err := time.ParseInLocation(layout, s, time.UTC)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if t.Nanosecond() != 0 {
|
||||
return t.UTC().Format(time.RFC3339Nano), true
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
const hexDigits = "0123456789abcdef"
|
||||
|
||||
func appendJSONString(dst, s []byte) []byte {
|
||||
dst = append(dst, '"')
|
||||
for _, c := range s {
|
||||
switch c {
|
||||
case '"', '\\':
|
||||
dst = append(dst, '\\', c)
|
||||
case '\n':
|
||||
dst = append(dst, '\\', 'n')
|
||||
case '\r':
|
||||
dst = append(dst, '\\', 'r')
|
||||
case '\t':
|
||||
dst = append(dst, '\\', 't')
|
||||
case '\b':
|
||||
dst = append(dst, '\\', 'b')
|
||||
case '\f':
|
||||
dst = append(dst, '\\', 'f')
|
||||
default:
|
||||
if c < 0x20 {
|
||||
dst = append(dst, '\\', 'u', '0', '0', hexDigits[c>>4], hexDigits[c&0xf])
|
||||
} else {
|
||||
dst = append(dst, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
return append(dst, '"')
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
// Package row defines scanned values, key tuples and the line encoders used
|
||||
// to serialize rows into dump, csv and jsonl shards.
|
||||
package row
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Value is one column value as returned by the MySQL driver with
|
||||
// parseTime disabled. A nil Raw means SQL NULL; an empty non-nil Raw means
|
||||
// the empty string.
|
||||
type Value struct {
|
||||
Raw []byte
|
||||
}
|
||||
|
||||
func (v Value) IsNull() bool { return v.Raw == nil }
|
||||
func (v Value) Bytes() []byte { return v.Raw }
|
||||
func (v Value) String() string { return string(v.Raw) }
|
||||
|
||||
// DriverValue returns the value as a bind parameter.
|
||||
func (v Value) DriverValue() any {
|
||||
if v.Raw == nil {
|
||||
return nil
|
||||
}
|
||||
return v.Raw
|
||||
}
|
||||
|
||||
// KeyPart is a single component of a key tuple. Values are held as text so a
|
||||
// key can be serialized into state.json and rebound to the driver without
|
||||
// carrying driver-specific types.
|
||||
type KeyPart struct {
|
||||
Null bool `json:"null,omitempty"`
|
||||
Val string `json:"val,omitempty"`
|
||||
}
|
||||
|
||||
// DriverValue converts the part into a value accepted by database/sql.
|
||||
func (p KeyPart) DriverValue() any {
|
||||
if p.Null {
|
||||
return nil
|
||||
}
|
||||
return p.Val
|
||||
}
|
||||
|
||||
// Key is a tuple of values identifying a row position in a unique index.
|
||||
type Key []KeyPart
|
||||
|
||||
func KeyOf(values ...string) Key {
|
||||
parts := make(Key, len(values))
|
||||
for i, v := range values {
|
||||
parts[i] = KeyPart{Val: v}
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
// Args returns the key as driver bind parameters.
|
||||
func (k Key) Args() []any {
|
||||
out := make([]any, len(k))
|
||||
for i, p := range k {
|
||||
out[i] = p.DriverValue()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (k Key) IsZero() bool { return len(k) == 0 }
|
||||
|
||||
func (k Key) Equal(o Key) bool {
|
||||
if len(k) != len(o) {
|
||||
return false
|
||||
}
|
||||
for i := range k {
|
||||
if k[i] != o[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (k Key) String() string {
|
||||
if len(k) == 0 {
|
||||
return "()"
|
||||
}
|
||||
parts := make([]string, len(k))
|
||||
for i, p := range k {
|
||||
if p.Null {
|
||||
parts[i] = "NULL"
|
||||
} else {
|
||||
parts[i] = p.Val
|
||||
}
|
||||
}
|
||||
return "(" + strings.Join(parts, ",") + ")"
|
||||
}
|
||||
|
||||
// FirstLast returns the two parts of a composite (or single) key.
|
||||
func (k Key) FirstLast() (KeyPart, KeyPart) {
|
||||
if len(k) == 0 {
|
||||
return KeyPart{Null: true}, KeyPart{Null: true}
|
||||
}
|
||||
return k[0], k[len(k)-1]
|
||||
}
|
||||
|
||||
// Row holds a scanned row in select-list order.
|
||||
type Row struct {
|
||||
Values []Value
|
||||
}
|
||||
|
||||
// Chunk is a batch of rows produced by one scan step. Keys is parallel to
|
||||
// Rows and holds the unique-index key of each row; Buckets is parallel to
|
||||
// Rows and is populated only in date sharding mode.
|
||||
type Chunk struct {
|
||||
Rows []Row
|
||||
Keys []Key
|
||||
Buckets []string
|
||||
FirstKey Key
|
||||
LastKey Key
|
||||
}
|
||||
|
||||
func (c *Chunk) Len() int { return len(c.Rows) }
|
||||
|
||||
// Column describes one exported column for encoding purposes.
|
||||
type Column struct {
|
||||
Name string
|
||||
Type string // lowercase information_schema DATA_TYPE
|
||||
}
|
||||
|
||||
func isOneOf(t string, set ...string) bool {
|
||||
for _, s := range set {
|
||||
if t == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func IsInteger(t string) bool {
|
||||
return isOneOf(t, "tinyint", "smallint", "mediumint", "int", "integer", "bigint", "year")
|
||||
}
|
||||
|
||||
func IsFloat(t string) bool {
|
||||
return isOneOf(t, "float", "double", "real")
|
||||
}
|
||||
|
||||
func IsDecimal(t string) bool { return isOneOf(t, "decimal", "numeric") }
|
||||
|
||||
func IsBinary(t string) bool {
|
||||
return isOneOf(t, "binary", "varbinary", "tinyblob", "blob", "mediumblob", "longblob", "bit", "geometry", "geometrycollection", "point", "linestring", "polygon", "multipoint", "multilinestring", "multipolygon")
|
||||
}
|
||||
|
||||
func IsDate(t string) bool { return t == "date" }
|
||||
|
||||
func IsDateTime(t string) bool { return isOneOf(t, "datetime", "timestamp") }
|
||||
|
||||
func IsTimeType(t string) bool { return isOneOf(t, "date", "datetime", "timestamp") }
|
||||
Reference in new issue
Block a user