Files
archer/internal/importer/dump.go
T
voidandClaude Sonnet 4.6 366d61ca25 Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-11-08 21:14:23 +08:00

84 lines
2.0 KiB
Go

package importer
import (
"bufio"
"bytes"
"io"
"archer/internal/row"
)
// DumpReader decodes the dump format produced by internal/row.dumpEncoder:
// TAB-separated fields, LF-terminated rows, \N for NULL and the MySQL escape
// table. It exists for the insert fallback engine; the load engine hands the
// bytes to the server unchanged.
type DumpReader struct {
br *bufio.Reader
}
func NewDumpReader(r io.Reader) *DumpReader {
return &DumpReader{br: bufio.NewReaderSize(r, 256*1024)}
}
// ReadRow returns the next row. It returns io.EOF exactly when the stream ends.
func (d *DumpReader) ReadRow() (row.Row, error) {
line, err := d.br.ReadBytes('\n')
if len(line) == 0 && err == io.EOF {
return row.Row{}, io.EOF
}
if err != nil && err != io.EOF {
return row.Row{}, err
}
line = bytes.TrimSuffix(line, []byte{'\n'})
fields := bytes.Split(line, []byte{'\t'})
out := row.Row{Values: make([]row.Value, len(fields))}
for i, f := range fields {
out.Values[i] = unescapeField(f)
}
return out, nil
}
// unescapeField mirrors MySQL's LOAD DATA escape handling. A field is NULL only
// when it is exactly \N; unknown escapes yield the escaped character, matching
// the server's default FIELDS ESCAPED BY '\\' behavior.
func unescapeField(f []byte) row.Value {
if len(f) == 2 && f[0] == '\\' && f[1] == 'N' {
return row.Value{}
}
if bytes.IndexByte(f, '\\') < 0 {
return row.Value{Raw: append([]byte{}, f...)}
}
out := make([]byte, 0, len(f))
for i := 0; i < len(f); i++ {
if f[i] != '\\' {
out = append(out, f[i])
continue
}
if i+1 >= len(f) {
out = append(out, '\\')
break
}
i++
switch f[i] {
case '\\':
out = append(out, '\\')
case 't':
out = append(out, '\t')
case 'n':
out = append(out, '\n')
case 'r':
out = append(out, '\r')
case '0':
out = append(out, 0)
case 'b':
out = append(out, '\b')
case 'Z':
out = append(out, 0x1a)
default:
// Includes \N in a mixed field: only a whole-field \N is NULL.
out = append(out, f[i])
}
}
return row.Value{Raw: out}
}