Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4a6b504f
commit
366d61ca25
49 files changed
+5099
-342
No files matched your search
@@ -0,0 +1,83 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"io"
|
||||
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
// DumpReader decodes the dump format produced by internal/row.dumpEncoder:
|
||||
// TAB-separated fields, LF-terminated rows, \N for NULL and the MySQL escape
|
||||
// table. It exists for the insert fallback engine; the load engine hands the
|
||||
// bytes to the server unchanged.
|
||||
type DumpReader struct {
|
||||
br *bufio.Reader
|
||||
}
|
||||
|
||||
func NewDumpReader(r io.Reader) *DumpReader {
|
||||
return &DumpReader{br: bufio.NewReaderSize(r, 256*1024)}
|
||||
}
|
||||
|
||||
// ReadRow returns the next row. It returns io.EOF exactly when the stream ends.
|
||||
func (d *DumpReader) ReadRow() (row.Row, error) {
|
||||
line, err := d.br.ReadBytes('\n')
|
||||
if len(line) == 0 && err == io.EOF {
|
||||
return row.Row{}, io.EOF
|
||||
}
|
||||
if err != nil && err != io.EOF {
|
||||
return row.Row{}, err
|
||||
}
|
||||
line = bytes.TrimSuffix(line, []byte{'\n'})
|
||||
fields := bytes.Split(line, []byte{'\t'})
|
||||
out := row.Row{Values: make([]row.Value, len(fields))}
|
||||
for i, f := range fields {
|
||||
out.Values[i] = unescapeField(f)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// unescapeField mirrors MySQL's LOAD DATA escape handling. A field is NULL only
|
||||
// when it is exactly \N; unknown escapes yield the escaped character, matching
|
||||
// the server's default FIELDS ESCAPED BY '\\' behavior.
|
||||
func unescapeField(f []byte) row.Value {
|
||||
if len(f) == 2 && f[0] == '\\' && f[1] == 'N' {
|
||||
return row.Value{}
|
||||
}
|
||||
if bytes.IndexByte(f, '\\') < 0 {
|
||||
return row.Value{Raw: append([]byte{}, f...)}
|
||||
}
|
||||
out := make([]byte, 0, len(f))
|
||||
for i := 0; i < len(f); i++ {
|
||||
if f[i] != '\\' {
|
||||
out = append(out, f[i])
|
||||
continue
|
||||
}
|
||||
if i+1 >= len(f) {
|
||||
out = append(out, '\\')
|
||||
break
|
||||
}
|
||||
i++
|
||||
switch f[i] {
|
||||
case '\\':
|
||||
out = append(out, '\\')
|
||||
case 't':
|
||||
out = append(out, '\t')
|
||||
case 'n':
|
||||
out = append(out, '\n')
|
||||
case 'r':
|
||||
out = append(out, '\r')
|
||||
case '0':
|
||||
out = append(out, 0)
|
||||
case 'b':
|
||||
out = append(out, '\b')
|
||||
case 'Z':
|
||||
out = append(out, 0x1a)
|
||||
default:
|
||||
// Includes \N in a mixed field: only a whole-field \N is NULL.
|
||||
out = append(out, f[i])
|
||||
}
|
||||
}
|
||||
return row.Value{Raw: out}
|
||||
}
|
||||
Reference in new issue
Block a user