Add archer import and allow date-mode bulk delete

archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-08 21:14:23 +08:00
1 parent bc4a6b504f
commit 366d61ca25
49 files changed
+5099 -342

No files matched your search

+163
View File
@@ -0,0 +1,163 @@
package importer
import (
"bufio"
"bytes"
"fmt"
"io"
"os"
"strings"
"archer/internal/archererr"
"archer/internal/schema"
"archer/internal/shard"
"archer/internal/sqlutil"
)
// Mapping translates file fields to target columns. It is the single place
// that decides whether a field is loaded into a named column or discarded into
// a user variable (generated columns under positional mapping).
type Mapping struct {
// FileColumns has one entry per file field: the canonical target column
// name, or "" when the field is intentionally discarded.
FileColumns []string
// InsertColumns and InsertFieldIdx describe the parameterized INSERT used
// by the insert engine. InsertFieldIdx points into FileColumns.
InsertColumns []string
InsertFieldIdx []int
// LoadList is the LOAD DATA column list: quoted identifiers and
// @_archer_skip_N variables.
LoadList []string
// Positional is true for --assume-all-columns; callers must warn.
Positional bool
}
// BuildMapping resolves the three permitted mapping sources. Columns recorded
// in the archive win over an explicit --columns by construction (the conflict
// is rejected earlier); assumeAll is only consulted when neither list exists.
func BuildMapping(table *schema.Table, archived, explicit []string, assumeAll bool) (*Mapping, error) {
return buildMapping(table.Name, table.Columns, archived, explicit, assumeAll)
}
func buildMapping(tableName string, columns []schema.Column, archived, explicit []string, assumeAll bool) (*Mapping, error) {
names := archived
if len(names) == 0 {
names = explicit
}
switch {
case len(names) > 0:
return mappingByName(tableName, columns, names)
case assumeAll:
return mappingPositional(tableName, columns)
default:
return nil, archererr.Newf(archererr.KindConfig,
"the archive did not record a column list; provide --columns in file field order or accept positional mapping with --assume-all-columns")
}
}
func mappingByName(tableName string, columns []schema.Column, names []string) (*Mapping, error) {
byName := make(map[string]schema.Column, len(columns))
for _, c := range columns {
byName[strings.ToLower(c.Name)] = c
}
m := &Mapping{}
seen := map[string]bool{}
for _, requested := range names {
col, ok := byName[strings.ToLower(requested)]
if !ok {
return nil, archererr.Newf(archererr.KindValidation,
"column %q from the file field mapping does not exist in target table %s", requested, tableName)
}
if isGenerated(col) {
return nil, archererr.Newf(archererr.KindValidation,
"column %q is a generated column and cannot be written from an archive", col.Name)
}
if seen[col.Name] {
return nil, archererr.Newf(archererr.KindValidation,
"column %q appears more than once in the file field mapping", col.Name)
}
seen[col.Name] = true
idx := len(m.FileColumns)
m.FileColumns = append(m.FileColumns, col.Name)
m.InsertColumns = append(m.InsertColumns, col.Name)
m.InsertFieldIdx = append(m.InsertFieldIdx, idx)
m.LoadList = append(m.LoadList, sqlutil.Ident(col.Name))
}
return m, nil
}
func mappingPositional(tableName string, columns []schema.Column) (*Mapping, error) {
m := &Mapping{Positional: true}
for i, col := range columns {
if isGenerated(col) {
// The file still contains this field, so keep a placeholder and
// discard the value; the target table recomputes it.
m.FileColumns = append(m.FileColumns, "")
m.LoadList = append(m.LoadList, fmt.Sprintf("@_archer_skip_%d", col.Ordinal))
continue
}
m.FileColumns = append(m.FileColumns, col.Name)
m.InsertColumns = append(m.InsertColumns, col.Name)
m.InsertFieldIdx = append(m.InsertFieldIdx, i)
m.LoadList = append(m.LoadList, sqlutil.Ident(col.Name))
}
if len(m.InsertColumns) == 0 {
return nil, archererr.Newf(archererr.KindValidation,
"target table %s has no writable columns", tableName)
}
return m, nil
}
func isGenerated(c schema.Column) bool {
return strings.Contains(strings.ToLower(c.Extra), "generated")
}
// CountFields counts dump fields in one raw line. Literal tabs are escaped by
// the dump encoder, so every 0x09 byte is a separator. An empty line is a
// single empty, non-NULL field.
func CountFields(line []byte) int {
return strings.Count(string(line), "\t") + 1
}
// FirstLineFieldCount reads the first decompressed line of a shard.
func FirstLineFieldCount(path string, comp *shard.Compressor) (fields int, hasRow bool, err error) {
f, err := os.Open(path)
if err != nil {
return 0, false, err
}
defer f.Close()
r, err := comp.Reader(f)
if err != nil {
return 0, false, err
}
defer r.Close()
br := bufio.NewReaderSize(r, 64*1024)
line, err := br.ReadBytes('\n')
if err == io.EOF {
if len(line) == 0 {
return 0, false, nil
}
} else if err != nil {
return 0, false, err
}
line = bytes.TrimSuffix(line, []byte{'\n'})
return CountFields(line), true, nil
}
// CheckFieldCount compares the first row's field count with the load list
// length. It is a sanity check only: it cannot prove that same-width columns
// are in the right order.
func CheckFieldCount(path string, comp *shard.Compressor, want int) error {
got, hasRow, err := FirstLineFieldCount(path, comp)
if err != nil {
return archererr.Wrap(archererr.KindValidation, fmt.Errorf("read first row of %s: %w", path, err))
}
if !hasRow {
return nil
}
if got != want {
return archererr.Newf(archererr.KindValidation,
"first row of %s has %d fields, but the load column list has %d; refusing to load", path, got, want)
}
return nil
}