Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4a6b504f
commit
366d61ca25
49 files changed
+5099
-342
No files matched your search
@@ -0,0 +1,163 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/schema"
|
||||
"archer/internal/shard"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
// Mapping translates file fields to target columns. It is the single place
|
||||
// that decides whether a field is loaded into a named column or discarded into
|
||||
// a user variable (generated columns under positional mapping).
|
||||
type Mapping struct {
|
||||
// FileColumns has one entry per file field: the canonical target column
|
||||
// name, or "" when the field is intentionally discarded.
|
||||
FileColumns []string
|
||||
// InsertColumns and InsertFieldIdx describe the parameterized INSERT used
|
||||
// by the insert engine. InsertFieldIdx points into FileColumns.
|
||||
InsertColumns []string
|
||||
InsertFieldIdx []int
|
||||
// LoadList is the LOAD DATA column list: quoted identifiers and
|
||||
// @_archer_skip_N variables.
|
||||
LoadList []string
|
||||
// Positional is true for --assume-all-columns; callers must warn.
|
||||
Positional bool
|
||||
}
|
||||
|
||||
// BuildMapping resolves the three permitted mapping sources. Columns recorded
|
||||
// in the archive win over an explicit --columns by construction (the conflict
|
||||
// is rejected earlier); assumeAll is only consulted when neither list exists.
|
||||
func BuildMapping(table *schema.Table, archived, explicit []string, assumeAll bool) (*Mapping, error) {
|
||||
return buildMapping(table.Name, table.Columns, archived, explicit, assumeAll)
|
||||
}
|
||||
|
||||
func buildMapping(tableName string, columns []schema.Column, archived, explicit []string, assumeAll bool) (*Mapping, error) {
|
||||
names := archived
|
||||
if len(names) == 0 {
|
||||
names = explicit
|
||||
}
|
||||
switch {
|
||||
case len(names) > 0:
|
||||
return mappingByName(tableName, columns, names)
|
||||
case assumeAll:
|
||||
return mappingPositional(tableName, columns)
|
||||
default:
|
||||
return nil, archererr.Newf(archererr.KindConfig,
|
||||
"the archive did not record a column list; provide --columns in file field order or accept positional mapping with --assume-all-columns")
|
||||
}
|
||||
}
|
||||
|
||||
func mappingByName(tableName string, columns []schema.Column, names []string) (*Mapping, error) {
|
||||
byName := make(map[string]schema.Column, len(columns))
|
||||
for _, c := range columns {
|
||||
byName[strings.ToLower(c.Name)] = c
|
||||
}
|
||||
m := &Mapping{}
|
||||
seen := map[string]bool{}
|
||||
for _, requested := range names {
|
||||
col, ok := byName[strings.ToLower(requested)]
|
||||
if !ok {
|
||||
return nil, archererr.Newf(archererr.KindValidation,
|
||||
"column %q from the file field mapping does not exist in target table %s", requested, tableName)
|
||||
}
|
||||
if isGenerated(col) {
|
||||
return nil, archererr.Newf(archererr.KindValidation,
|
||||
"column %q is a generated column and cannot be written from an archive", col.Name)
|
||||
}
|
||||
if seen[col.Name] {
|
||||
return nil, archererr.Newf(archererr.KindValidation,
|
||||
"column %q appears more than once in the file field mapping", col.Name)
|
||||
}
|
||||
seen[col.Name] = true
|
||||
idx := len(m.FileColumns)
|
||||
m.FileColumns = append(m.FileColumns, col.Name)
|
||||
m.InsertColumns = append(m.InsertColumns, col.Name)
|
||||
m.InsertFieldIdx = append(m.InsertFieldIdx, idx)
|
||||
m.LoadList = append(m.LoadList, sqlutil.Ident(col.Name))
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func mappingPositional(tableName string, columns []schema.Column) (*Mapping, error) {
|
||||
m := &Mapping{Positional: true}
|
||||
for i, col := range columns {
|
||||
if isGenerated(col) {
|
||||
// The file still contains this field, so keep a placeholder and
|
||||
// discard the value; the target table recomputes it.
|
||||
m.FileColumns = append(m.FileColumns, "")
|
||||
m.LoadList = append(m.LoadList, fmt.Sprintf("@_archer_skip_%d", col.Ordinal))
|
||||
continue
|
||||
}
|
||||
m.FileColumns = append(m.FileColumns, col.Name)
|
||||
m.InsertColumns = append(m.InsertColumns, col.Name)
|
||||
m.InsertFieldIdx = append(m.InsertFieldIdx, i)
|
||||
m.LoadList = append(m.LoadList, sqlutil.Ident(col.Name))
|
||||
}
|
||||
if len(m.InsertColumns) == 0 {
|
||||
return nil, archererr.Newf(archererr.KindValidation,
|
||||
"target table %s has no writable columns", tableName)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func isGenerated(c schema.Column) bool {
|
||||
return strings.Contains(strings.ToLower(c.Extra), "generated")
|
||||
}
|
||||
|
||||
// CountFields counts dump fields in one raw line. Literal tabs are escaped by
|
||||
// the dump encoder, so every 0x09 byte is a separator. An empty line is a
|
||||
// single empty, non-NULL field.
|
||||
func CountFields(line []byte) int {
|
||||
return strings.Count(string(line), "\t") + 1
|
||||
}
|
||||
|
||||
// FirstLineFieldCount reads the first decompressed line of a shard.
|
||||
func FirstLineFieldCount(path string, comp *shard.Compressor) (fields int, hasRow bool, err error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
defer f.Close()
|
||||
r, err := comp.Reader(f)
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
defer r.Close()
|
||||
br := bufio.NewReaderSize(r, 64*1024)
|
||||
line, err := br.ReadBytes('\n')
|
||||
if err == io.EOF {
|
||||
if len(line) == 0 {
|
||||
return 0, false, nil
|
||||
}
|
||||
} else if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
line = bytes.TrimSuffix(line, []byte{'\n'})
|
||||
return CountFields(line), true, nil
|
||||
}
|
||||
|
||||
// CheckFieldCount compares the first row's field count with the load list
|
||||
// length. It is a sanity check only: it cannot prove that same-width columns
|
||||
// are in the right order.
|
||||
func CheckFieldCount(path string, comp *shard.Compressor, want int) error {
|
||||
got, hasRow, err := FirstLineFieldCount(path, comp)
|
||||
if err != nil {
|
||||
return archererr.Wrap(archererr.KindValidation, fmt.Errorf("read first row of %s: %w", path, err))
|
||||
}
|
||||
if !hasRow {
|
||||
return nil
|
||||
}
|
||||
if got != want {
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"first row of %s has %d fields, but the load column list has %d; refusing to load", path, got, want)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user