archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
164 lines
5.4 KiB
Go
164 lines
5.4 KiB
Go
package importer
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
|
|
"archer/internal/archererr"
|
|
"archer/internal/schema"
|
|
"archer/internal/shard"
|
|
"archer/internal/sqlutil"
|
|
)
|
|
|
|
// Mapping translates file fields to target columns. It is the single place
|
|
// that decides whether a field is loaded into a named column or discarded into
|
|
// a user variable (generated columns under positional mapping).
|
|
type Mapping struct {
|
|
// FileColumns has one entry per file field: the canonical target column
|
|
// name, or "" when the field is intentionally discarded.
|
|
FileColumns []string
|
|
// InsertColumns and InsertFieldIdx describe the parameterized INSERT used
|
|
// by the insert engine. InsertFieldIdx points into FileColumns.
|
|
InsertColumns []string
|
|
InsertFieldIdx []int
|
|
// LoadList is the LOAD DATA column list: quoted identifiers and
|
|
// @_archer_skip_N variables.
|
|
LoadList []string
|
|
// Positional is true for --assume-all-columns; callers must warn.
|
|
Positional bool
|
|
}
|
|
|
|
// BuildMapping resolves the three permitted mapping sources. Columns recorded
|
|
// in the archive win over an explicit --columns by construction (the conflict
|
|
// is rejected earlier); assumeAll is only consulted when neither list exists.
|
|
func BuildMapping(table *schema.Table, archived, explicit []string, assumeAll bool) (*Mapping, error) {
|
|
return buildMapping(table.Name, table.Columns, archived, explicit, assumeAll)
|
|
}
|
|
|
|
func buildMapping(tableName string, columns []schema.Column, archived, explicit []string, assumeAll bool) (*Mapping, error) {
|
|
names := archived
|
|
if len(names) == 0 {
|
|
names = explicit
|
|
}
|
|
switch {
|
|
case len(names) > 0:
|
|
return mappingByName(tableName, columns, names)
|
|
case assumeAll:
|
|
return mappingPositional(tableName, columns)
|
|
default:
|
|
return nil, archererr.Newf(archererr.KindConfig,
|
|
"the archive did not record a column list; provide --columns in file field order or accept positional mapping with --assume-all-columns")
|
|
}
|
|
}
|
|
|
|
func mappingByName(tableName string, columns []schema.Column, names []string) (*Mapping, error) {
|
|
byName := make(map[string]schema.Column, len(columns))
|
|
for _, c := range columns {
|
|
byName[strings.ToLower(c.Name)] = c
|
|
}
|
|
m := &Mapping{}
|
|
seen := map[string]bool{}
|
|
for _, requested := range names {
|
|
col, ok := byName[strings.ToLower(requested)]
|
|
if !ok {
|
|
return nil, archererr.Newf(archererr.KindValidation,
|
|
"column %q from the file field mapping does not exist in target table %s", requested, tableName)
|
|
}
|
|
if isGenerated(col) {
|
|
return nil, archererr.Newf(archererr.KindValidation,
|
|
"column %q is a generated column and cannot be written from an archive", col.Name)
|
|
}
|
|
if seen[col.Name] {
|
|
return nil, archererr.Newf(archererr.KindValidation,
|
|
"column %q appears more than once in the file field mapping", col.Name)
|
|
}
|
|
seen[col.Name] = true
|
|
idx := len(m.FileColumns)
|
|
m.FileColumns = append(m.FileColumns, col.Name)
|
|
m.InsertColumns = append(m.InsertColumns, col.Name)
|
|
m.InsertFieldIdx = append(m.InsertFieldIdx, idx)
|
|
m.LoadList = append(m.LoadList, sqlutil.Ident(col.Name))
|
|
}
|
|
return m, nil
|
|
}
|
|
|
|
func mappingPositional(tableName string, columns []schema.Column) (*Mapping, error) {
|
|
m := &Mapping{Positional: true}
|
|
for i, col := range columns {
|
|
if isGenerated(col) {
|
|
// The file still contains this field, so keep a placeholder and
|
|
// discard the value; the target table recomputes it.
|
|
m.FileColumns = append(m.FileColumns, "")
|
|
m.LoadList = append(m.LoadList, fmt.Sprintf("@_archer_skip_%d", col.Ordinal))
|
|
continue
|
|
}
|
|
m.FileColumns = append(m.FileColumns, col.Name)
|
|
m.InsertColumns = append(m.InsertColumns, col.Name)
|
|
m.InsertFieldIdx = append(m.InsertFieldIdx, i)
|
|
m.LoadList = append(m.LoadList, sqlutil.Ident(col.Name))
|
|
}
|
|
if len(m.InsertColumns) == 0 {
|
|
return nil, archererr.Newf(archererr.KindValidation,
|
|
"target table %s has no writable columns", tableName)
|
|
}
|
|
return m, nil
|
|
}
|
|
|
|
func isGenerated(c schema.Column) bool {
|
|
return strings.Contains(strings.ToLower(c.Extra), "generated")
|
|
}
|
|
|
|
// CountFields counts dump fields in one raw line. Literal tabs are escaped by
|
|
// the dump encoder, so every 0x09 byte is a separator. An empty line is a
|
|
// single empty, non-NULL field.
|
|
func CountFields(line []byte) int {
|
|
return strings.Count(string(line), "\t") + 1
|
|
}
|
|
|
|
// FirstLineFieldCount reads the first decompressed line of a shard.
|
|
func FirstLineFieldCount(path string, comp *shard.Compressor) (fields int, hasRow bool, err error) {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return 0, false, err
|
|
}
|
|
defer f.Close()
|
|
r, err := comp.Reader(f)
|
|
if err != nil {
|
|
return 0, false, err
|
|
}
|
|
defer r.Close()
|
|
br := bufio.NewReaderSize(r, 64*1024)
|
|
line, err := br.ReadBytes('\n')
|
|
if err == io.EOF {
|
|
if len(line) == 0 {
|
|
return 0, false, nil
|
|
}
|
|
} else if err != nil {
|
|
return 0, false, err
|
|
}
|
|
line = bytes.TrimSuffix(line, []byte{'\n'})
|
|
return CountFields(line), true, nil
|
|
}
|
|
|
|
// CheckFieldCount compares the first row's field count with the load list
|
|
// length. It is a sanity check only: it cannot prove that same-width columns
|
|
// are in the right order.
|
|
func CheckFieldCount(path string, comp *shard.Compressor, want int) error {
|
|
got, hasRow, err := FirstLineFieldCount(path, comp)
|
|
if err != nil {
|
|
return archererr.Wrap(archererr.KindValidation, fmt.Errorf("read first row of %s: %w", path, err))
|
|
}
|
|
if !hasRow {
|
|
return nil
|
|
}
|
|
if got != want {
|
|
return archererr.Newf(archererr.KindValidation,
|
|
"first row of %s has %d fields, but the load column list has %d; refusing to load", path, got, want)
|
|
}
|
|
return nil
|
|
}
|