Add archer import and allow date-mode bulk delete

archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-08 21:14:23 +08:00
1 parent bc4a6b504f
commit 366d61ca25
49 files changed
+5099 -342

No files matched your search

+167
View File
@@ -0,0 +1,167 @@
package importer
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"fmt"
"io"
"os"
"strings"
"archer/internal/archererr"
"archer/internal/shard"
"archer/internal/sqlutil"
)
// BuildLoadSQL renders the LOAD DATA statement. Field and line rules match
// internal/row.dumpEncoder exactly: TAB, LF, backslash escaping and \N.
func BuildLoadSQL(dbName, table, charset string, m *Mapping, mode, readerName string) string {
var b strings.Builder
b.WriteString("LOAD DATA LOCAL INFILE '")
b.WriteString(escapeSQLString("Reader::" + readerName))
b.WriteString("'")
switch mode {
case "ignore":
b.WriteString(" IGNORE")
case "replace":
b.WriteString(" REPLACE")
}
b.WriteString(" INTO TABLE ")
b.WriteString(sqlutil.Qualified(dbName, table))
b.WriteString(" CHARACTER SET ")
b.WriteString(charset)
b.WriteString(` FIELDS TERMINATED BY '\t' ESCAPED BY '\\' LINES TERMINATED BY '\n'`)
if len(m.LoadList) > 0 {
b.WriteString(" (")
b.WriteString(strings.Join(m.LoadList, ", "))
b.WriteString(")")
}
return b.String()
}
func escapeSQLString(s string) string {
s = strings.ReplaceAll(s, `\`, `\\`)
return strings.ReplaceAll(s, `'`, `''`)
}
// CheckRowRule applies the per-mode affected-row rule and returns the number
// of rows that were skipped (only meaningful for ignore mode).
func CheckRowRule(mode string, expected, affected int64) (int64, error) {
switch mode {
case "insert":
if affected != expected {
return 0, archererr.Newf(archererr.KindValidation,
"insert expected %d rows but wrote %d", expected, affected)
}
return 0, nil
case "ignore":
if affected > expected {
return 0, archererr.Newf(archererr.KindValidation,
"ignore wrote %d rows but the manifest records only %d", affected, expected)
}
return expected - affected, nil
default: // replace may delete+insert, so affected can exceed rows
return 0, nil
}
}
// VerifyShard streams the compressed shard through SHA-256 and compares it
// with the manifest before any bytes are sent to the server.
func VerifyShard(path string, e shard.Entry) error {
if e.SHA256 == "" {
return nil
}
f, err := os.Open(path)
if err != nil {
return archererr.Newf(archererr.KindValidation, "open shard %s: %v", e.Path, err)
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return archererr.Newf(archererr.KindValidation, "hash shard %s: %v", e.Path, err)
}
got := hex.EncodeToString(h.Sum(nil))
if !strings.EqualFold(got, e.SHA256) {
return archererr.Newf(archererr.KindValidation,
"shard %d (%s) failed verification: manifest sha256 %s, actual %s", e.Seq, e.Path, e.SHA256, got)
}
return nil
}
// Warning is one SHOW WARNINGS row.
type Warning struct {
Level string
Code int
Message string
}
func (w Warning) String() string { return fmt.Sprintf("%s %d: %s", w.Level, w.Code, w.Message) }
// ShowWarnings returns the current session warnings.
func ShowWarnings(ctx context.Context, conn *sql.Conn) ([]Warning, error) {
rows, err := conn.QueryContext(ctx, "SHOW WARNINGS")
if err != nil {
return nil, err
}
defer rows.Close()
var out []Warning
for rows.Next() {
var w Warning
if err := rows.Scan(&w.Level, &w.Code, &w.Message); err != nil {
return nil, err
}
out = append(out, w)
}
return out, rows.Err()
}
// CheckWarnings turns server warnings into a per-shard failure unless the
// operator explicitly allowed them. Duplicate-key warnings under ignore mode
// are expected skips (they are the mechanism that makes re-import idempotent),
// so they are not failures; the skipped count comes from affected rows.
func CheckWarnings(warnings []Warning, allow bool, mode string, seq int64) error {
if allow || len(warnings) == 0 {
return nil
}
fatal := warnings[:0:0]
for _, w := range warnings {
if mode == "ignore" && w.Code == 1062 {
continue
}
fatal = append(fatal, w)
}
if len(fatal) == 0 {
return nil
}
const maxShown = 5
shown := fatal
suffix := ""
if len(shown) > maxShown {
suffix = fmt.Sprintf(" (and %d more)", len(shown)-maxShown)
shown = shown[:maxShown]
}
parts := make([]string, len(shown))
for i, w := range shown {
parts[i] = w.String()
}
return archererr.Newf(archererr.KindValidation,
"shard %d produced %d server warning(s); pass --allow-warnings to continue: %s%s",
seq, len(fatal), strings.Join(parts, "; "), suffix)
}
// SetSession applies the loader's character set and optional time zone to one
// fixed connection.
func SetSession(ctx context.Context, conn *sql.Conn, charset, timeZone string) error {
if _, err := conn.ExecContext(ctx, "SET NAMES "+charset); err != nil {
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("set character set %s: %w", charset, err))
}
if timeZone != "" {
q := "SET time_zone = '" + escapeSQLString(timeZone) + "'"
if _, err := conn.ExecContext(ctx, q); err != nil {
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("set time_zone %q: %w", timeZone, err))
}
}
return nil
}