Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4a6b504f
commit
366d61ca25
49 files changed
+5099
-342
No files matched your search
@@ -0,0 +1,167 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/shard"
|
||||
"archer/internal/sqlutil"
|
||||
)
|
||||
|
||||
// BuildLoadSQL renders the LOAD DATA statement. Field and line rules match
|
||||
// internal/row.dumpEncoder exactly: TAB, LF, backslash escaping and \N.
|
||||
func BuildLoadSQL(dbName, table, charset string, m *Mapping, mode, readerName string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("LOAD DATA LOCAL INFILE '")
|
||||
b.WriteString(escapeSQLString("Reader::" + readerName))
|
||||
b.WriteString("'")
|
||||
switch mode {
|
||||
case "ignore":
|
||||
b.WriteString(" IGNORE")
|
||||
case "replace":
|
||||
b.WriteString(" REPLACE")
|
||||
}
|
||||
b.WriteString(" INTO TABLE ")
|
||||
b.WriteString(sqlutil.Qualified(dbName, table))
|
||||
b.WriteString(" CHARACTER SET ")
|
||||
b.WriteString(charset)
|
||||
b.WriteString(` FIELDS TERMINATED BY '\t' ESCAPED BY '\\' LINES TERMINATED BY '\n'`)
|
||||
if len(m.LoadList) > 0 {
|
||||
b.WriteString(" (")
|
||||
b.WriteString(strings.Join(m.LoadList, ", "))
|
||||
b.WriteString(")")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func escapeSQLString(s string) string {
|
||||
s = strings.ReplaceAll(s, `\`, `\\`)
|
||||
return strings.ReplaceAll(s, `'`, `''`)
|
||||
}
|
||||
|
||||
// CheckRowRule applies the per-mode affected-row rule and returns the number
|
||||
// of rows that were skipped (only meaningful for ignore mode).
|
||||
func CheckRowRule(mode string, expected, affected int64) (int64, error) {
|
||||
switch mode {
|
||||
case "insert":
|
||||
if affected != expected {
|
||||
return 0, archererr.Newf(archererr.KindValidation,
|
||||
"insert expected %d rows but wrote %d", expected, affected)
|
||||
}
|
||||
return 0, nil
|
||||
case "ignore":
|
||||
if affected > expected {
|
||||
return 0, archererr.Newf(archererr.KindValidation,
|
||||
"ignore wrote %d rows but the manifest records only %d", affected, expected)
|
||||
}
|
||||
return expected - affected, nil
|
||||
default: // replace may delete+insert, so affected can exceed rows
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
// VerifyShard streams the compressed shard through SHA-256 and compares it
|
||||
// with the manifest before any bytes are sent to the server.
|
||||
func VerifyShard(path string, e shard.Entry) error {
|
||||
if e.SHA256 == "" {
|
||||
return nil
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return archererr.Newf(archererr.KindValidation, "open shard %s: %v", e.Path, err)
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return archererr.Newf(archererr.KindValidation, "hash shard %s: %v", e.Path, err)
|
||||
}
|
||||
got := hex.EncodeToString(h.Sum(nil))
|
||||
if !strings.EqualFold(got, e.SHA256) {
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"shard %d (%s) failed verification: manifest sha256 %s, actual %s", e.Seq, e.Path, e.SHA256, got)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Warning is one SHOW WARNINGS row.
|
||||
type Warning struct {
|
||||
Level string
|
||||
Code int
|
||||
Message string
|
||||
}
|
||||
|
||||
func (w Warning) String() string { return fmt.Sprintf("%s %d: %s", w.Level, w.Code, w.Message) }
|
||||
|
||||
// ShowWarnings returns the current session warnings.
|
||||
func ShowWarnings(ctx context.Context, conn *sql.Conn) ([]Warning, error) {
|
||||
rows, err := conn.QueryContext(ctx, "SHOW WARNINGS")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Warning
|
||||
for rows.Next() {
|
||||
var w Warning
|
||||
if err := rows.Scan(&w.Level, &w.Code, &w.Message); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, w)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// CheckWarnings turns server warnings into a per-shard failure unless the
|
||||
// operator explicitly allowed them. Duplicate-key warnings under ignore mode
|
||||
// are expected skips (they are the mechanism that makes re-import idempotent),
|
||||
// so they are not failures; the skipped count comes from affected rows.
|
||||
func CheckWarnings(warnings []Warning, allow bool, mode string, seq int64) error {
|
||||
if allow || len(warnings) == 0 {
|
||||
return nil
|
||||
}
|
||||
fatal := warnings[:0:0]
|
||||
for _, w := range warnings {
|
||||
if mode == "ignore" && w.Code == 1062 {
|
||||
continue
|
||||
}
|
||||
fatal = append(fatal, w)
|
||||
}
|
||||
if len(fatal) == 0 {
|
||||
return nil
|
||||
}
|
||||
const maxShown = 5
|
||||
shown := fatal
|
||||
suffix := ""
|
||||
if len(shown) > maxShown {
|
||||
suffix = fmt.Sprintf(" (and %d more)", len(shown)-maxShown)
|
||||
shown = shown[:maxShown]
|
||||
}
|
||||
parts := make([]string, len(shown))
|
||||
for i, w := range shown {
|
||||
parts[i] = w.String()
|
||||
}
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"shard %d produced %d server warning(s); pass --allow-warnings to continue: %s%s",
|
||||
seq, len(fatal), strings.Join(parts, "; "), suffix)
|
||||
}
|
||||
|
||||
// SetSession applies the loader's character set and optional time zone to one
|
||||
// fixed connection.
|
||||
func SetSession(ctx context.Context, conn *sql.Conn, charset, timeZone string) error {
|
||||
if _, err := conn.ExecContext(ctx, "SET NAMES "+charset); err != nil {
|
||||
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("set character set %s: %w", charset, err))
|
||||
}
|
||||
if timeZone != "" {
|
||||
q := "SET time_zone = '" + escapeSQLString(timeZone) + "'"
|
||||
if _, err := conn.ExecContext(ctx, q); err != nil {
|
||||
return archererr.Wrap(archererr.KindConnection, fmt.Errorf("set time_zone %q: %w", timeZone, err))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user