Files
archer/internal/importer/archive.go
T
voidandClaude Sonnet 4.6 366d61ca25 Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard
selection (`--select`), SHA-256 pre-verification, idempotent modes
(`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT
engines. It reads manifest.jsonl strictly so corruption is reported
instead of skipped, and supports --dry-run, --allow-partial, and
per-shard workers.

--bulk-delete no longer rejects --shard-by date. Range deletion now
derives its bounds from the chunk's actual keys (date scanners rewrite
LastKey into a resumption cursor) and keeps safe-auto-increment
semantics by excluding the current max key. A startup warning documents
the monotonic-shard-column assumption.

Also splits CLI/format/sharding/progress/safety reference docs out of
the README, and adds the corresponding OpenSpec change artifacts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-11-08 21:14:23 +08:00

215 lines
7.0 KiB
Go

// Package importer replays a dump archive into an existing MySQL table.
//
// The archive's manifest.jsonl is the only authority: files on disk that are
// not registered are never imported, and unparseable manifest lines abort the
// run before a database connection is opened. Each shard is a complete,
// independently loadable file, so shards are the unit of selection, atomicity
// and failure reporting.
package importer
import (
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"archer/internal/archererr"
"archer/internal/checkpoint"
"archer/internal/shard"
)
// Archive is a loaded, validated-by-reading view of an export directory.
type Archive struct {
Root string
Entries []shard.Entry
State *checkpoint.State
Complete bool
SourceTable string
ArchivedColumns []string
Format string
Compress string
Compressor *shard.Compressor
}
// LoadArchive reads manifest.jsonl strictly and state.json when present. It
// never writes to the archive directory.
func LoadArchive(root string) (*Archive, error) {
st, err := os.Stat(root)
if err != nil {
return nil, archererr.Newf(archererr.KindConfig, "--file %s: %v", root, err)
}
if !st.IsDir() {
return nil, archererr.Newf(archererr.KindConfig, "--file %s is not a directory", root)
}
entries, err := shard.LoadManifest(root)
if err != nil {
return nil, archererr.Wrap(archererr.KindValidation, err)
}
state, err := checkpoint.Load(root)
if err != nil {
return nil, archererr.Wrap(archererr.KindValidation, err)
}
a := &Archive{Root: root, Entries: entries, State: state}
if state != nil {
a.Complete = state.Complete
a.SourceTable = strings.TrimSpace(state.Config["table"])
if cols := strings.TrimSpace(state.Config["columns"]); cols != "" {
for _, part := range strings.Split(cols, ",") {
if part = strings.TrimSpace(part); part != "" {
a.ArchivedColumns = append(a.ArchivedColumns, part)
}
}
}
}
return a, nil
}
// CheckComplete enforces the completion gate. A manifest only proves which
// shards are sealed, not that the export covered the whole table, so an
// incomplete archive is refused unless the operator opts in explicitly.
func (a *Archive) CheckComplete(allowPartial bool) error {
if allowPartial {
return nil
}
if a.State == nil {
return archererr.Newf(archererr.KindValidation,
"archive %s has no state.json; cannot confirm the export finished (pass --allow-partial to import its sealed shards)", a.Root)
}
if !a.State.Complete {
return archererr.Newf(archererr.KindValidation,
"archive %s is incomplete; it may contain only part of the table (pass --allow-partial to import its sealed shards)", a.Root)
}
return nil
}
// CheckMappingChoice rejects an explicit mapping when the archive already
// recorded its own column list: two sources of truth would silently disagree.
func (a *Archive) CheckMappingChoice(columns []string, assumeAll bool) error {
if len(a.ArchivedColumns) == 0 {
return nil
}
if len(columns) > 0 {
return archererr.Newf(archererr.KindConfig,
"archive records its own column list (%s); --columns cannot be combined with it", strings.Join(a.ArchivedColumns, ", "))
}
if assumeAll {
return archererr.Newf(archererr.KindConfig,
"archive records its own column list (%s); --assume-all-columns cannot be combined with it", strings.Join(a.ArchivedColumns, ", "))
}
return nil
}
var shardFileRE = regexp.MustCompile(`^\d+\..*\.(dump|csv|jsonl)(\.gz|\.zst|\.xz)?$`)
// UnregisteredFiles counts shard-looking files that the manifest does not
// list. They are ignored by design; the count is surfaced in --statistics so
// they are not mistaken for imported data.
func UnregisteredFiles(root string, entries []shard.Entry) int {
registered := make(map[string]bool, len(entries))
for _, e := range entries {
registered[filepath.ToSlash(e.Path)] = true
}
n := 0
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil || d.IsDir() {
return nil
}
name := d.Name()
if name == "manifest.jsonl" || name == "state.json" || strings.HasSuffix(name, ".partial") ||
strings.HasSuffix(name, ".partial.meta") || !shardFileRE.MatchString(name) {
return nil
}
rel, err := filepath.Rel(root, path)
if err != nil {
return nil
}
if !registered[filepath.ToSlash(rel)] {
n++
}
return nil
})
return n
}
// ShardFormat derives the output format and compression from a manifest path
// such as 000003.100-200.dump.gz.
func ShardFormat(p string) (format, compress string, err error) {
name := filepath.Base(p)
switch {
case strings.HasSuffix(name, ".gz"):
compress, name = "gzip", strings.TrimSuffix(name, ".gz")
case strings.HasSuffix(name, ".zst"):
compress, name = "zstd", strings.TrimSuffix(name, ".zst")
case strings.HasSuffix(name, ".xz"):
compress, name = "xz", strings.TrimSuffix(name, ".xz")
default:
compress = "none"
}
ext := strings.ToLower(filepath.Ext(name))
if ext == "" {
return "", "", fmt.Errorf("shard %s has no format extension", p)
}
format = strings.TrimPrefix(ext, ".")
switch format {
case "dump", "csv", "jsonl":
return format, compress, nil
default:
return "", "", fmt.Errorf("shard %s has unknown format %q", p, format)
}
}
// ResolveFormat checks that every selected shard agrees on format and
// compression, and that the format is dump (the only one v1 can replay).
func ResolveFormat(entries []shard.Entry) (format, compress string, comp *shard.Compressor, err error) {
for i, e := range entries {
f, c, ferr := ShardFormat(e.Path)
if ferr != nil {
return "", "", nil, archererr.Wrap(archererr.KindValidation, ferr)
}
if i == 0 {
format, compress = f, c
continue
}
if f != format {
return "", "", nil, archererr.Newf(archererr.KindValidation,
"selected shards mix formats: %s is %s but %s is %s", entries[0].Path, format, e.Path, f)
}
if c != compress {
return "", "", nil, archererr.Newf(archererr.KindValidation,
"selected shards mix compression: %s is %s but %s is %s", entries[0].Path, compress, e.Path, c)
}
}
if format != "dump" {
return "", "", nil, archererr.Newf(archererr.KindValidation,
"import supports only dump archives; the selected shards are %s", format)
}
comp, err = shard.NewCompressor(compress, shard.DefaultLevel)
if err != nil {
return "", "", nil, archererr.Wrap(archererr.KindValidation, err)
}
return format, compress, comp, nil
}
// CheckShardFiles verifies every selected manifest entry has a readable file
// before any connection is opened.
func CheckShardFiles(root string, entries []shard.Entry) error {
for _, e := range entries {
path := filepath.Join(root, filepath.FromSlash(e.Path))
st, err := os.Stat(path)
if err != nil {
return archererr.Newf(archererr.KindValidation,
"manifest entry seq %d points to missing shard %s: %v", e.Seq, e.Path, err)
}
if st.IsDir() {
return archererr.Newf(archererr.KindValidation,
"manifest entry seq %d points to a directory, not a shard: %s", e.Seq, e.Path)
}
}
return nil
}
func shardPath(root string, e shard.Entry) string {
return filepath.Join(root, filepath.FromSlash(e.Path))
}