archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
215 lines
7.0 KiB
Go
215 lines
7.0 KiB
Go
// Package importer replays a dump archive into an existing MySQL table.
|
|
//
|
|
// The archive's manifest.jsonl is the only authority: files on disk that are
|
|
// not registered are never imported, and unparseable manifest lines abort the
|
|
// run before a database connection is opened. Each shard is a complete,
|
|
// independently loadable file, so shards are the unit of selection, atomicity
|
|
// and failure reporting.
|
|
package importer
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
|
|
"archer/internal/archererr"
|
|
"archer/internal/checkpoint"
|
|
"archer/internal/shard"
|
|
)
|
|
|
|
// Archive is a loaded, validated-by-reading view of an export directory.
|
|
type Archive struct {
|
|
Root string
|
|
Entries []shard.Entry
|
|
State *checkpoint.State
|
|
Complete bool
|
|
SourceTable string
|
|
ArchivedColumns []string
|
|
Format string
|
|
Compress string
|
|
Compressor *shard.Compressor
|
|
}
|
|
|
|
// LoadArchive reads manifest.jsonl strictly and state.json when present. It
|
|
// never writes to the archive directory.
|
|
func LoadArchive(root string) (*Archive, error) {
|
|
st, err := os.Stat(root)
|
|
if err != nil {
|
|
return nil, archererr.Newf(archererr.KindConfig, "--file %s: %v", root, err)
|
|
}
|
|
if !st.IsDir() {
|
|
return nil, archererr.Newf(archererr.KindConfig, "--file %s is not a directory", root)
|
|
}
|
|
entries, err := shard.LoadManifest(root)
|
|
if err != nil {
|
|
return nil, archererr.Wrap(archererr.KindValidation, err)
|
|
}
|
|
state, err := checkpoint.Load(root)
|
|
if err != nil {
|
|
return nil, archererr.Wrap(archererr.KindValidation, err)
|
|
}
|
|
a := &Archive{Root: root, Entries: entries, State: state}
|
|
if state != nil {
|
|
a.Complete = state.Complete
|
|
a.SourceTable = strings.TrimSpace(state.Config["table"])
|
|
if cols := strings.TrimSpace(state.Config["columns"]); cols != "" {
|
|
for _, part := range strings.Split(cols, ",") {
|
|
if part = strings.TrimSpace(part); part != "" {
|
|
a.ArchivedColumns = append(a.ArchivedColumns, part)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return a, nil
|
|
}
|
|
|
|
// CheckComplete enforces the completion gate. A manifest only proves which
|
|
// shards are sealed, not that the export covered the whole table, so an
|
|
// incomplete archive is refused unless the operator opts in explicitly.
|
|
func (a *Archive) CheckComplete(allowPartial bool) error {
|
|
if allowPartial {
|
|
return nil
|
|
}
|
|
if a.State == nil {
|
|
return archererr.Newf(archererr.KindValidation,
|
|
"archive %s has no state.json; cannot confirm the export finished (pass --allow-partial to import its sealed shards)", a.Root)
|
|
}
|
|
if !a.State.Complete {
|
|
return archererr.Newf(archererr.KindValidation,
|
|
"archive %s is incomplete; it may contain only part of the table (pass --allow-partial to import its sealed shards)", a.Root)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// CheckMappingChoice rejects an explicit mapping when the archive already
|
|
// recorded its own column list: two sources of truth would silently disagree.
|
|
func (a *Archive) CheckMappingChoice(columns []string, assumeAll bool) error {
|
|
if len(a.ArchivedColumns) == 0 {
|
|
return nil
|
|
}
|
|
if len(columns) > 0 {
|
|
return archererr.Newf(archererr.KindConfig,
|
|
"archive records its own column list (%s); --columns cannot be combined with it", strings.Join(a.ArchivedColumns, ", "))
|
|
}
|
|
if assumeAll {
|
|
return archererr.Newf(archererr.KindConfig,
|
|
"archive records its own column list (%s); --assume-all-columns cannot be combined with it", strings.Join(a.ArchivedColumns, ", "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
var shardFileRE = regexp.MustCompile(`^\d+\..*\.(dump|csv|jsonl)(\.gz|\.zst|\.xz)?$`)
|
|
|
|
// UnregisteredFiles counts shard-looking files that the manifest does not
|
|
// list. They are ignored by design; the count is surfaced in --statistics so
|
|
// they are not mistaken for imported data.
|
|
func UnregisteredFiles(root string, entries []shard.Entry) int {
|
|
registered := make(map[string]bool, len(entries))
|
|
for _, e := range entries {
|
|
registered[filepath.ToSlash(e.Path)] = true
|
|
}
|
|
n := 0
|
|
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
|
if err != nil || d.IsDir() {
|
|
return nil
|
|
}
|
|
name := d.Name()
|
|
if name == "manifest.jsonl" || name == "state.json" || strings.HasSuffix(name, ".partial") ||
|
|
strings.HasSuffix(name, ".partial.meta") || !shardFileRE.MatchString(name) {
|
|
return nil
|
|
}
|
|
rel, err := filepath.Rel(root, path)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
if !registered[filepath.ToSlash(rel)] {
|
|
n++
|
|
}
|
|
return nil
|
|
})
|
|
return n
|
|
}
|
|
|
|
// ShardFormat derives the output format and compression from a manifest path
|
|
// such as 000003.100-200.dump.gz.
|
|
func ShardFormat(p string) (format, compress string, err error) {
|
|
name := filepath.Base(p)
|
|
switch {
|
|
case strings.HasSuffix(name, ".gz"):
|
|
compress, name = "gzip", strings.TrimSuffix(name, ".gz")
|
|
case strings.HasSuffix(name, ".zst"):
|
|
compress, name = "zstd", strings.TrimSuffix(name, ".zst")
|
|
case strings.HasSuffix(name, ".xz"):
|
|
compress, name = "xz", strings.TrimSuffix(name, ".xz")
|
|
default:
|
|
compress = "none"
|
|
}
|
|
ext := strings.ToLower(filepath.Ext(name))
|
|
if ext == "" {
|
|
return "", "", fmt.Errorf("shard %s has no format extension", p)
|
|
}
|
|
format = strings.TrimPrefix(ext, ".")
|
|
switch format {
|
|
case "dump", "csv", "jsonl":
|
|
return format, compress, nil
|
|
default:
|
|
return "", "", fmt.Errorf("shard %s has unknown format %q", p, format)
|
|
}
|
|
}
|
|
|
|
// ResolveFormat checks that every selected shard agrees on format and
|
|
// compression, and that the format is dump (the only one v1 can replay).
|
|
func ResolveFormat(entries []shard.Entry) (format, compress string, comp *shard.Compressor, err error) {
|
|
for i, e := range entries {
|
|
f, c, ferr := ShardFormat(e.Path)
|
|
if ferr != nil {
|
|
return "", "", nil, archererr.Wrap(archererr.KindValidation, ferr)
|
|
}
|
|
if i == 0 {
|
|
format, compress = f, c
|
|
continue
|
|
}
|
|
if f != format {
|
|
return "", "", nil, archererr.Newf(archererr.KindValidation,
|
|
"selected shards mix formats: %s is %s but %s is %s", entries[0].Path, format, e.Path, f)
|
|
}
|
|
if c != compress {
|
|
return "", "", nil, archererr.Newf(archererr.KindValidation,
|
|
"selected shards mix compression: %s is %s but %s is %s", entries[0].Path, compress, e.Path, c)
|
|
}
|
|
}
|
|
if format != "dump" {
|
|
return "", "", nil, archererr.Newf(archererr.KindValidation,
|
|
"import supports only dump archives; the selected shards are %s", format)
|
|
}
|
|
comp, err = shard.NewCompressor(compress, shard.DefaultLevel)
|
|
if err != nil {
|
|
return "", "", nil, archererr.Wrap(archererr.KindValidation, err)
|
|
}
|
|
return format, compress, comp, nil
|
|
}
|
|
|
|
// CheckShardFiles verifies every selected manifest entry has a readable file
|
|
// before any connection is opened.
|
|
func CheckShardFiles(root string, entries []shard.Entry) error {
|
|
for _, e := range entries {
|
|
path := filepath.Join(root, filepath.FromSlash(e.Path))
|
|
st, err := os.Stat(path)
|
|
if err != nil {
|
|
return archererr.Newf(archererr.KindValidation,
|
|
"manifest entry seq %d points to missing shard %s: %v", e.Seq, e.Path, err)
|
|
}
|
|
if st.IsDir() {
|
|
return archererr.Newf(archererr.KindValidation,
|
|
"manifest entry seq %d points to a directory, not a shard: %s", e.Seq, e.Path)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func shardPath(root string, e shard.Entry) string {
|
|
return filepath.Join(root, filepath.FromSlash(e.Path))
|
|
}
|