Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4a6b504f
commit
366d61ca25
49 files changed
+5099
-342
No files matched your search
@@ -0,0 +1,214 @@
|
||||
// Package importer replays a dump archive into an existing MySQL table.
|
||||
//
|
||||
// The archive's manifest.jsonl is the only authority: files on disk that are
|
||||
// not registered are never imported, and unparseable manifest lines abort the
|
||||
// run before a database connection is opened. Each shard is a complete,
|
||||
// independently loadable file, so shards are the unit of selection, atomicity
|
||||
// and failure reporting.
|
||||
package importer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"archer/internal/archererr"
|
||||
"archer/internal/checkpoint"
|
||||
"archer/internal/shard"
|
||||
)
|
||||
|
||||
// Archive is a loaded, validated-by-reading view of an export directory.
|
||||
type Archive struct {
|
||||
Root string
|
||||
Entries []shard.Entry
|
||||
State *checkpoint.State
|
||||
Complete bool
|
||||
SourceTable string
|
||||
ArchivedColumns []string
|
||||
Format string
|
||||
Compress string
|
||||
Compressor *shard.Compressor
|
||||
}
|
||||
|
||||
// LoadArchive reads manifest.jsonl strictly and state.json when present. It
|
||||
// never writes to the archive directory.
|
||||
func LoadArchive(root string) (*Archive, error) {
|
||||
st, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return nil, archererr.Newf(archererr.KindConfig, "--file %s: %v", root, err)
|
||||
}
|
||||
if !st.IsDir() {
|
||||
return nil, archererr.Newf(archererr.KindConfig, "--file %s is not a directory", root)
|
||||
}
|
||||
entries, err := shard.LoadManifest(root)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
state, err := checkpoint.Load(root)
|
||||
if err != nil {
|
||||
return nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
a := &Archive{Root: root, Entries: entries, State: state}
|
||||
if state != nil {
|
||||
a.Complete = state.Complete
|
||||
a.SourceTable = strings.TrimSpace(state.Config["table"])
|
||||
if cols := strings.TrimSpace(state.Config["columns"]); cols != "" {
|
||||
for _, part := range strings.Split(cols, ",") {
|
||||
if part = strings.TrimSpace(part); part != "" {
|
||||
a.ArchivedColumns = append(a.ArchivedColumns, part)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// CheckComplete enforces the completion gate. A manifest only proves which
|
||||
// shards are sealed, not that the export covered the whole table, so an
|
||||
// incomplete archive is refused unless the operator opts in explicitly.
|
||||
func (a *Archive) CheckComplete(allowPartial bool) error {
|
||||
if allowPartial {
|
||||
return nil
|
||||
}
|
||||
if a.State == nil {
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"archive %s has no state.json; cannot confirm the export finished (pass --allow-partial to import its sealed shards)", a.Root)
|
||||
}
|
||||
if !a.State.Complete {
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"archive %s is incomplete; it may contain only part of the table (pass --allow-partial to import its sealed shards)", a.Root)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CheckMappingChoice rejects an explicit mapping when the archive already
|
||||
// recorded its own column list: two sources of truth would silently disagree.
|
||||
func (a *Archive) CheckMappingChoice(columns []string, assumeAll bool) error {
|
||||
if len(a.ArchivedColumns) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(columns) > 0 {
|
||||
return archererr.Newf(archererr.KindConfig,
|
||||
"archive records its own column list (%s); --columns cannot be combined with it", strings.Join(a.ArchivedColumns, ", "))
|
||||
}
|
||||
if assumeAll {
|
||||
return archererr.Newf(archererr.KindConfig,
|
||||
"archive records its own column list (%s); --assume-all-columns cannot be combined with it", strings.Join(a.ArchivedColumns, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var shardFileRE = regexp.MustCompile(`^\d+\..*\.(dump|csv|jsonl)(\.gz|\.zst|\.xz)?$`)
|
||||
|
||||
// UnregisteredFiles counts shard-looking files that the manifest does not
|
||||
// list. They are ignored by design; the count is surfaced in --statistics so
|
||||
// they are not mistaken for imported data.
|
||||
func UnregisteredFiles(root string, entries []shard.Entry) int {
|
||||
registered := make(map[string]bool, len(entries))
|
||||
for _, e := range entries {
|
||||
registered[filepath.ToSlash(e.Path)] = true
|
||||
}
|
||||
n := 0
|
||||
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
if name == "manifest.jsonl" || name == "state.json" || strings.HasSuffix(name, ".partial") ||
|
||||
strings.HasSuffix(name, ".partial.meta") || !shardFileRE.MatchString(name) {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if !registered[filepath.ToSlash(rel)] {
|
||||
n++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return n
|
||||
}
|
||||
|
||||
// ShardFormat derives the output format and compression from a manifest path
|
||||
// such as 000003.100-200.dump.gz.
|
||||
func ShardFormat(p string) (format, compress string, err error) {
|
||||
name := filepath.Base(p)
|
||||
switch {
|
||||
case strings.HasSuffix(name, ".gz"):
|
||||
compress, name = "gzip", strings.TrimSuffix(name, ".gz")
|
||||
case strings.HasSuffix(name, ".zst"):
|
||||
compress, name = "zstd", strings.TrimSuffix(name, ".zst")
|
||||
case strings.HasSuffix(name, ".xz"):
|
||||
compress, name = "xz", strings.TrimSuffix(name, ".xz")
|
||||
default:
|
||||
compress = "none"
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(name))
|
||||
if ext == "" {
|
||||
return "", "", fmt.Errorf("shard %s has no format extension", p)
|
||||
}
|
||||
format = strings.TrimPrefix(ext, ".")
|
||||
switch format {
|
||||
case "dump", "csv", "jsonl":
|
||||
return format, compress, nil
|
||||
default:
|
||||
return "", "", fmt.Errorf("shard %s has unknown format %q", p, format)
|
||||
}
|
||||
}
|
||||
|
||||
// ResolveFormat checks that every selected shard agrees on format and
|
||||
// compression, and that the format is dump (the only one v1 can replay).
|
||||
func ResolveFormat(entries []shard.Entry) (format, compress string, comp *shard.Compressor, err error) {
|
||||
for i, e := range entries {
|
||||
f, c, ferr := ShardFormat(e.Path)
|
||||
if ferr != nil {
|
||||
return "", "", nil, archererr.Wrap(archererr.KindValidation, ferr)
|
||||
}
|
||||
if i == 0 {
|
||||
format, compress = f, c
|
||||
continue
|
||||
}
|
||||
if f != format {
|
||||
return "", "", nil, archererr.Newf(archererr.KindValidation,
|
||||
"selected shards mix formats: %s is %s but %s is %s", entries[0].Path, format, e.Path, f)
|
||||
}
|
||||
if c != compress {
|
||||
return "", "", nil, archererr.Newf(archererr.KindValidation,
|
||||
"selected shards mix compression: %s is %s but %s is %s", entries[0].Path, compress, e.Path, c)
|
||||
}
|
||||
}
|
||||
if format != "dump" {
|
||||
return "", "", nil, archererr.Newf(archererr.KindValidation,
|
||||
"import supports only dump archives; the selected shards are %s", format)
|
||||
}
|
||||
comp, err = shard.NewCompressor(compress, shard.DefaultLevel)
|
||||
if err != nil {
|
||||
return "", "", nil, archererr.Wrap(archererr.KindValidation, err)
|
||||
}
|
||||
return format, compress, comp, nil
|
||||
}
|
||||
|
||||
// CheckShardFiles verifies every selected manifest entry has a readable file
|
||||
// before any connection is opened.
|
||||
func CheckShardFiles(root string, entries []shard.Entry) error {
|
||||
for _, e := range entries {
|
||||
path := filepath.Join(root, filepath.FromSlash(e.Path))
|
||||
st, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"manifest entry seq %d points to missing shard %s: %v", e.Seq, e.Path, err)
|
||||
}
|
||||
if st.IsDir() {
|
||||
return archererr.Newf(archererr.KindValidation,
|
||||
"manifest entry seq %d points to a directory, not a shard: %s", e.Seq, e.Path)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func shardPath(root string, e shard.Entry) string {
|
||||
return filepath.Join(root, filepath.FromSlash(e.Path))
|
||||
}
|
||||
Reference in new issue
Block a user