// Package importer replays a dump archive into an existing MySQL table. // // The archive's manifest.jsonl is the only authority: files on disk that are // not registered are never imported, and unparseable manifest lines abort the // run before a database connection is opened. Each shard is a complete, // independently loadable file, so shards are the unit of selection, atomicity // and failure reporting. package importer import ( "fmt" "os" "path/filepath" "regexp" "strings" "archer/internal/archererr" "archer/internal/checkpoint" "archer/internal/shard" ) // Archive is a loaded, validated-by-reading view of an export directory. type Archive struct { Root string Entries []shard.Entry State *checkpoint.State Complete bool SourceTable string ArchivedColumns []string Format string Compress string Compressor *shard.Compressor } // LoadArchive reads manifest.jsonl strictly and state.json when present. It // never writes to the archive directory. func LoadArchive(root string) (*Archive, error) { st, err := os.Stat(root) if err != nil { return nil, archererr.Newf(archererr.KindConfig, "--file %s: %v", root, err) } if !st.IsDir() { return nil, archererr.Newf(archererr.KindConfig, "--file %s is not a directory", root) } entries, err := shard.LoadManifest(root) if err != nil { return nil, archererr.Wrap(archererr.KindValidation, err) } state, err := checkpoint.Load(root) if err != nil { return nil, archererr.Wrap(archererr.KindValidation, err) } a := &Archive{Root: root, Entries: entries, State: state} if state != nil { a.Complete = state.Complete a.SourceTable = strings.TrimSpace(state.Config["table"]) if cols := strings.TrimSpace(state.Config["columns"]); cols != "" { for _, part := range strings.Split(cols, ",") { if part = strings.TrimSpace(part); part != "" { a.ArchivedColumns = append(a.ArchivedColumns, part) } } } } return a, nil } // CheckComplete enforces the completion gate. A manifest only proves which // shards are sealed, not that the export covered the whole table, so an // incomplete archive is refused unless the operator opts in explicitly. func (a *Archive) CheckComplete(allowPartial bool) error { if allowPartial { return nil } if a.State == nil { return archererr.Newf(archererr.KindValidation, "archive %s has no state.json; cannot confirm the export finished (pass --allow-partial to import its sealed shards)", a.Root) } if !a.State.Complete { return archererr.Newf(archererr.KindValidation, "archive %s is incomplete; it may contain only part of the table (pass --allow-partial to import its sealed shards)", a.Root) } return nil } // CheckMappingChoice rejects an explicit mapping when the archive already // recorded its own column list: two sources of truth would silently disagree. func (a *Archive) CheckMappingChoice(columns []string, assumeAll bool) error { if len(a.ArchivedColumns) == 0 { return nil } if len(columns) > 0 { return archererr.Newf(archererr.KindConfig, "archive records its own column list (%s); --columns cannot be combined with it", strings.Join(a.ArchivedColumns, ", ")) } if assumeAll { return archererr.Newf(archererr.KindConfig, "archive records its own column list (%s); --assume-all-columns cannot be combined with it", strings.Join(a.ArchivedColumns, ", ")) } return nil } var shardFileRE = regexp.MustCompile(`^\d+\..*\.(dump|csv|jsonl)(\.gz|\.zst|\.xz)?$`) // UnregisteredFiles counts shard-looking files that the manifest does not // list. They are ignored by design; the count is surfaced in --statistics so // they are not mistaken for imported data. func UnregisteredFiles(root string, entries []shard.Entry) int { registered := make(map[string]bool, len(entries)) for _, e := range entries { registered[filepath.ToSlash(e.Path)] = true } n := 0 _ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { if err != nil || d.IsDir() { return nil } name := d.Name() if name == "manifest.jsonl" || name == "state.json" || strings.HasSuffix(name, ".partial") || strings.HasSuffix(name, ".partial.meta") || !shardFileRE.MatchString(name) { return nil } rel, err := filepath.Rel(root, path) if err != nil { return nil } if !registered[filepath.ToSlash(rel)] { n++ } return nil }) return n } // ShardFormat derives the output format and compression from a manifest path // such as 000003.100-200.dump.gz. func ShardFormat(p string) (format, compress string, err error) { name := filepath.Base(p) switch { case strings.HasSuffix(name, ".gz"): compress, name = "gzip", strings.TrimSuffix(name, ".gz") case strings.HasSuffix(name, ".zst"): compress, name = "zstd", strings.TrimSuffix(name, ".zst") case strings.HasSuffix(name, ".xz"): compress, name = "xz", strings.TrimSuffix(name, ".xz") default: compress = "none" } ext := strings.ToLower(filepath.Ext(name)) if ext == "" { return "", "", fmt.Errorf("shard %s has no format extension", p) } format = strings.TrimPrefix(ext, ".") switch format { case "dump", "csv", "jsonl": return format, compress, nil default: return "", "", fmt.Errorf("shard %s has unknown format %q", p, format) } } // ResolveFormat checks that every selected shard agrees on format and // compression, and that the format is dump (the only one v1 can replay). func ResolveFormat(entries []shard.Entry) (format, compress string, comp *shard.Compressor, err error) { for i, e := range entries { f, c, ferr := ShardFormat(e.Path) if ferr != nil { return "", "", nil, archererr.Wrap(archererr.KindValidation, ferr) } if i == 0 { format, compress = f, c continue } if f != format { return "", "", nil, archererr.Newf(archererr.KindValidation, "selected shards mix formats: %s is %s but %s is %s", entries[0].Path, format, e.Path, f) } if c != compress { return "", "", nil, archererr.Newf(archererr.KindValidation, "selected shards mix compression: %s is %s but %s is %s", entries[0].Path, compress, e.Path, c) } } if format != "dump" { return "", "", nil, archererr.Newf(archererr.KindValidation, "import supports only dump archives; the selected shards are %s", format) } comp, err = shard.NewCompressor(compress, shard.DefaultLevel) if err != nil { return "", "", nil, archererr.Wrap(archererr.KindValidation, err) } return format, compress, comp, nil } // CheckShardFiles verifies every selected manifest entry has a readable file // before any connection is opened. func CheckShardFiles(root string, entries []shard.Entry) error { for _, e := range entries { path := filepath.Join(root, filepath.FromSlash(e.Path)) st, err := os.Stat(path) if err != nil { return archererr.Newf(archererr.KindValidation, "manifest entry seq %d points to missing shard %s: %v", e.Seq, e.Path, err) } if st.IsDir() { return archererr.Newf(archererr.KindValidation, "manifest entry seq %d points to a directory, not a shard: %s", e.Seq, e.Path) } } return nil } func shardPath(root string, e shard.Entry) string { return filepath.Join(root, filepath.FromSlash(e.Path)) }