package importer import ( "context" "database/sql" "fmt" "io" "os" "sync" "time" "github.com/go-sql-driver/mysql" "archer/internal/archererr" "archer/internal/cli" "archer/internal/progress" "archer/internal/schema" "archer/internal/shard" ) // Stats collects the --statistics counters for one import run. type Stats struct { Started time.Time Elapsed time.Duration Selected int NotSelected int Succeeded int Failed int ExpectedRows int64 ImportedRows int64 SkippedRows int64 Bytes int64 IgnoredFiles int Partial bool } // Run executes one import. func Run(ctx context.Context, cfg *cli.ImportConfig) error { if err := cfg.Validate(); err != nil { return err } stats := &Stats{Started: time.Now()} archive, err := LoadArchive(cfg.FileDir) if err != nil { return err } if err := archive.CheckComplete(cfg.AllowPartial); err != nil { return err } if err := archive.CheckMappingChoice(cfg.Columns, cfg.AssumeAll); err != nil { return err } stats.Partial = cfg.AllowPartial && (archive.State == nil || !archive.State.Complete) selected, warnings, err := shard.ApplySelection(archive.Entries, cfg.Selectors) if err != nil { return archererr.Newf(archererr.KindConfig, "%v", err) } if err := CheckShardFiles(archive.Root, selected); err != nil { return err } format, compress, comp, err := ResolveFormat(selected) if err != nil { return err } archive.Format, archive.Compress, archive.Compressor = format, compress, comp stats.Selected = len(selected) stats.NotSelected = len(archive.Entries) - len(selected) for _, e := range selected { stats.ExpectedRows += e.Rows stats.Bytes += e.Bytes } if cfg.DestTable == "" { cfg.DestTable = archive.SourceTable } if cfg.DestTable == "" { return archererr.Newf(archererr.KindConfig, "--dest-table is required: the archive does not record the source table name") } if cfg.DryRun { fields, hasRow, ferr := FirstLineFieldCount(shardPath(archive.Root, selected[0]), comp) if ferr != nil { return archererr.Wrap(archererr.KindValidation, ferr) } return PrintDryRun(os.Stdout, cfg, archive, selected, warnings, compress, fields, hasRow) } for _, w := range warnings { fmt.Fprintln(os.Stderr, "warning:", w) } if len(warnings) > 0 { fmt.Fprintln(os.Stderr, "warning: shard selection is whole-shard; rows outside the requested key range may be imported") } destCfg, err := cfg.DestConfig() if err != nil { return archererr.Wrap(archererr.KindConfig, err) } db, err := openDest(destCfg, cfg.Workers) if err != nil { return err } defer db.Close() if err := checkTarget(ctx, db, cfg.Engine); err != nil { return err } if err := checkTableExists(ctx, db, destCfg.DBName, cfg.DestTable); err != nil { return err } table, err := schema.Load(ctx, db, destCfg.DBName, cfg.DestTable) if err != nil { return archererr.Wrap(archererr.KindValidation, err) } mapping, err := BuildMapping(table, archive.ArchivedColumns, cfg.Columns, cfg.AssumeAll) if err != nil { return err } if mapping.Positional { fmt.Fprintln(os.Stderr, "warning: --assume-all-columns accepts positional mapping; only the field count is checked, not column names or order") } if err := CheckFieldCount(shardPath(archive.Root, selected[0]), comp, len(mapping.FileColumns)); err != nil { return err } rep := startReporter(cfg, stats.ExpectedRows) if rep != nil { defer rep.Stop() } if err := runWorkers(ctx, db, destCfg.DBName, cfg, archive, mapping, selected, stats, rep); err != nil { if archererr.KindOf(err) == archererr.KindInterrupted && rep != nil { rep.Interrupted() } return err } stats.Elapsed = time.Since(stats.Started) stats.IgnoredFiles = UnregisteredFiles(archive.Root, archive.Entries) if rep != nil { rep.SetShards(int64(len(selected))) rep.SetBytes(stats.Bytes) rep.Done() } if cfg.Statistics { PrintStats(os.Stdout, archive, stats) } return nil } func openDest(cfg *mysql.Config, workers int) (*sql.DB, error) { db, err := sql.Open("mysql", cfg.FormatDSN()) if err != nil { return nil, archererr.Wrap(archererr.KindConnection, err) } db.SetMaxOpenConns(workers + 2) db.SetMaxIdleConns(workers + 1) db.SetConnMaxLifetime(time.Hour) if err := db.Ping(); err != nil { db.Close() return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("connect %s: %w", cfg.Addr, err)) } return db, nil } func checkTarget(ctx context.Context, db *sql.DB, engine string) error { var localInfile int var version string if err := db.QueryRowContext(ctx, "SELECT @@local_infile, @@version").Scan(&localInfile, &version); err != nil { return archererr.Wrap(archererr.KindConnection, fmt.Errorf("query local_infile: %w", err)) } if engine == "load" && localInfile == 0 { return archererr.Newf(archererr.KindConfig, "target server (%s) has local_infile disabled; enable it on the server/proxy, or use --engine insert", version) } return nil } func checkTableExists(ctx context.Context, db *sql.DB, dbName, table string) error { var n int err := db.QueryRowContext(ctx, ` SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = ? AND table_name = ?`, dbName, table).Scan(&n) if err != nil { return archererr.Wrap(archererr.KindConnection, fmt.Errorf("check target table: %w", err)) } if n == 0 { return archererr.Newf(archererr.KindValidation, "target table %s.%s does not exist; create it before importing (archer does not create tables)", dbName, table) } return nil } func startReporter(cfg *cli.ImportConfig, total int64) *progress.Reporter { tty := progress.IsTerminal(os.Stderr) enabled := cfg.Progress.Enabled(tty) r := progress.New(progress.Options{ Out: os.Stderr, Enabled: enabled, Live: enabled && tty, Interval: time.Second, RowEvery: cfg.Progress.Rows, }) r.Start() r.SetEstimate(&progress.Estimate{Kind: "rows", Lo: 0, Hi: float64(total)}) r.SetPosition(0) return r } func runWorkers(ctx context.Context, db *sql.DB, dbName string, cfg *cli.ImportConfig, archive *Archive, mapping *Mapping, selected []shard.Entry, stats *Stats, rep *progress.Reporter) error { parent := ctx ctx, cancel := context.WithCancel(ctx) defer cancel() var ( wg sync.WaitGroup mu sync.Mutex firstErr error failedSeq int64 processedRows int64 rowsAtFailure int64 importedRows int64 skippedRows int64 okShards int ) recordFailure := func(e shard.Entry, err error) { mu.Lock() defer mu.Unlock() if firstErr != nil { return } if parent.Err() != nil { firstErr = archererr.Wrap(archererr.KindInterrupted, parent.Err()) } else { firstErr = err } failedSeq = e.Seq rowsAtFailure = processedRows } ranges := splitRanges(len(selected), cfg.Workers) for _, rg := range ranges { if rg[0] >= rg[1] { continue } part := selected[rg[0]:rg[1]] wg.Add(1) go func(part []shard.Entry) { defer wg.Done() if err := ctx.Err(); err != nil { recordFailure(part[0], err) return } w, err := newWorker(ctx, db, cfg, archive, mapping, dbName) if err != nil { recordFailure(part[0], err) cancel() return } defer w.close() for _, e := range part { if err := ctx.Err(); err != nil { recordFailure(e, err) return } affected, skipped, err := w.importShard(ctx, e) if err != nil { recordFailure(e, err) cancel() return } mu.Lock() processedRows += e.Rows importedRows += affected skippedRows += skipped okShards++ done := processedRows mu.Unlock() if rep != nil { rep.AddRows(e.Rows) rep.SetPosition(float64(done)) } } }(part) } wg.Wait() mu.Lock() stats.Succeeded = okShards stats.ImportedRows = importedRows stats.SkippedRows = skippedRows err := firstErr seq := failedSeq atFailure := rowsAtFailure doneRows := processedRows mu.Unlock() if err != nil { if archererr.KindOf(err) == archererr.KindInterrupted { return fmt.Errorf("interrupted after %d rows: %w", doneRows, err) } stats.Failed = 1 return fmt.Errorf("shard %d failed after %d rows: %w", seq, atFailure, err) } return nil } func splitRanges(n, workers int) [][2]int { if n <= 0 || workers <= 0 { return nil } if workers > n { workers = n } out := make([][2]int, 0, workers) base, rem := n/workers, n%workers start := 0 for i := 0; i < workers; i++ { size := base if i < rem { size++ } out = append(out, [2]int{start, start + size}) start += size } return out } // PrintDryRun prints the resolved configuration and selection without opening // a database connection. func PrintDryRun(w io.Writer, cfg *cli.ImportConfig, archive *Archive, selected []shard.Entry, warnings []string, compress string, fields int, hasRow bool) error { fmt.Fprintln(w, "== import configuration ==") fmt.Fprintf(w, "archive: %s\n", archive.Root) fmt.Fprintf(w, "dest: %s\n", redactDSN(cfg.DestDSN)) fmt.Fprintf(w, "dest table: %s\n", cfg.DestTable) if len(archive.ArchivedColumns) > 0 { fmt.Fprintf(w, "archive columns: %s\n", join(archive.ArchivedColumns)) } else if len(cfg.Columns) > 0 { fmt.Fprintf(w, "columns: %s\n", join(cfg.Columns)) } else if cfg.AssumeAll { fmt.Fprintf(w, "columns: positional (--assume-all-columns)\n") } else { fmt.Fprintf(w, "columns: (unresolved; --columns or --assume-all-columns is required)\n") } fmt.Fprintf(w, "engine: %s\n", cfg.Engine) fmt.Fprintf(w, "mode: %s\n", cfg.Mode) fmt.Fprintf(w, "verify: %s\n", cfg.Verify) fmt.Fprintf(w, "workers: %d\n", cfg.Workers) fmt.Fprintf(w, "charset: %s\n", cfg.Charset) if cfg.TimeZone != "" { fmt.Fprintf(w, "time zone: %s\n", cfg.TimeZone) } fmt.Fprintf(w, "compression: %s\n", compress) switch { case archive.State == nil: fmt.Fprintln(w, "archive complete: unknown (state.json missing)") case archive.Complete: fmt.Fprintln(w, "archive complete: true") default: fmt.Fprintln(w, "archive complete: false (--allow-partial given)") } if hasRow { fmt.Fprintf(w, "first row fields: %d\n", fields) } fmt.Fprintf(w, "\n== selected shards (%d) ==\n", len(selected)) var rows, bytes int64 for _, e := range selected { date := e.Date if date == "" { date = "-" } fmt.Fprintf(w, " seq=%-6d pk=%-24s date=%-12s rows=%-10d bytes=%-10d %s\n", e.Seq, e.PKLo+"-"+e.PKHi, date, e.Rows, e.Bytes, e.Path) rows += e.Rows bytes += e.Bytes } fmt.Fprintf(w, "total: %d shards, %d rows, %d bytes\n", len(selected), rows, bytes) fmt.Fprintln(w, "\nno database connection was made and no data was written") return nil } // PrintStats prints the --statistics summary. func PrintStats(w io.Writer, archive *Archive, s *Stats) { fmt.Fprintln(w, "import summary:") fmt.Fprintf(w, " archive: %s\n", archive.Root) fmt.Fprintf(w, " partial archive: %t\n", s.Partial) fmt.Fprintf(w, " shards selected: %d\n", s.Selected) fmt.Fprintf(w, " shards succeeded: %d\n", s.Succeeded) fmt.Fprintf(w, " shards failed: %d\n", s.Failed) fmt.Fprintf(w, " shards not selected: %d\n", s.NotSelected) fmt.Fprintf(w, " rows expected: %d\n", s.ExpectedRows) fmt.Fprintf(w, " rows affected: %d\n", s.ImportedRows) fmt.Fprintf(w, " rows skipped: %d\n", s.SkippedRows) fmt.Fprintf(w, " bytes: %d\n", s.Bytes) fmt.Fprintf(w, " ignored files: %d (present on disk, not in manifest)\n", s.IgnoredFiles) fmt.Fprintf(w, " elapsed: %s\n", s.Elapsed.Round(time.Millisecond)) } func join(parts []string) string { out := "" for i, p := range parts { if i > 0 { out += ", " } out += p } return out } func redactDSN(dsn string) string { c, err := mysql.ParseDSN(dsn) if err != nil { return "" } c.Passwd = "" return c.FormatDSN() }