package importer import ( "context" "database/sql" "fmt" "io" "os" "sync/atomic" "github.com/go-sql-driver/mysql" "archer/internal/archererr" "archer/internal/cli" "archer/internal/row" "archer/internal/shard" "archer/internal/sink" ) // insertBatchRows bounds the decoder's memory independently of // max_allowed_packet; sink.DB still splits each batch by packet size. const insertBatchRows = 1000 type worker struct { conn *sql.Conn sink *sink.DB cfg *cli.ImportConfig archive *Archive mapping *Mapping dbName string } func newWorker(ctx context.Context, db *sql.DB, cfg *cli.ImportConfig, archive *Archive, mapping *Mapping, dbName string) (*worker, error) { conn, err := db.Conn(ctx) if err != nil { return nil, archererr.Wrap(archererr.KindConnection, fmt.Errorf("acquire connection: %w", err)) } if err := SetSession(ctx, conn, cfg.Charset, cfg.TimeZone); err != nil { conn.Close() return nil, err } w := &worker{conn: conn, cfg: cfg, archive: archive, mapping: mapping, dbName: dbName} if cfg.Engine == "insert" { s, err := sink.NewDB(ctx, db, dbName, cfg.DestTable, mapping.InsertColumns, cfg.Mode, mapping.InsertFieldIdx) if err != nil { conn.Close() return nil, archererr.Wrap(archererr.KindValidation, err) } w.sink = s } return w, nil } func (w *worker) close() { if w.conn != nil { _ = w.conn.Close() } } // importShard loads one shard and returns affected rows plus, for ignore mode, // the number of rows the server skipped. func (w *worker) importShard(ctx context.Context, e shard.Entry) (affected, skipped int64, err error) { path := shardPath(w.archive.Root, e) if w.cfg.Verify == "pre" { if err := VerifyShard(path, e); err != nil { return 0, 0, err } } if w.cfg.Engine == "insert" { return w.insertShard(ctx, e, path) } return w.loadShard(ctx, e, path) } func (w *worker) loadShard(ctx context.Context, e shard.Entry, path string) (int64, int64, error) { name := newReaderName(e.Seq) mysql.RegisterReaderHandler(name, func() io.Reader { return openShardReader(path, w.archive.Compressor) }) defer mysql.DeregisterReaderHandler(name) q := BuildLoadSQL(w.dbName, w.cfg.DestTable, w.cfg.Charset, w.mapping, w.cfg.Mode, name) res, err := w.conn.ExecContext(ctx, q) if err != nil { return 0, 0, archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: LOAD DATA: %w", e.Seq, err)) } affected, _ := res.RowsAffected() if err := w.checkWarnings(ctx, e.Seq); err != nil { return affected, 0, err } skipped, err := CheckRowRule(w.cfg.Mode, e.Rows, affected) if err != nil { return affected, 0, fmt.Errorf("shard %d: %w", e.Seq, err) } return affected, skipped, nil } func (w *worker) insertShard(ctx context.Context, e shard.Entry, path string) (int64, int64, error) { f, err := os.Open(path) if err != nil { return 0, 0, archererr.Wrap(archererr.KindValidation, fmt.Errorf("shard %d: open: %w", e.Seq, err)) } defer f.Close() r, err := w.archive.Compressor.Reader(f) if err != nil { return 0, 0, archererr.Wrap(archererr.KindValidation, fmt.Errorf("shard %d: decompress: %w", e.Seq, err)) } defer r.Close() tx, err := w.conn.BeginTx(ctx, nil) if err != nil { return 0, 0, archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: begin transaction: %w", e.Seq, err)) } committed := false defer func() { if !committed { _ = tx.Rollback() } }() dr := NewDumpReader(r) chunk := &row.Chunk{} var affected int64 flush := func() error { if len(chunk.Rows) == 0 { return nil } n, err := w.sink.InsertChunk(ctx, tx, chunk) if err != nil { return archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: %w", e.Seq, err)) } affected += n chunk = &row.Chunk{} return w.checkWarnings(ctx, e.Seq) } for { rw, err := dr.ReadRow() if err == io.EOF { break } if err != nil { return affected, 0, archererr.Wrap(archererr.KindValidation, fmt.Errorf("shard %d: decode: %w", e.Seq, err)) } if len(rw.Values) != len(w.mapping.FileColumns) { return affected, 0, archererr.Newf(archererr.KindValidation, "shard %d: row has %d fields but the mapping expects %d", e.Seq, len(rw.Values), len(w.mapping.FileColumns)) } chunk.Rows = append(chunk.Rows, rw) if len(chunk.Rows) >= insertBatchRows { if err := flush(); err != nil { return affected, 0, err } } } if err := flush(); err != nil { return affected, 0, err } skipped, err := CheckRowRule(w.cfg.Mode, e.Rows, affected) if err != nil { return affected, 0, fmt.Errorf("shard %d: %w", e.Seq, err) } if err := tx.Commit(); err != nil { return affected, 0, archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: commit: %w", e.Seq, err)) } committed = true return affected, skipped, nil } func (w *worker) checkWarnings(ctx context.Context, seq int64) error { warnings, err := ShowWarnings(ctx, w.conn) if err != nil { return archererr.Wrap(archererr.KindWrite, fmt.Errorf("shard %d: SHOW WARNINGS: %w", seq, err)) } return CheckWarnings(warnings, w.cfg.AllowWarnings, w.cfg.Mode, seq) } var readerCounter atomic.Uint64 func newReaderName(seq int64) string { return fmt.Sprintf("archer-import-%d-%d-%d", os.Getpid(), seq, readerCounter.Add(1)) } type errReader struct{ err error } func (e errReader) Read([]byte) (int, error) { return 0, e.err } type readCloserPair struct { r io.ReadCloser f *os.File } func (p *readCloserPair) Read(b []byte) (int, error) { return p.r.Read(b) } func (p *readCloserPair) Close() error { err := p.r.Close() if ferr := p.f.Close(); err == nil { err = ferr } return err } func openShardReader(path string, comp *shard.Compressor) io.Reader { f, err := os.Open(path) if err != nil { return errReader{err} } r, err := comp.Reader(f) if err != nil { _ = f.Close() return errReader{err} } return &readCloserPair{r: r, f: f} }