package importer import ( "context" "crypto/sha256" "database/sql" "encoding/hex" "fmt" "io" "os" "strings" "archer/internal/archererr" "archer/internal/shard" "archer/internal/sqlutil" ) // BuildLoadSQL renders the LOAD DATA statement. Field and line rules match // internal/row.dumpEncoder exactly: TAB, LF, backslash escaping and \N. func BuildLoadSQL(dbName, table, charset string, m *Mapping, mode, readerName string) string { var b strings.Builder b.WriteString("LOAD DATA LOCAL INFILE '") b.WriteString(escapeSQLString("Reader::" + readerName)) b.WriteString("'") switch mode { case "ignore": b.WriteString(" IGNORE") case "replace": b.WriteString(" REPLACE") } b.WriteString(" INTO TABLE ") b.WriteString(sqlutil.Qualified(dbName, table)) b.WriteString(" CHARACTER SET ") b.WriteString(charset) b.WriteString(` FIELDS TERMINATED BY '\t' ESCAPED BY '\\' LINES TERMINATED BY '\n'`) if len(m.LoadList) > 0 { b.WriteString(" (") b.WriteString(strings.Join(m.LoadList, ", ")) b.WriteString(")") } return b.String() } func escapeSQLString(s string) string { s = strings.ReplaceAll(s, `\`, `\\`) return strings.ReplaceAll(s, `'`, `''`) } // CheckRowRule applies the per-mode affected-row rule and returns the number // of rows that were skipped (only meaningful for ignore mode). func CheckRowRule(mode string, expected, affected int64) (int64, error) { switch mode { case "insert": if affected != expected { return 0, archererr.Newf(archererr.KindValidation, "insert expected %d rows but wrote %d", expected, affected) } return 0, nil case "ignore": if affected > expected { return 0, archererr.Newf(archererr.KindValidation, "ignore wrote %d rows but the manifest records only %d", affected, expected) } return expected - affected, nil default: // replace may delete+insert, so affected can exceed rows return 0, nil } } // VerifyShard streams the compressed shard through SHA-256 and compares it // with the manifest before any bytes are sent to the server. func VerifyShard(path string, e shard.Entry) error { if e.SHA256 == "" { return nil } f, err := os.Open(path) if err != nil { return archererr.Newf(archererr.KindValidation, "open shard %s: %v", e.Path, err) } defer f.Close() h := sha256.New() if _, err := io.Copy(h, f); err != nil { return archererr.Newf(archererr.KindValidation, "hash shard %s: %v", e.Path, err) } got := hex.EncodeToString(h.Sum(nil)) if !strings.EqualFold(got, e.SHA256) { return archererr.Newf(archererr.KindValidation, "shard %d (%s) failed verification: manifest sha256 %s, actual %s", e.Seq, e.Path, e.SHA256, got) } return nil } // Warning is one SHOW WARNINGS row. type Warning struct { Level string Code int Message string } func (w Warning) String() string { return fmt.Sprintf("%s %d: %s", w.Level, w.Code, w.Message) } // ShowWarnings returns the current session warnings. func ShowWarnings(ctx context.Context, conn *sql.Conn) ([]Warning, error) { rows, err := conn.QueryContext(ctx, "SHOW WARNINGS") if err != nil { return nil, err } defer rows.Close() var out []Warning for rows.Next() { var w Warning if err := rows.Scan(&w.Level, &w.Code, &w.Message); err != nil { return nil, err } out = append(out, w) } return out, rows.Err() } // CheckWarnings turns server warnings into a per-shard failure unless the // operator explicitly allowed them. Duplicate-key warnings under ignore mode // are expected skips (they are the mechanism that makes re-import idempotent), // so they are not failures; the skipped count comes from affected rows. func CheckWarnings(warnings []Warning, allow bool, mode string, seq int64) error { if allow || len(warnings) == 0 { return nil } fatal := warnings[:0:0] for _, w := range warnings { if mode == "ignore" && w.Code == 1062 { continue } fatal = append(fatal, w) } if len(fatal) == 0 { return nil } const maxShown = 5 shown := fatal suffix := "" if len(shown) > maxShown { suffix = fmt.Sprintf(" (and %d more)", len(shown)-maxShown) shown = shown[:maxShown] } parts := make([]string, len(shown)) for i, w := range shown { parts[i] = w.String() } return archererr.Newf(archererr.KindValidation, "shard %d produced %d server warning(s); pass --allow-warnings to continue: %s%s", seq, len(fatal), strings.Join(parts, "; "), suffix) } // SetSession applies the loader's character set and optional time zone to one // fixed connection. func SetSession(ctx context.Context, conn *sql.Conn, charset, timeZone string) error { if _, err := conn.ExecContext(ctx, "SET NAMES "+charset); err != nil { return archererr.Wrap(archererr.KindConnection, fmt.Errorf("set character set %s: %w", charset, err)) } if timeZone != "" { q := "SET time_zone = '" + escapeSQLString(timeZone) + "'" if _, err := conn.ExecContext(ctx, q); err != nil { return archererr.Wrap(archererr.KindConnection, fmt.Errorf("set time_zone %q: %w", timeZone, err)) } } return nil }