package scan import ( "context" "database/sql" "fmt" "io" "strconv" "strings" "time" "archer/internal/row" "archer/internal/schema" "archer/internal/sqlutil" ) const zeroDay = "0000-00-00" // DateBScanner enumerates days one at a time and streams each day's rows over // a dedicated connection. It is used when only a single-column index on the // shard column exists. type DateBScanner struct { o *Options conn *sql.Conn phase string // null | day rows *sql.Rows cursor row.Key done bool daysInited bool hasDays bool zeroPending bool zeroDone bool nextDay time.Time maxDay time.Time curDay string nextDayStr string resumeDay string resumeKey row.Key resumeUse bool } func NewDateB(ctx context.Context, o Options) (*DateBScanner, error) { conn, err := o.DB.Conn(ctx) if err != nil { return nil, fmt.Errorf("acquire streaming connection: %w", err) } s := &DateBScanner{o: &o, conn: conn, phase: "null"} if o.Resume != nil { switch o.Resume.Phase { case "null": s.phase = "null" s.cursor = o.Resume.Key case "day": s.phase = "day" s.resumeDay = o.Resume.DayStart s.resumeKey = o.Resume.Key } } return s, nil } func (s *DateBScanner) Next(ctx context.Context) (*row.Chunk, error) { for { if s.done { return nil, io.EOF } if s.rows == nil { if err := s.openNext(ctx); err != nil { if err == io.EOF { s.done = true return nil, io.EOF } return nil, err } } c, exhausted, err := s.o.readChunk(s.rows, s.o.ChunkSize) if err != nil { return nil, err } if err := s.o.fillBuckets(c); err != nil { return nil, err } if c.Len() > 0 { if s.phase == "day" { s.cursor = s.o.compositeCursor(c.Rows[len(c.Rows)-1]) } else { s.cursor = c.Keys[len(c.Keys)-1] } c.LastKey = s.cursor } if exhausted { if err := s.rows.Close(); err != nil { return nil, err } s.rows = nil if s.phase == "null" { s.phase = "day" s.cursor = nil } } if c.Len() == 0 { continue } return c, nil } } func (s *DateBScanner) openNext(ctx context.Context) error { if s.phase == "null" { return s.openNull(ctx) } if !s.daysInited { if err := s.initDays(ctx); err != nil { return err } } if s.zeroPending && !s.zeroDone { s.zeroPending = false s.zeroDone = true s.curDay = zeroDay s.nextDayStr = zeroDay return s.openZero(ctx) } if !s.advanceDay() { return io.EOF } return s.openDay(ctx) } func (s *DateBScanner) openNull(ctx context.Context) error { var b strings.Builder b.WriteString(s.o.selectSQL()) b.WriteString(" AND ") b.WriteString(sqlutil.Ident(s.o.Shard.Column)) b.WriteString(" IS NULL") var args []any if len(s.cursor) > 0 { b.WriteString(" AND ") b.WriteString(sqlutil.GreaterThan(s.o.KeyCols)) args = append(args, s.cursor.Args()...) } b.WriteString(" ORDER BY ") b.WriteString(sqlutil.Columns(s.o.KeyCols)) return s.query(ctx, b.String(), args) } func (s *DateBScanner) openZero(ctx context.Context) error { var b strings.Builder b.WriteString(s.o.selectSQL()) b.WriteString(" AND ") b.WriteString(sqlutil.Ident(s.o.Shard.Column)) b.WriteString(" < ?") upper := "0001-01-01" if s.o.Shard.Kind == schema.ShardDateTime { upper = "0001-01-01 00:00:00" } args := []any{upper} cols := append([]string{s.o.Shard.Column}, s.o.KeyCols...) if len(s.cursor) > 0 { b.WriteString(" AND ") b.WriteString(sqlutil.GreaterThan(cols)) args = append(args, s.cursor.Args()...) } b.WriteString(" ORDER BY ") b.WriteString(sqlutil.Columns(cols)) s.nextDayStr = zeroDay return s.query(ctx, b.String(), args) } func (s *DateBScanner) openDay(ctx context.Context) error { var b strings.Builder b.WriteString(s.o.selectSQL()) b.WriteString(" AND ") col := sqlutil.Ident(s.o.Shard.Column) b.WriteString(col) b.WriteString(" >= ? AND ") b.WriteString(col) b.WriteString(" < ?") var lo, hi any switch s.o.Shard.Kind { case schema.ShardDate: lo = s.curDay hi = s.nextDayStr case schema.ShardDateTime: lo = s.curDay + " 00:00:00" hi = s.nextDayStr + " 00:00:00" case schema.ShardEpoch: lo = schema.TimeToEpoch(s.dayTime(s.curDay), s.o.Shard.Unit) hi = schema.TimeToEpoch(s.dayTime(s.curDay).AddDate(0, 0, 1), s.o.Shard.Unit) } args := []any{lo, hi} cols := append([]string{s.o.Shard.Column}, s.o.KeyCols...) if len(s.cursor) > 0 { b.WriteString(" AND ") b.WriteString(sqlutil.GreaterThan(cols)) args = append(args, s.cursor.Args()...) } b.WriteString(" ORDER BY ") b.WriteString(sqlutil.Columns(cols)) return s.query(ctx, b.String(), args) } func (s *DateBScanner) query(ctx context.Context, q string, args []any) error { rows, err := s.conn.QueryContext(ctx, q, args...) if err != nil { return err } s.rows = rows return nil } func (s *DateBScanner) dayTime(day string) time.Time { t, _ := time.ParseInLocation("2006-01-02", day, s.o.ShardLoc) return t } func (s *DateBScanner) initDays(ctx context.Context) error { s.daysInited = true q := fmt.Sprintf("SELECT MIN(%s), MAX(%s) FROM %s WHERE (%s) AND %s IS NOT NULL", sqlutil.Ident(s.o.Shard.Column), sqlutil.Ident(s.o.Shard.Column), s.o.qualified(), s.o.Where, sqlutil.Ident(s.o.Shard.Column)) var minRaw, maxRaw sql.NullString if err := s.conn.QueryRowContext(ctx, q).Scan(&minRaw, &maxRaw); err != nil { return fmt.Errorf("read shard column range: %w", err) } if !minRaw.Valid || !maxRaw.Valid { return nil } switch s.o.Shard.Kind { case schema.ShardDate, schema.ShardDateTime: minDay := s.first10(minRaw.String) maxDay := s.first10(maxRaw.String) zeroExists := strings.HasPrefix(minDay, "0000-") if zeroExists { minDay = "0001-01-01" } s.zeroPending = s.wantZero(zeroExists) if strings.HasPrefix(maxDay, "0000-") { return nil } minT, err := time.ParseInLocation("2006-01-02", minDay, s.o.ShardLoc) if err != nil { return fmt.Errorf("parse shard range minimum %q: %w", minRaw.String, err) } maxT, err := time.ParseInLocation("2006-01-02", maxDay, s.o.ShardLoc) if err != nil { return fmt.Errorf("parse shard range maximum %q: %w", maxRaw.String, err) } s.minMax(minT, maxT) case schema.ShardEpoch: minV, err := strconv.ParseInt(strings.TrimSpace(minRaw.String), 10, 64) if err != nil { return fmt.Errorf("parse shard range minimum: %w", err) } maxV, err := strconv.ParseInt(strings.TrimSpace(maxRaw.String), 10, 64) if err != nil { return fmt.Errorf("parse shard range maximum: %w", err) } minT := schema.EpochTime(minV, s.o.Shard.Unit, s.o.ShardLoc) maxT := schema.EpochTime(maxV, s.o.Shard.Unit, s.o.ShardLoc) s.minMax(dayStart(minT, s.o.ShardLoc), dayStart(maxT, s.o.ShardLoc)) } return nil } func (s *DateBScanner) wantZero(zeroExists bool) bool { if s.resumeDay == "" { return zeroExists } return s.resumeDay == zeroDay } func (s *DateBScanner) minMax(minT, maxT time.Time) { s.hasDays = true s.nextDay = minT s.maxDay = maxT if s.resumeDay != "" && s.resumeDay != zeroDay { if d, err := time.ParseInLocation("2006-01-02", s.resumeDay, s.o.ShardLoc); err == nil && d.After(minT) { s.nextDay = d } } if s.resumeDay == zeroDay { s.nextDay = minT } } func (s *DateBScanner) advanceDay() bool { if !s.hasDays { return false } for !s.nextDay.After(s.maxDay) { day := s.nextDay.Format("2006-01-02") if s.resumeDay != "" && s.resumeDay != zeroDay && day < s.resumeDay && !s.resumeUse { s.nextDay = s.nextDay.AddDate(0, 0, 1) continue } s.curDay = day s.nextDayStr = s.nextDay.AddDate(0, 0, 1).Format("2006-01-02") s.nextDay = s.nextDay.AddDate(0, 0, 1) if s.resumeDay != "" && day == s.resumeDay && !s.resumeUse { s.cursor = s.resumeKey s.resumeUse = true } else { s.cursor = nil } return true } return false } func (s *DateBScanner) first10(v string) string { if len(v) < 10 { return v } return v[:10] } func dayStart(t time.Time, loc *time.Location) time.Time { return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, loc) } func (s *DateBScanner) ResumeState() CursorState { phase := s.phase day := "" if phase == "day" { day = s.curDay } return CursorState{Phase: phase, Key: s.cursor, DayStart: day} } func (s *DateBScanner) Close() error { var firstErr error if s.rows != nil { if err := s.rows.Close(); err != nil { firstErr = err } s.rows = nil } if s.conn != nil { if err := s.conn.Close(); err != nil && firstErr == nil { firstErr = err } s.conn = nil } return firstErr }