Add archer import and allow date-mode bulk delete
archer import replays a sealed archive into an existing table: shard selection (`--select`), SHA-256 pre-verification, idempotent modes (`ignore`/`insert`/`replace`), and both LOAD DATA and batched-INSERT engines. It reads manifest.jsonl strictly so corruption is reported instead of skipped, and supports --dry-run, --allow-partial, and per-shard workers. --bulk-delete no longer rejects --shard-by date. Range deletion now derives its bounds from the chunk's actual keys (date scanners rewrite LastKey into a resumption cursor) and keeps safe-auto-increment semantics by excluding the current max key. A startup warning documents the monotonic-shard-column assumption. Also splits CLI/format/sharding/progress/safety reference docs out of the README, and adds the corresponding OpenSpec change artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4a6b504f
commit
366d61ca25
49 files changed
+5099
-342
No files matched your search
@@ -28,6 +28,43 @@ type Entry struct {
|
||||
ClosedAt string `json:"closed_at"`
|
||||
}
|
||||
|
||||
// LoadManifest reads manifest.jsonl strictly for replay. Unlike OpenManifest it
|
||||
// never creates or appends to the archive, and a malformed line is an error
|
||||
// rather than a line to skip: an importer must not mistake corruption for a
|
||||
// missing shard.
|
||||
func LoadManifest(root string) ([]Entry, error) {
|
||||
path := filepath.Join(root, manifestName)
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open manifest: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var entries []Entry
|
||||
sc := bufio.NewScanner(f)
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
|
||||
line := 0
|
||||
for sc.Scan() {
|
||||
line++
|
||||
text := strings.TrimSpace(sc.Text())
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
var e Entry
|
||||
if err := json.Unmarshal([]byte(text), &e); err != nil {
|
||||
return nil, fmt.Errorf("parse manifest.jsonl line %d: %w", line, err)
|
||||
}
|
||||
if e.Path == "" {
|
||||
return nil, fmt.Errorf("manifest.jsonl line %d: missing path", line)
|
||||
}
|
||||
entries = append(entries, e)
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return nil, fmt.Errorf("read manifest: %w", err)
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// Manifest appends sealed-shard records to manifest.jsonl and knows which
|
||||
// shards already exist.
|
||||
type Manifest struct {
|
||||
|
||||
Reference in new issue
Block a user