Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,98 @@
|
||||
// Package checkpoint persists and restores scan progress in state.json.
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"archer/internal/atomicfile"
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
const Version = 1
|
||||
|
||||
// Cursor identifies where the scanner should resume. Phase is one of
|
||||
// "pk", "null", "value" or "day"; the meaning of Key depends on it.
|
||||
type Cursor struct {
|
||||
Phase string `json:"phase,omitempty"`
|
||||
Key row.Key `json:"key,omitempty"`
|
||||
DayStart string `json:"day_start,omitempty"`
|
||||
}
|
||||
|
||||
// State is the entire persisted checkpoint.
|
||||
type State struct {
|
||||
Version int `json:"version"`
|
||||
RunID string `json:"run_id"`
|
||||
ConfigHash string `json:"config_hash"`
|
||||
Config map[string]string `json:"config,omitempty"`
|
||||
ShardBy string `json:"shard_by"`
|
||||
Cursor Cursor `json:"cursor"`
|
||||
LastBucket string `json:"last_bucket,omitempty"`
|
||||
LastShardSeq int64 `json:"last_shard_seq"`
|
||||
RowsDone int64 `json:"rows_done"`
|
||||
Complete bool `json:"complete"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func Path(dir string) string { return filepath.Join(dir, "state.json") }
|
||||
|
||||
// Load reads state.json. A missing file returns (nil, nil).
|
||||
func Load(dir string) (*State, error) {
|
||||
b, err := os.ReadFile(Path(dir))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read state: %w", err)
|
||||
}
|
||||
var st State
|
||||
if err := json.Unmarshal(b, &st); err != nil {
|
||||
return nil, fmt.Errorf("parse state.json: %w", err)
|
||||
}
|
||||
if st.Version != Version {
|
||||
return nil, fmt.Errorf("unsupported state.json version %d", st.Version)
|
||||
}
|
||||
return &st, nil
|
||||
}
|
||||
|
||||
// Save atomically replaces state.json using a temporary file, fsync and
|
||||
// rename.
|
||||
func Save(dir string, st *State) error {
|
||||
st.Version = Version
|
||||
st.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
|
||||
b, err := json.MarshalIndent(st, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode state: %w", err)
|
||||
}
|
||||
b = append(b, '\n')
|
||||
|
||||
final := Path(dir)
|
||||
tmp := final + ".tmp"
|
||||
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create state temp: %w", err)
|
||||
}
|
||||
if _, err := f.Write(b); err != nil {
|
||||
f.Close()
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("write state temp: %w", err)
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
f.Close()
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("sync state temp: %w", err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("close state temp: %w", err)
|
||||
}
|
||||
if err := atomicfile.Replace(tmp, final); err != nil {
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("replace state.json: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"archer/internal/row"
|
||||
)
|
||||
|
||||
func TestSaveLoadRoundTrip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
st := &State{
|
||||
RunID: "run-1",
|
||||
ConfigHash: "abc",
|
||||
ShardBy: "date",
|
||||
Cursor: Cursor{Phase: "value", Key: row.KeyOf("2025-01-01 00:00:00", "42")},
|
||||
LastBucket: "2025-01-01",
|
||||
RowsDone: 99,
|
||||
}
|
||||
if err := Save(dir, st); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(Path(dir) + ".tmp"); !os.IsNotExist(err) {
|
||||
t.Fatal("temp state file should not remain")
|
||||
}
|
||||
got, err := Load(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got == nil || got.RunID != "run-1" || got.RowsDone != 99 || got.LastBucket != "2025-01-01" {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
if got.Cursor.Phase != "value" || len(got.Cursor.Key) != 2 || got.Cursor.Key[1].Val != "42" {
|
||||
t.Fatalf("cursor %+v", got.Cursor)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadMissing(t *testing.T) {
|
||||
st, err := Load(t.TempDir())
|
||||
if err != nil || st != nil {
|
||||
t.Fatalf("got %v %v", st, err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user