Initial commit: archer, a Go-native MySQL/MariaDB table archiver
Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
bc4a6b504f
99 files changed
+12752
No files matched your search
@@ -0,0 +1,24 @@
|
||||
param(
|
||||
[string]$Version = "0.1.0",
|
||||
[string]$Commit = $(if ($env:ARCHER_COMMIT) { $env:ARCHER_COMMIT } else { "local" })
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$root = Split-Path -Parent $PSScriptRoot
|
||||
$date = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
|
||||
$ldflags = "-s -w -X archer/internal/version.Version=$Version -X archer/internal/version.Commit=$Commit -X archer/internal/version.Date=$date"
|
||||
|
||||
New-Item -ItemType Directory -Force -Path (Join-Path $root "dist") | Out-Null
|
||||
Push-Location $root
|
||||
try {
|
||||
go build -trimpath -ldflags $ldflags -o dist/archer_windows_amd64.exe ./cmd/archer
|
||||
$env:GOOS = "linux"
|
||||
$env:GOARCH = "amd64"
|
||||
go build -trimpath -ldflags $ldflags -o dist/archer_linux_amd64 ./cmd/archer
|
||||
Remove-Item Env:GOOS -ErrorAction SilentlyContinue
|
||||
Remove-Item Env:GOARCH -ErrorAction SilentlyContinue
|
||||
Write-Host "built dist/archer_windows_amd64.exe and dist/archer_linux_amd64 ($Version, $Commit)"
|
||||
}
|
||||
finally {
|
||||
Pop-Location
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
version="${VERSION:-0.1.0}"
|
||||
commit="${COMMIT:-local}"
|
||||
date="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
root="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
ldflags="-s -w -X archer/internal/version.Version=${version} -X archer/internal/version.Commit=${commit} -X archer/internal/version.Date=${date}"
|
||||
|
||||
mkdir -p "$root/dist"
|
||||
cd "$root"
|
||||
GOOS=windows GOARCH=amd64 go build -trimpath -ldflags "$ldflags" -o dist/archer_windows_amd64.exe ./cmd/archer
|
||||
GOOS=linux GOARCH=amd64 go build -trimpath -ldflags "$ldflags" -o dist/archer_linux_amd64 ./cmd/archer
|
||||
echo "built dist/archer_windows_amd64.exe and dist/archer_linux_amd64 (${version}, ${commit})"
|
||||
@@ -0,0 +1,213 @@
|
||||
@echo off
|
||||
chcp 65001 >nul 2>&1
|
||||
setlocal EnableDelayedExpansion
|
||||
|
||||
REM =====================================================================
|
||||
REM export-all.cmd
|
||||
REM
|
||||
REM 用 archer 把某个 MySQL 库的「所有表」导出为按行数分片 + gzip 压缩的 dump。
|
||||
REM
|
||||
REM 用法:
|
||||
REM scripts\export-all.cmd
|
||||
REM 使用下面「配置区」的默认值
|
||||
REM
|
||||
REM scripts\export-all.cmd DSN OUTDIR SHARD_ROWS CHUNK_SIZE
|
||||
REM 例: scripts\export-all.cmd ^
|
||||
REM "root:pass@tcp(10.0.0.5:3306)/app" "D:\dump" 500000 5000
|
||||
REM
|
||||
REM 产物:
|
||||
REM <OUTDIR>\<表名>\000001.<lo>-<hi>.dump.gz 分片文件(按主键连续、不重叠)
|
||||
REM <OUTDIR>\<表名>\manifest.jsonl 每片的行数/字节/sha256
|
||||
REM <OUTDIR>\<表名>\state.json 断点续跑状态
|
||||
REM <OUTDIR>\<表名>.log 该表 archer 的完整输出
|
||||
REM
|
||||
REM 断点续跑:
|
||||
REM 用同样的参数再跑一遍即可,archer 从 state.json 继续;
|
||||
REM state.json 里 complete=true 的表会被跳过。
|
||||
REM 想重导某张表,先删掉它的目录。
|
||||
REM
|
||||
REM 依赖:
|
||||
REM dist\archer_windows_amd64.exe (scripts\build.ps1 构建)
|
||||
REM mysql.exe (只用来枚举表名,需在 PATH 里)
|
||||
REM =====================================================================
|
||||
|
||||
REM ------------------------------ 配置区 ------------------------------
|
||||
REM 源库连接。改动源库时,SRC 与下面 5 个变量要一起改
|
||||
REM (mysql 客户端要从后者取参数去枚举表名)。
|
||||
set "SRC=root:xyTyfxhn4h@tcp(s1.void01.com:3306)/lexis"
|
||||
set "DB=lexis"
|
||||
set "DB_HOST=s1.void01.com"
|
||||
set "DB_PORT=3306"
|
||||
set "DB_USER=root"
|
||||
set "DB_PASS=xyTyfxhn4h"
|
||||
|
||||
REM 输出根目录(每张表会在其下建一个同名子目录)
|
||||
set "OUTDIR=%~dp0..\export"
|
||||
|
||||
REM 每个分片文件的行数;小于表行数时只会得到 1 个分片
|
||||
set "SHARD_ROWS=1000000"
|
||||
|
||||
REM 每次 SELECT / 事务 / checkpoint 的行数,与分片大小无关
|
||||
set "CHUNK_SIZE=5000"
|
||||
|
||||
REM 输出格式 dump|csv|jsonl 与压缩方式 none|gzip|zstd
|
||||
set "FMT=dump"
|
||||
set "COMPRESS=gzip"
|
||||
|
||||
REM 只导指定表(逗号分隔);留空表示导全部
|
||||
set "ONLY_TABLES="
|
||||
|
||||
REM 排除表名通配符,如 tmp_* REM_*;留空表示不排除
|
||||
set "EXCLUDE="
|
||||
REM --------------------------------------------------------------------
|
||||
|
||||
REM 命令行覆盖
|
||||
if not "%~1"=="" set "SRC=%~1"
|
||||
if not "%~2"=="" set "OUTDIR=%~2"
|
||||
if not "%~3"=="" set "SHARD_ROWS=%~3"
|
||||
if not "%~4"=="" set "CHUNK_SIZE=%~4"
|
||||
|
||||
set "EXE=%~dp0..\dist\archer_windows_amd64.exe"
|
||||
if not exist "%EXE%" (
|
||||
echo [错误] 找不到 archer: %EXE%
|
||||
echo 先运行 scripts\build.ps1 构建
|
||||
goto :fail
|
||||
)
|
||||
|
||||
where mysql.exe >nul 2>&1
|
||||
if errorlevel 1 (
|
||||
echo [错误] PATH 中找不到 mysql.exe,无法枚举表名
|
||||
echo 把 mysql 的 bin 目录加进 PATH,或用 ARCHER 逐表手写命令
|
||||
goto :fail
|
||||
)
|
||||
|
||||
if not exist "%OUTDIR%" mkdir "%OUTDIR%" >nul 2>&1
|
||||
if not exist "%OUTDIR%" (
|
||||
echo [错误] 无法创建输出目录: %OUTDIR%
|
||||
goto :fail
|
||||
)
|
||||
|
||||
REM mysql 用环境变量传密码,避免出现在命令行/进程列表里
|
||||
set "MYSQL_PWD=%DB_PASS%"
|
||||
set "MYSQL_CMD=mysql.exe --protocol=TCP -h%DB_HOST% -P%DB_PORT% -u%DB_USER% --default-character-set=utf8mb4 -N -B"
|
||||
|
||||
echo.
|
||||
echo archer : %EXE%
|
||||
echo schema : %DB% (%DB_HOST%:%DB_PORT%)
|
||||
echo out dir : %OUTDIR%
|
||||
echo shard : %FMT% + %COMPRESS%, %SHARD_ROWS% rows/shard, chunk %CHUNK_SIZE%
|
||||
echo.
|
||||
|
||||
REM ---------------------------- 表清单 ----------------------------
|
||||
if defined ONLY_TABLES (
|
||||
set "TABLE_LIST=%ONLY_TABLES:,= %"
|
||||
set "TABLE_LIST=!TABLE_LIST: =!|"
|
||||
) else (
|
||||
set "TABLE_LIST="
|
||||
for /f "usebackq delims=" %%t in (`%MYSQL_CMD% -e "SELECT table_name FROM information_schema.tables WHERE table_schema='%DB%' AND table_type='BASE TABLE' ORDER BY table_name" 2^>nul`) do (
|
||||
set "TABLE_LIST=!TABLE_LIST!%%t|"
|
||||
)
|
||||
)
|
||||
|
||||
if "!TABLE_LIST!"=="" (
|
||||
echo [错误] 没有取到任何表,检查库名/账号/网络
|
||||
goto :fail
|
||||
)
|
||||
|
||||
REM ---------------------------- 逐表导出 ----------------------------
|
||||
set "OK=0"
|
||||
set "SKIP=0"
|
||||
set "FAILED="
|
||||
set /a IDX=0
|
||||
|
||||
for %%t in ("!TABLE_LIST:|=" "!") do (
|
||||
set "T=%%~t"
|
||||
if not "!T!"=="" (
|
||||
call :one_table "!T!"
|
||||
)
|
||||
)
|
||||
|
||||
echo.
|
||||
if defined FAILED (
|
||||
echo 失败的表: %FAILED%
|
||||
echo.
|
||||
echo 完成: 成功 %OK%,跳过 %SKIP%,失败若干
|
||||
goto :fail
|
||||
)
|
||||
echo 完成: 成功 %OK%,跳过 %SKIP%,失败 0
|
||||
endlocal
|
||||
exit /b 0
|
||||
|
||||
REM ====================================================================
|
||||
REM 单表导出
|
||||
REM ====================================================================
|
||||
:one_table
|
||||
set "T=%~1"
|
||||
set /a IDX+=1
|
||||
|
||||
if defined EXCLUDE (
|
||||
REM 简单通配:把 * 替换成批处理的 .*
|
||||
set "PAT=!EXCLUDE:*=.!"
|
||||
for %%p in (!PAT!) do (
|
||||
for /f "delims=" %%m in ("!T!") do (
|
||||
if "%%m"=="%%m" (
|
||||
REM 用 findstr 做通配匹配
|
||||
)
|
||||
)
|
||||
)
|
||||
echo !T! | findstr /r /i /c:"^!EXCLUDE:\*=.!$" >nul 2>&1
|
||||
if not errorlevel 1 (
|
||||
echo [!IDX!] !T! 跳过(被 EXCLUDE 匹配)
|
||||
goto :eof
|
||||
)
|
||||
)
|
||||
|
||||
set "TDIR=%OUTDIR%\!T!"
|
||||
|
||||
REM 已完成则跳过
|
||||
if exist "!TDIR!\state.json" (
|
||||
findstr /c:"\"complete\": true" "!TDIR!\state.json" >nul 2>&1
|
||||
if not errorlevel 1 (
|
||||
echo [!IDX!] !T! 跳过(已完成)
|
||||
set /a SKIP+=1
|
||||
goto :eof
|
||||
)
|
||||
echo [!IDX!] !T! 续跑
|
||||
)
|
||||
|
||||
set "LOG=%OUTDIR%\!T!.log"
|
||||
set "START=%TIME%"
|
||||
|
||||
"%EXE%" ^
|
||||
--source "%SRC%" ^
|
||||
--table "!T!" ^
|
||||
--where "1=1" ^
|
||||
--file "!TDIR!" ^
|
||||
--format %FMT% ^
|
||||
--compress %COMPRESS% ^
|
||||
--shard-by count ^
|
||||
--shard-rows %SHARD_ROWS% ^
|
||||
--chunk-size %CHUNK_SIZE% ^
|
||||
--statistics > "!LOG!" 2>&1
|
||||
|
||||
if errorlevel 1 (
|
||||
echo [!IDX!] !T! 失败,日志: !LOG!
|
||||
for /f "delims=" %%l in ('findstr /c:"archer:" "!LOG!" 2^>nul') do echo %%l
|
||||
set "FAILED=!FAILED! !T!"
|
||||
goto :eof
|
||||
)
|
||||
|
||||
REM 统计分片数
|
||||
set "SHARDS=0"
|
||||
if exist "!TDIR!\manifest.jsonl" (
|
||||
for /f %%n in ('type "!TDIR!\manifest.jsonl" 2^>nul ^| find /c /v ""') do set "SHARDS=%%n"
|
||||
)
|
||||
|
||||
echo [!IDX!] !T! 完成: !SHARDS! 片
|
||||
set /a OK+=1
|
||||
goto :eof
|
||||
|
||||
REM ====================================================================
|
||||
:fail
|
||||
endlocal
|
||||
exit /b 1
|
||||
@@ -0,0 +1,336 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
用 archer 导出某个 MySQL 库中的全部表,按行数分片并压缩。
|
||||
|
||||
.DESCRIPTION
|
||||
自动遍历 information_schema 里指定库的所有基础表,逐表调用 archer:
|
||||
--shard-by count --format dump --compress gzip
|
||||
每张表输出到独立目录 <OutDir>\<Schema>\<table>,因为 archer 的
|
||||
manifest.jsonl / state.json 是「按输出目录」维护的,混在一起会互相覆盖。
|
||||
|
||||
断点续跑:archer 自己会在同目录 + 同配置下续跑,脚本默认跳过
|
||||
state.json 中 complete=true 的表。需要重导某张表时用 -Restart。
|
||||
|
||||
.PARAMETER Source
|
||||
源库 DSN,形如 user:pass@tcp(host:3306)/db。默认取环境变量 ARCHER_SOURCE。
|
||||
|
||||
.PARAMETER Schema
|
||||
要导出的库名。默认取 DSN 中的库名。
|
||||
|
||||
.PARAMETER OutDir
|
||||
输出根目录。默认 <仓库根>\export。
|
||||
|
||||
.PARAMETER ShardRows
|
||||
每个分片文件的行数,默认 1000000。表行数小于该值时只有 1 个分片。
|
||||
|
||||
.PARAMETER ChunkSize
|
||||
每次 SELECT / 事务 / checkpoint 的行数,默认 5000。
|
||||
|
||||
.PARAMETER Format
|
||||
分片格式 dump|csv|jsonl,默认 dump(LOAD DATA 兼容 TSV)。
|
||||
|
||||
.PARAMETER Compress
|
||||
压缩算法 none|gzip|zstd,默认 gzip。
|
||||
|
||||
.PARAMETER Tables
|
||||
只导这些表(表名数组),默认全会话库内所有基础表。
|
||||
|
||||
.PARAMETER Exclude
|
||||
排除的表名通配符,如 'tmp_*'。
|
||||
|
||||
.PARAMETER Retries
|
||||
单表失败后的重试次数,默认 3(0 表示不重试)。archer 的 state.json 会续跑,
|
||||
因此重试是安全的:Windows 上偶发 state.json rename "Access is denied"
|
||||
(杀毒/索引器短暂占用句柄),重试一次即恢复。
|
||||
|
||||
.PARAMETER Restart
|
||||
删除已存在的表输出目录后重新导出(默认会跳过已完成 / 续跑未完成的表)。
|
||||
|
||||
.PARAMETER DryRun
|
||||
只打印将要执行的 archer 命令,不实际执行。
|
||||
|
||||
.PARAMETER Verify
|
||||
导出后逐表比对 state.json 的 rows_done 与源表 count(*)。
|
||||
注意 rows_done 现在是累计值(含续跑前已提交的行数),因此续跑过的表也能与源表
|
||||
正确对齐;升级前的旧 checkpoint 只记录上次运行行数,首次校验可能仍报一次差异,
|
||||
重新完成后即恢复。
|
||||
|
||||
.PARAMETER ProgressMode
|
||||
archer 的 --progress 模式:auto|on|off,默认 auto。脚本把 stdout+stderr 重定向进
|
||||
<OutDir>\<Schema>.<table>.log,重定向时 auto 不输出进度;设为 on 可让日志每秒
|
||||
追加一行完整进度,便于 Get-Content -Wait / tail 观察长任务。
|
||||
|
||||
.EXAMPLE
|
||||
.\export-all.ps1 -Source 'root:pass@tcp(s1.void01.com:3306)/lexis'
|
||||
|
||||
.EXAMPLE
|
||||
.\export-all.ps1 -Source 'root:pass@tcp(s1.void01.com:3306)/lexis' `
|
||||
-OutDir D:\dump -ShardRows 250000 -Verify
|
||||
|
||||
.EXAMPLE
|
||||
.\export-all.ps1 -Source 'root:pass@tcp(s1.void01.com:3306)/lexis' `
|
||||
-ProgressMode on
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$Source = $env:ARCHER_SOURCE,
|
||||
[string]$Schema = '',
|
||||
[string]$OutDir = '',
|
||||
[int]$ShardRows = 1000000,
|
||||
[int]$ChunkSize = 5000,
|
||||
[ValidateSet('dump', 'csv', 'jsonl')]
|
||||
[string]$Format = 'dump',
|
||||
[ValidateSet('none', 'gzip', 'zstd')]
|
||||
[string]$Compress = 'gzip',
|
||||
[string[]]$Tables = @(),
|
||||
[string]$Exclude = '',
|
||||
[string]$Archer = '',
|
||||
[string]$Mysql = '',
|
||||
[int]$Retries = 3,
|
||||
[ValidateSet('auto', 'on', 'off')]
|
||||
[string]$ProgressMode = 'auto',
|
||||
[switch]$Restart,
|
||||
[switch]$DryRun,
|
||||
[switch]$Verify
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 路径与可执行文件
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
$root = Split-Path -Parent $PSScriptRoot
|
||||
|
||||
if (-not $Archer) {
|
||||
$candidates = @(
|
||||
(Join-Path $root 'dist\archer_windows_amd64.exe'),
|
||||
(Join-Path $root 'dist\archer.exe')
|
||||
)
|
||||
$Archer = $candidates | Where-Object { Test-Path $_ } | Select-Object -First 1
|
||||
if (-not $Archer) {
|
||||
$cmd = Get-Command archer.exe -ErrorAction SilentlyContinue
|
||||
if ($cmd) { $Archer = $cmd.Source }
|
||||
}
|
||||
}
|
||||
if (-not $Archer -or -not (Test-Path $Archer)) {
|
||||
throw "找不到 archer 可执行文件,请先运行 scripts\build.ps1,或用 -Archer 指定路径"
|
||||
}
|
||||
|
||||
if (-not $Mysql) {
|
||||
$cmd = Get-Command mysql.exe -ErrorAction SilentlyContinue
|
||||
if ($cmd) { $Mysql = $cmd.Source }
|
||||
elseif (Test-Path 'C:\portable-apps\mysql-8.3.0-winx64\bin\mysql.exe') {
|
||||
$Mysql = 'C:\portable-apps\mysql-8.3.0-winx64\bin\mysql.exe'
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $Source) {
|
||||
throw "必须提供 -Source 'user:pass@tcp(host:3306)/db'(或设置环境变量 ARCHER_SOURCE)"
|
||||
}
|
||||
|
||||
# user:pass@tcp(host:port)/db
|
||||
$m = [regex]::Match($Source, '^(?<user>[^:@/]*)(:(?<pass>[^@]*))?@tcp\((?<host>[^:)]+)(:(?<port>\d+))?\)/(?<db>[^?]*)')
|
||||
if (-not $m.Success) {
|
||||
throw "无法解析 DSN,期望形如 user:pass@tcp(host:3306)/db,实际: $Source"
|
||||
}
|
||||
$dbUser = $m.Groups['user'].Value
|
||||
$dbPass = $m.Groups['pass'].Value
|
||||
$dbHost = $m.Groups['host'].Value
|
||||
$dbPort = if ($m.Groups['port'].Success) { $m.Groups['port'].Value } else { '3306' }
|
||||
if (-not $Schema) { $Schema = $m.Groups['db'].Value }
|
||||
if (-not $Schema) { throw "DSN 中没有库名,且未提供 -Schema" }
|
||||
|
||||
if (-not $OutDir) { $OutDir = Join-Path $root 'export' }
|
||||
$OutDir = [System.IO.Path]::GetFullPath($OutDir)
|
||||
$schemaDir = Join-Path $OutDir $Schema
|
||||
# 日志写在 $OutDir 下,先确保它存在
|
||||
New-Item -ItemType Directory -Force -Path $OutDir | Out-Null
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# mysql 客户端查询(只用于列举表名与校验行数)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
function Invoke-MysqlQuery {
|
||||
param([Parameter(Mandatory)][string]$Sql)
|
||||
|
||||
if (-not $Mysql -or -not (Test-Path $Mysql)) {
|
||||
throw "找不到 mysql.exe,无法查询 information_schema;请用 -Tables 显式列出表名,或用 -Mysql 指定路径"
|
||||
}
|
||||
$prev = $env:MYSQL_PWD
|
||||
$env:MYSQL_PWD = $dbPass
|
||||
try {
|
||||
$lines = & $Mysql --protocol=TCP "--host=$dbHost" "--port=$dbPort" `
|
||||
"--user=$dbUser" --default-character-set=utf8mb4 -N -B -e $Sql 2>&1
|
||||
$code = $LASTEXITCODE
|
||||
}
|
||||
finally {
|
||||
$env:MYSQL_PWD = $prev
|
||||
}
|
||||
if ($code -ne 0) {
|
||||
throw "mysql 查询失败(exit $code): $($lines -join ' ')"
|
||||
}
|
||||
# 过滤 mysql 客户端自身的 [Warning] 行
|
||||
return @($lines | Where-Object { "$_" -notmatch '^mysql: \[Warning\]' })
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 收集表清单
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# -File 调用时 -Tables a,b 会作为一个字符串传入,这里按逗号拆开
|
||||
$Tables = @($Tables | ForEach-Object { $_ -split ',' } | ForEach-Object { $_.Trim() } | Where-Object { $_ })
|
||||
|
||||
if ($Tables.Count -gt 0) {
|
||||
$tables = $Tables
|
||||
}
|
||||
else {
|
||||
if ($DryRun) { Write-Host "解析表清单..." }
|
||||
$tables = Invoke-MysqlQuery ("SELECT table_name FROM information_schema.tables " +
|
||||
"WHERE table_schema = '$Schema' AND table_type = 'BASE TABLE' ORDER BY table_name")
|
||||
}
|
||||
if ($Exclude) {
|
||||
$tables = $tables | Where-Object { $_ -notlike $Exclude }
|
||||
}
|
||||
$tables = @($tables | Where-Object { $_ })
|
||||
if ($tables.Count -eq 0) { throw "库 $Schema 中没有匹配的表" }
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 主流程
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Write-Host ''
|
||||
Write-Host "archer : $Archer"
|
||||
Write-Host "schema : $Schema ($dbHost`:$dbPort)"
|
||||
Write-Host "out dir : $schemaDir"
|
||||
Write-Host "shard : count / $ShardRows rows, chunk $ChunkSize, $Format + $Compress"
|
||||
Write-Host "tables : $($tables.Count)"
|
||||
Write-Host ''
|
||||
|
||||
$failed = @()
|
||||
$skipped = @()
|
||||
$ok = @()
|
||||
$idx = 0
|
||||
|
||||
foreach ($table in $tables) {
|
||||
$idx++
|
||||
$prefix = "[$idx/$($tables.Count)] $table"
|
||||
$tableDir = Join-Path $schemaDir $table
|
||||
$stateFile = Join-Path $tableDir 'state.json'
|
||||
|
||||
# 已完成则跳过(archer 对 complete=true 的目录会直接退出,这里提前判断避免无谓启动)
|
||||
if (-not $Restart -and (Test-Path $stateFile)) {
|
||||
try {
|
||||
$state = Get-Content -Raw -Path $stateFile | ConvertFrom-Json
|
||||
}
|
||||
catch { $state = $null }
|
||||
if ($state -and $state.complete) {
|
||||
Write-Host "$prefix 跳过(已完成,rows_done=$($state.rows_done))"
|
||||
$skipped += $table
|
||||
continue
|
||||
}
|
||||
Write-Host "$prefix 续跑(rows_done=$($state.rows_done))"
|
||||
}
|
||||
|
||||
if ($Restart -and (Test-Path $tableDir)) {
|
||||
Write-Host "$prefix -Restart:删除 $tableDir"
|
||||
Remove-Item -Recurse -Force $tableDir
|
||||
}
|
||||
|
||||
$argv = @(
|
||||
'--source', $Source,
|
||||
'--table', $table,
|
||||
'--where', '1=1',
|
||||
'--file', $tableDir,
|
||||
'--format', $Format,
|
||||
'--compress', $Compress,
|
||||
'--shard-by', 'count',
|
||||
'--shard-rows', "$ShardRows",
|
||||
'--chunk-size', "$ChunkSize",
|
||||
'--progress', $ProgressMode,
|
||||
'--statistics'
|
||||
)
|
||||
|
||||
if ($DryRun) {
|
||||
Write-Host "$prefix"
|
||||
Write-Host (" " + $Archer + " " + ($argv -join ' '))
|
||||
$ok += $table
|
||||
continue
|
||||
}
|
||||
|
||||
$log = Join-Path $OutDir "$Schema.$table.log"
|
||||
$sw = [System.Diagnostics.Stopwatch]::StartNew()
|
||||
# archer 会往 stderr 打提示;PowerShell 在 EAP=Stop 下会把原生 stderr 当成终止性错误,
|
||||
# 这里临时降级为 Continue,并把 stdout+stderr 一起落日志。
|
||||
$prevEap = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$code = 0
|
||||
try {
|
||||
$attempt = 0
|
||||
while ($true) {
|
||||
$attempt++
|
||||
& $Archer @argv 2>&1 | Out-File -FilePath $log -Encoding utf8
|
||||
$code = $LASTEXITCODE
|
||||
if ($code -eq 0 -or $attempt -gt $Retries) { break }
|
||||
Write-Host "$prefix 第 $attempt 次失败(exit $code),重试中(archer 自动续跑)..." -ForegroundColor Yellow
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $prevEap
|
||||
}
|
||||
$sw.Stop()
|
||||
|
||||
if ($code -ne 0) {
|
||||
Write-Host "$prefix 失败(exit $code,耗时 $([int]$sw.Elapsed.TotalSeconds)s)—— 见 $log" -ForegroundColor Red
|
||||
Get-Content $log -Tail 5 | ForEach-Object { Write-Host " $_" }
|
||||
$failed += [pscustomobject]@{ Table = $table; Code = $code; Log = $log }
|
||||
continue
|
||||
}
|
||||
|
||||
# 汇总本次分片情况
|
||||
$manifest = Join-Path $tableDir 'manifest.jsonl'
|
||||
$shards = 0; $rows = 0; $bytes = 0
|
||||
if (Test-Path $manifest) {
|
||||
foreach ($line in Get-Content $manifest) {
|
||||
if (-not $line.Trim()) { continue }
|
||||
$e = $line | ConvertFrom-Json
|
||||
$shards++
|
||||
$rows += [int64]$e.rows
|
||||
$bytes += [int64]$e.bytes
|
||||
}
|
||||
}
|
||||
Write-Host ("$prefix 完成:{0} 分片, {1} 行, {2:N1} MB, {3}s" -f `
|
||||
$shards, $rows, ($bytes / 1MB), [int]$sw.Elapsed.TotalSeconds)
|
||||
$ok += $table
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 校验
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
if ($Verify -and -not $DryRun -and $ok.Count -gt 0) {
|
||||
Write-Host ''
|
||||
Write-Host '== 行数校验 =='
|
||||
foreach ($table in $ok) {
|
||||
$stateFile = Join-Path (Join-Path $schemaDir $table) 'state.json'
|
||||
if (-not (Test-Path $stateFile)) { continue }
|
||||
$state = Get-Content -Raw -Path $stateFile | ConvertFrom-Json
|
||||
$want = [int64]$state.rows_done
|
||||
$got = [int64](Invoke-MysqlQuery "SELECT COUNT(*) FROM ``$Schema``.``$table``" | Select-Object -First 1)
|
||||
if ($want -eq $got) {
|
||||
Write-Host (" {0,-34} {1,10} OK" -f $table, $got)
|
||||
}
|
||||
else {
|
||||
Write-Host (" {0,-34} 导出 {1} / 源 {2} <== DIFF" -f $table, $want, $got) -ForegroundColor Yellow
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host ''
|
||||
Write-Host ("完成:成功 {0},跳过 {1},失败 {2};输出 $schemaDir" -f $ok.Count, $skipped.Count, $failed.Count)
|
||||
if ($failed.Count -gt 0) {
|
||||
$failed | ForEach-Object { Write-Host (" 失败 {0} (exit {1}) 日志 {2}" -f $_.Table, $_.Code, $_.Log) }
|
||||
exit 1
|
||||
}
|
||||
exit 0
|
||||
Reference in new issue
Block a user