Renamed from arcdb: module path, cmd/archer, internal/archererr, docs, build scripts and ARCHER_* environment variables. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
214 lines
6.1 KiB
Batchfile
214 lines
6.1 KiB
Batchfile
@echo off
|
||
chcp 65001 >nul 2>&1
|
||
setlocal EnableDelayedExpansion
|
||
|
||
REM =====================================================================
|
||
REM export-all.cmd
|
||
REM
|
||
REM 用 archer 把某个 MySQL 库的「所有表」导出为按行数分片 + gzip 压缩的 dump。
|
||
REM
|
||
REM 用法:
|
||
REM scripts\export-all.cmd
|
||
REM 使用下面「配置区」的默认值
|
||
REM
|
||
REM scripts\export-all.cmd DSN OUTDIR SHARD_ROWS CHUNK_SIZE
|
||
REM 例: scripts\export-all.cmd ^
|
||
REM "root:pass@tcp(10.0.0.5:3306)/app" "D:\dump" 500000 5000
|
||
REM
|
||
REM 产物:
|
||
REM <OUTDIR>\<表名>\000001.<lo>-<hi>.dump.gz 分片文件(按主键连续、不重叠)
|
||
REM <OUTDIR>\<表名>\manifest.jsonl 每片的行数/字节/sha256
|
||
REM <OUTDIR>\<表名>\state.json 断点续跑状态
|
||
REM <OUTDIR>\<表名>.log 该表 archer 的完整输出
|
||
REM
|
||
REM 断点续跑:
|
||
REM 用同样的参数再跑一遍即可,archer 从 state.json 继续;
|
||
REM state.json 里 complete=true 的表会被跳过。
|
||
REM 想重导某张表,先删掉它的目录。
|
||
REM
|
||
REM 依赖:
|
||
REM dist\archer_windows_amd64.exe (scripts\build.ps1 构建)
|
||
REM mysql.exe (只用来枚举表名,需在 PATH 里)
|
||
REM =====================================================================
|
||
|
||
REM ------------------------------ 配置区 ------------------------------
|
||
REM 源库连接。改动源库时,SRC 与下面 5 个变量要一起改
|
||
REM (mysql 客户端要从后者取参数去枚举表名)。
|
||
set "SRC=root:xyTyfxhn4h@tcp(s1.void01.com:3306)/lexis"
|
||
set "DB=lexis"
|
||
set "DB_HOST=s1.void01.com"
|
||
set "DB_PORT=3306"
|
||
set "DB_USER=root"
|
||
set "DB_PASS=xyTyfxhn4h"
|
||
|
||
REM 输出根目录(每张表会在其下建一个同名子目录)
|
||
set "OUTDIR=%~dp0..\export"
|
||
|
||
REM 每个分片文件的行数;小于表行数时只会得到 1 个分片
|
||
set "SHARD_ROWS=1000000"
|
||
|
||
REM 每次 SELECT / 事务 / checkpoint 的行数,与分片大小无关
|
||
set "CHUNK_SIZE=5000"
|
||
|
||
REM 输出格式 dump|csv|jsonl 与压缩方式 none|gzip|zstd
|
||
set "FMT=dump"
|
||
set "COMPRESS=gzip"
|
||
|
||
REM 只导指定表(逗号分隔);留空表示导全部
|
||
set "ONLY_TABLES="
|
||
|
||
REM 排除表名通配符,如 tmp_* REM_*;留空表示不排除
|
||
set "EXCLUDE="
|
||
REM --------------------------------------------------------------------
|
||
|
||
REM 命令行覆盖
|
||
if not "%~1"=="" set "SRC=%~1"
|
||
if not "%~2"=="" set "OUTDIR=%~2"
|
||
if not "%~3"=="" set "SHARD_ROWS=%~3"
|
||
if not "%~4"=="" set "CHUNK_SIZE=%~4"
|
||
|
||
set "EXE=%~dp0..\dist\archer_windows_amd64.exe"
|
||
if not exist "%EXE%" (
|
||
echo [错误] 找不到 archer: %EXE%
|
||
echo 先运行 scripts\build.ps1 构建
|
||
goto :fail
|
||
)
|
||
|
||
where mysql.exe >nul 2>&1
|
||
if errorlevel 1 (
|
||
echo [错误] PATH 中找不到 mysql.exe,无法枚举表名
|
||
echo 把 mysql 的 bin 目录加进 PATH,或用 ARCHER 逐表手写命令
|
||
goto :fail
|
||
)
|
||
|
||
if not exist "%OUTDIR%" mkdir "%OUTDIR%" >nul 2>&1
|
||
if not exist "%OUTDIR%" (
|
||
echo [错误] 无法创建输出目录: %OUTDIR%
|
||
goto :fail
|
||
)
|
||
|
||
REM mysql 用环境变量传密码,避免出现在命令行/进程列表里
|
||
set "MYSQL_PWD=%DB_PASS%"
|
||
set "MYSQL_CMD=mysql.exe --protocol=TCP -h%DB_HOST% -P%DB_PORT% -u%DB_USER% --default-character-set=utf8mb4 -N -B"
|
||
|
||
echo.
|
||
echo archer : %EXE%
|
||
echo schema : %DB% (%DB_HOST%:%DB_PORT%)
|
||
echo out dir : %OUTDIR%
|
||
echo shard : %FMT% + %COMPRESS%, %SHARD_ROWS% rows/shard, chunk %CHUNK_SIZE%
|
||
echo.
|
||
|
||
REM ---------------------------- 表清单 ----------------------------
|
||
if defined ONLY_TABLES (
|
||
set "TABLE_LIST=%ONLY_TABLES:,= %"
|
||
set "TABLE_LIST=!TABLE_LIST: =!|"
|
||
) else (
|
||
set "TABLE_LIST="
|
||
for /f "usebackq delims=" %%t in (`%MYSQL_CMD% -e "SELECT table_name FROM information_schema.tables WHERE table_schema='%DB%' AND table_type='BASE TABLE' ORDER BY table_name" 2^>nul`) do (
|
||
set "TABLE_LIST=!TABLE_LIST!%%t|"
|
||
)
|
||
)
|
||
|
||
if "!TABLE_LIST!"=="" (
|
||
echo [错误] 没有取到任何表,检查库名/账号/网络
|
||
goto :fail
|
||
)
|
||
|
||
REM ---------------------------- 逐表导出 ----------------------------
|
||
set "OK=0"
|
||
set "SKIP=0"
|
||
set "FAILED="
|
||
set /a IDX=0
|
||
|
||
for %%t in ("!TABLE_LIST:|=" "!") do (
|
||
set "T=%%~t"
|
||
if not "!T!"=="" (
|
||
call :one_table "!T!"
|
||
)
|
||
)
|
||
|
||
echo.
|
||
if defined FAILED (
|
||
echo 失败的表: %FAILED%
|
||
echo.
|
||
echo 完成: 成功 %OK%,跳过 %SKIP%,失败若干
|
||
goto :fail
|
||
)
|
||
echo 完成: 成功 %OK%,跳过 %SKIP%,失败 0
|
||
endlocal
|
||
exit /b 0
|
||
|
||
REM ====================================================================
|
||
REM 单表导出
|
||
REM ====================================================================
|
||
:one_table
|
||
set "T=%~1"
|
||
set /a IDX+=1
|
||
|
||
if defined EXCLUDE (
|
||
REM 简单通配:把 * 替换成批处理的 .*
|
||
set "PAT=!EXCLUDE:*=.!"
|
||
for %%p in (!PAT!) do (
|
||
for /f "delims=" %%m in ("!T!") do (
|
||
if "%%m"=="%%m" (
|
||
REM 用 findstr 做通配匹配
|
||
)
|
||
)
|
||
)
|
||
echo !T! | findstr /r /i /c:"^!EXCLUDE:\*=.!$" >nul 2>&1
|
||
if not errorlevel 1 (
|
||
echo [!IDX!] !T! 跳过(被 EXCLUDE 匹配)
|
||
goto :eof
|
||
)
|
||
)
|
||
|
||
set "TDIR=%OUTDIR%\!T!"
|
||
|
||
REM 已完成则跳过
|
||
if exist "!TDIR!\state.json" (
|
||
findstr /c:"\"complete\": true" "!TDIR!\state.json" >nul 2>&1
|
||
if not errorlevel 1 (
|
||
echo [!IDX!] !T! 跳过(已完成)
|
||
set /a SKIP+=1
|
||
goto :eof
|
||
)
|
||
echo [!IDX!] !T! 续跑
|
||
)
|
||
|
||
set "LOG=%OUTDIR%\!T!.log"
|
||
set "START=%TIME%"
|
||
|
||
"%EXE%" ^
|
||
--source "%SRC%" ^
|
||
--table "!T!" ^
|
||
--where "1=1" ^
|
||
--file "!TDIR!" ^
|
||
--format %FMT% ^
|
||
--compress %COMPRESS% ^
|
||
--shard-by count ^
|
||
--shard-rows %SHARD_ROWS% ^
|
||
--chunk-size %CHUNK_SIZE% ^
|
||
--statistics > "!LOG!" 2>&1
|
||
|
||
if errorlevel 1 (
|
||
echo [!IDX!] !T! 失败,日志: !LOG!
|
||
for /f "delims=" %%l in ('findstr /c:"archer:" "!LOG!" 2^>nul') do echo %%l
|
||
set "FAILED=!FAILED! !T!"
|
||
goto :eof
|
||
)
|
||
|
||
REM 统计分片数
|
||
set "SHARDS=0"
|
||
if exist "!TDIR!\manifest.jsonl" (
|
||
for /f %%n in ('type "!TDIR!\manifest.jsonl" 2^>nul ^| find /c /v ""') do set "SHARDS=%%n"
|
||
)
|
||
|
||
echo [!IDX!] !T! 完成: !SHARDS! 片
|
||
set /a OK+=1
|
||
goto :eof
|
||
|
||
REM ====================================================================
|
||
:fail
|
||
endlocal
|
||
exit /b 1
|