feat(importer): 添加 Lexis 词书导出工具链
三步主流程(下载目录 -> 下载词书并解密导出 -> 下载并解密 ZPK), 以及可独立调用的 SQLCipher2 / ZPK 解密工具与词根、近义词导出脚本。 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
a39ba9081e
commit
f030334243
9 files changed
+3517
No files matched your search
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env python3
|
||||
"""逐个调用 Lexis 三步导出流程。
|
||||
|
||||
步骤1:下载词书目录
|
||||
步骤2:逐个下载所有词书、解密并导出单词
|
||||
步骤3:逐个下载所有词书所有单词的 ZPK 并解密为 JSONL
|
||||
|
||||
所有产物统一输出到 output/,每步产出均为可读明文。
|
||||
|
||||
用法:
|
||||
python run.py
|
||||
python run.py --wordbook-workers 16 --zpk-workers 96 --zpk-timeout 6
|
||||
|
||||
任一步骤失败(非零退出码)即停止,不再执行后续步骤。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
if hasattr(sys.stderr, "reconfigure"):
|
||||
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def run_step(name: str, step_args: list[str]) -> int:
|
||||
print(f"\n{'=' * 64}\n[INFO] {name}:status=started\n{'=' * 64}", flush=True)
|
||||
process = subprocess.Popen([sys.executable, *step_args], cwd=ROOT)
|
||||
try:
|
||||
returncode = process.wait()
|
||||
except KeyboardInterrupt:
|
||||
print(f"\n[{name}] 收到 Ctrl+C,正在终止子进程...", file=sys.stderr, flush=True)
|
||||
process.terminate()
|
||||
try:
|
||||
returncode = process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
returncode = process.wait()
|
||||
return 130
|
||||
if returncode != 0:
|
||||
print(f"\n[{name}] 退出码 {returncode}", file=sys.stderr, flush=True)
|
||||
return returncode
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="逐个执行 Lexis 三步导出流程")
|
||||
parser.add_argument("--wordbook-workers", type=int, default=16, help="步骤2 下载词书的并发数")
|
||||
parser.add_argument("--zpk-workers", type=int, default=96, help="步骤3 下载 ZPK 的并发数")
|
||||
parser.add_argument("--zpk-timeout", type=float, default=6, help="步骤3 单个 CDN 请求超时秒数")
|
||||
parser.add_argument("--progress-every", type=int, default=1000, help="步骤3 进度打印间隔")
|
||||
args = parser.parse_args()
|
||||
|
||||
steps = [
|
||||
("步骤 1/3:下载词书目录", ["src/fetch-book-catalog.py"]),
|
||||
(
|
||||
"步骤 2/3:逐个下载所有词书(下载 -> 解密 -> 导出单词)",
|
||||
["src/download-wordbooks.py", "--workers", str(args.wordbook_workers)],
|
||||
),
|
||||
(
|
||||
"步骤 3/3:逐个下载所有词书所有单词的 ZPK 并解密为 JSONL",
|
||||
[
|
||||
"src/download-zpk.py",
|
||||
"--workers", str(args.zpk_workers),
|
||||
"--timeout", str(args.zpk_timeout),
|
||||
"--progress-every", str(args.progress_every),
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
for name, step_args in steps:
|
||||
code = run_step(name, step_args)
|
||||
if code != 0:
|
||||
print(f"\n流程中断:'{name}' 失败(退出码 {code})", file=sys.stderr)
|
||||
return code
|
||||
|
||||
print("\n[INFO] 全部流程执行完成:output=output/", flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user