三步主流程(下载目录 -> 下载词书并解密导出 -> 下载并解密 ZPK), 以及可独立调用的 SQLCipher2 / ZPK 解密工具与词根、近义词导出脚本。 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
460 lines
17 KiB
Python
460 lines
17 KiB
Python
#!/usr/bin/env python3
|
||
"""步骤2:逐个下载所有词书,解包、解密 SQLCipher2 DB 并导出单词与 zpk 字段。
|
||
|
||
每本词书走完整链路:下载 zip -> 校验 MD5 -> 解包 -> 解密 -> 导出单词 CSV。
|
||
解密逻辑复用独立工具 decrypt-sqlcipher2-db.py(可单独调用)。
|
||
|
||
已下载的 DB 自动跳过(可用 --force 重下),已解密的 DB 按 mtime 复用(可用 --force-decrypt 重解)。
|
||
|
||
输出(默认统一到 output/):
|
||
- output/databases/LangEasyLexisV3_<code>.db 原始加密 DB(中间缓存)
|
||
- output/decrypted-databases/LangEasyLexisV3_<code>.sqlite 解密后 DB(可读)
|
||
- output/wordbook-words/<code>.csv 每本词书单词(可读)
|
||
- output/all-wordbook-words.csv 全量合并词表(可读,步骤3 的输入)
|
||
- output/wordbook-download-report.csv 下载报告
|
||
- output/wordbook-word-summary.csv 导出报告
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import base64
|
||
import csv
|
||
import hashlib
|
||
import importlib.util
|
||
import json
|
||
import os
|
||
import re
|
||
import sqlite3
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
import zipfile
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from pathlib import Path
|
||
from typing import Iterable
|
||
from urllib.error import HTTPError, URLError
|
||
from urllib.request import Request, urlopen
|
||
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
OUT_DIR = ROOT / "output"
|
||
DEFAULT_CATALOG = OUT_DIR / "book-catalog.json"
|
||
WORDBOOK_DIR = OUT_DIR / "wordbooks"
|
||
DB_DIR = WORDBOOK_DIR / "original"
|
||
DECRYPTED_DIR = WORDBOOK_DIR / "decrypted"
|
||
|
||
DECRYPTOR_PATH = ROOT / "src" / "decrypt-sqlcipher2-db.py"
|
||
|
||
BASE_STATIC_URLS = [
|
||
"http://static.beingfine.cn",
|
||
"https://hwstatic.beingfine.cn",
|
||
"https://ccastatic.beingfine.cn",
|
||
]
|
||
REFERER = "https://app.bbdc.cn"
|
||
USER_AGENT = "LangEasyLexis/5.11.1"
|
||
|
||
WORD_FIELDS = (
|
||
"book_id",
|
||
"book_code",
|
||
"book_name",
|
||
"book_count",
|
||
"book_exam",
|
||
"book_version",
|
||
"id",
|
||
"word",
|
||
"main_word",
|
||
"seq_book",
|
||
"seq_az",
|
||
"seq_freq",
|
||
"seq_rand",
|
||
"zpk",
|
||
)
|
||
|
||
SUMMARY_FIELDS = (
|
||
"book_id",
|
||
"book_code",
|
||
"book_name",
|
||
"book_count",
|
||
"exported_count",
|
||
"status",
|
||
"table",
|
||
"local_db",
|
||
"decrypted_db",
|
||
"export_csv",
|
||
"error",
|
||
)
|
||
|
||
DOWNLOAD_REPORT_FIELDS = ("code", "status", "bytes", "url")
|
||
|
||
|
||
def load_decrypt_db():
|
||
"""加载独立解密工具 decrypt-sqlcipher2-db.py 的 decrypt_db 函数。"""
|
||
spec = importlib.util.spec_from_file_location("decrypt_sqlcipher2_db", DECRYPTOR_PATH)
|
||
if spec is None or spec.loader is None:
|
||
raise RuntimeError(f"Cannot load decryptor: {DECRYPTOR_PATH}")
|
||
module = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(module)
|
||
return module.decrypt_db
|
||
|
||
|
||
def load_books(path: Path) -> list[dict]:
|
||
with path.open("r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
if isinstance(data, dict) and isinstance(data.get("books"), list):
|
||
return data["books"]
|
||
if isinstance(data, list):
|
||
return data
|
||
raise ValueError(f"Unsupported catalog shape: {path}")
|
||
|
||
|
||
def book_url(book: dict, base_url: str) -> str:
|
||
code = book["code"]
|
||
version = str(book.get("version") or 0)
|
||
db_name = f"LangEasyLexisV3_{code}.db"
|
||
return f"{base_url}/dicts/v3/{code}/{version}/{db_name}.zip"
|
||
|
||
|
||
def request_url(url: str):
|
||
req = Request(url, headers={"Referer": REFERER, "User-Agent": USER_AGENT})
|
||
return urlopen(req, timeout=45)
|
||
|
||
|
||
def md5_b64(path: Path) -> str:
|
||
h = hashlib.md5()
|
||
with path.open("rb") as f:
|
||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||
h.update(chunk)
|
||
return base64.b64encode(h.digest()).decode("ascii")
|
||
|
||
|
||
def safe_code_filename(code: object) -> str:
|
||
text = str(code or "UNKNOWN")
|
||
return re.sub(r"[^A-Za-z0-9_.-]+", "_", text).strip("._") or "UNKNOWN"
|
||
|
||
|
||
def quote_identifier(name: str) -> str:
|
||
return '"' + name.replace('"', '""') + '"'
|
||
|
||
|
||
def list_tables(conn: sqlite3.Connection) -> list[str]:
|
||
return [
|
||
row[0]
|
||
for row in conn.execute(
|
||
"select name from sqlite_master where type='table' and name not like 'sqlite_%' order by name"
|
||
)
|
||
]
|
||
|
||
|
||
def table_columns(conn: sqlite3.Connection, table: str) -> set[str]:
|
||
return {row[1] for row in conn.execute(f"pragma table_info({quote_identifier(table)})")}
|
||
|
||
|
||
def choose_word_table(conn: sqlite3.Connection, code: str) -> str:
|
||
tables = list_tables(conn)
|
||
preferred = f"LangEasyLexisV3_{code}"
|
||
if preferred in tables:
|
||
return preferred
|
||
|
||
for table in tables:
|
||
if {"id", "word"}.issubset(table_columns(conn, table)):
|
||
return table
|
||
|
||
raise RuntimeError(f"no word table found; tables={tables}")
|
||
|
||
|
||
def iter_words(conn: sqlite3.Connection, table: str) -> Iterable[sqlite3.Row]:
|
||
conn.row_factory = sqlite3.Row
|
||
cols = table_columns(conn, table)
|
||
required = {"id", "word", "main_word", "seq_book", "seq_az", "seq_freq", "seq_rand", "zpk"}
|
||
missing = sorted(required - cols)
|
||
if missing:
|
||
raise RuntimeError(f"table {table} missing columns: {missing}")
|
||
|
||
query = (
|
||
f"select id, word, main_word, seq_book, seq_az, seq_freq, seq_rand, zpk "
|
||
f"from {quote_identifier(table)} order by seq_book, id"
|
||
)
|
||
return conn.execute(query)
|
||
|
||
|
||
def download_book_db(book: dict, force: bool) -> dict:
|
||
"""下载单本词书 DB zip 并解包,返回下载状态 dict。"""
|
||
code = book["code"]
|
||
db_path = DB_DIR / f"LangEasyLexisV3_{code}.db"
|
||
|
||
if db_path.exists() and not force:
|
||
return {"status": "skip-db-exists", "url": "", "bytes": db_path.stat().st_size}
|
||
|
||
last_error = ""
|
||
for base in BASE_STATIC_URLS:
|
||
url = book_url(book, base)
|
||
try:
|
||
with request_url(url) as resp:
|
||
status = getattr(resp, "status", 200)
|
||
if status != 200:
|
||
last_error = f"http-{status}"
|
||
continue
|
||
|
||
tmp_file = tempfile.NamedTemporaryFile(prefix=f"lexis-{code}-", suffix=".zip", delete=False)
|
||
tmp_path = Path(tmp_file.name)
|
||
with tmp_file as out:
|
||
while True:
|
||
chunk = resp.read(1024 * 1024)
|
||
if not chunk:
|
||
break
|
||
out.write(chunk)
|
||
|
||
content_md5 = resp.headers.get("Content-MD5")
|
||
if content_md5 and md5_b64(tmp_path) != content_md5:
|
||
tmp_path.unlink(missing_ok=True)
|
||
last_error = "md5-mismatch"
|
||
continue
|
||
|
||
DB_DIR.mkdir(parents=True, exist_ok=True)
|
||
with zipfile.ZipFile(tmp_path) as zf:
|
||
names = zf.namelist()
|
||
wanted = f"LangEasyLexisV3_{code}.db"
|
||
member = wanted if wanted in names else names[0]
|
||
zf.extract(member, DB_DIR)
|
||
extracted = DB_DIR / member
|
||
if extracted != db_path:
|
||
extracted.replace(db_path)
|
||
|
||
tmp_path.unlink(missing_ok=True)
|
||
return {"status": "downloaded", "url": url, "bytes": db_path.stat().st_size}
|
||
except HTTPError as e:
|
||
last_error = f"http-{e.code}"
|
||
except (URLError, TimeoutError, zipfile.BadZipFile, OSError) as e:
|
||
last_error = type(e).__name__
|
||
finally:
|
||
if "tmp_path" in locals():
|
||
tmp_path.unlink(missing_ok=True)
|
||
|
||
return {"status": f"failed:{last_error}", "url": "", "bytes": 0}
|
||
|
||
|
||
def export_book_words(row: dict, decrypt_db, force_decrypt: bool) -> dict:
|
||
"""解密并导出一本词书的单词,返回导出 summary dict。"""
|
||
code = str(row.get("code") or "")
|
||
src = DB_DIR / f"LangEasyLexisV3_{code}.db"
|
||
if not src.exists():
|
||
return summary(row, "missing_db", "", src, Path(), Path(), 0, f"not found: {src}")
|
||
|
||
export_csv = DECRYPTED_DIR / f"LangEasyLexisV3_{safe_code_filename(code)}.csv"
|
||
DECRYPTED_DIR.mkdir(parents=True, exist_ok=True)
|
||
|
||
# 解密 DB:已存在且比源新则复用
|
||
# SQLite 仅作为临时解密载体,不作为 output 产物。
|
||
# Windows 可能在 SQLite 连接关闭后仍短暂持有文件句柄;清理失败不能影响导出结果。
|
||
with tempfile.TemporaryDirectory(prefix="lexis-decrypted-", ignore_cleanup_errors=True) as temp_dir:
|
||
decrypted = Path(temp_dir) / src.with_suffix(".sqlite").name
|
||
decrypt_db(src, decrypted)
|
||
with sqlite3.connect(decrypted) as conn:
|
||
table = choose_word_table(conn, code)
|
||
count = 0
|
||
with export_csv.open("w", encoding="utf-8-sig", newline="") as fh:
|
||
writer = csv.DictWriter(fh, fieldnames=WORD_FIELDS)
|
||
writer.writeheader()
|
||
for word_row in iter_words(conn, table):
|
||
writer.writerow(
|
||
{
|
||
"book_id": row.get("id", ""),
|
||
"book_code": code,
|
||
"book_name": row.get("name", ""),
|
||
"book_count": row.get("count", ""),
|
||
"book_exam": row.get("exam", ""),
|
||
"book_version": row.get("version", ""),
|
||
"id": word_row["id"],
|
||
"word": word_row["word"],
|
||
"main_word": word_row["main_word"],
|
||
"seq_book": word_row["seq_book"],
|
||
"seq_az": word_row["seq_az"],
|
||
"seq_freq": word_row["seq_freq"],
|
||
"seq_rand": word_row["seq_rand"],
|
||
"zpk": word_row["zpk"],
|
||
}
|
||
)
|
||
count += 1
|
||
|
||
return summary(row, "ok", table, src, decrypted, export_csv, count, "")
|
||
|
||
|
||
def summary(
|
||
row: dict,
|
||
status: str,
|
||
table: str,
|
||
src: Path,
|
||
decrypted: Path,
|
||
export_csv: Path,
|
||
exported_count: int,
|
||
error: str,
|
||
) -> dict:
|
||
return {
|
||
"book_id": row.get("id", ""),
|
||
"book_code": row.get("code", ""),
|
||
"book_name": row.get("name", ""),
|
||
"book_count": row.get("count", ""),
|
||
"exported_count": exported_count,
|
||
"status": status,
|
||
"table": table,
|
||
"local_db": str(src) if src else "",
|
||
"decrypted_db": str(decrypted) if decrypted else "",
|
||
"export_csv": str(export_csv) if export_csv else "",
|
||
"error": error,
|
||
}
|
||
|
||
|
||
def process_one(book: dict, decrypt_db, force_download: bool, force_decrypt: bool) -> dict:
|
||
"""处理一本词书:下载 + 解密 + 导出,返回合并结果 dict。"""
|
||
code = book["code"]
|
||
result = {
|
||
"code": code,
|
||
"dl_status": "",
|
||
"dl_url": "",
|
||
"dl_bytes": 0,
|
||
"export": None,
|
||
}
|
||
|
||
if book.get("url") == "*":
|
||
result["dl_status"] = "skip-virtual-book"
|
||
return result
|
||
|
||
dl = download_book_db(book, force_download)
|
||
result["dl_status"] = dl["status"]
|
||
result["dl_url"] = dl["url"]
|
||
result["dl_bytes"] = dl["bytes"]
|
||
|
||
if dl["status"] not in {"downloaded", "skip-db-exists"}:
|
||
result["export"] = summary(book, "download_failed", "", DB_DIR / f"LangEasyLexisV3_{code}.db", Path(), Path(), 0, dl["status"])
|
||
return result
|
||
|
||
result["export"] = export_book_words(book, decrypt_db, force_decrypt)
|
||
return result
|
||
|
||
|
||
def write_download_report(results: list[dict], path: Path) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
with path.open("w", newline="", encoding="utf-8-sig") as f:
|
||
writer = csv.DictWriter(f, fieldnames=DOWNLOAD_REPORT_FIELDS)
|
||
writer.writeheader()
|
||
for r in results:
|
||
writer.writerow(
|
||
{"code": r["code"], "status": r["dl_status"], "bytes": r["dl_bytes"], "url": r["dl_url"]}
|
||
)
|
||
|
||
|
||
def write_summary(summaries: list[dict], path: Path) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
with path.open("w", encoding="utf-8-sig", newline="") as fh:
|
||
writer = csv.DictWriter(fh, fieldnames=SUMMARY_FIELDS)
|
||
writer.writeheader()
|
||
writer.writerows(summaries)
|
||
|
||
|
||
def write_combined(summaries: list[dict], path: Path) -> int:
|
||
total = 0
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
with path.open("w", encoding="utf-8-sig", newline="") as out_fh:
|
||
writer = csv.DictWriter(out_fh, fieldnames=WORD_FIELDS)
|
||
writer.writeheader()
|
||
for item in summaries:
|
||
if item["status"] != "ok":
|
||
continue
|
||
with Path(item["export_csv"]).open("r", encoding="utf-8-sig", newline="") as in_fh:
|
||
reader = csv.DictReader(in_fh)
|
||
for row in reader:
|
||
writer.writerow(row)
|
||
total += 1
|
||
return total
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description="逐个下载所有词书:下载 -> 解密 -> 导出单词")
|
||
parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
|
||
parser.add_argument("--force", action="store_true", help="重新下载已存在的词书 DB")
|
||
parser.add_argument("--force-decrypt", action="store_true", help="重新解密已存在的解密 DB")
|
||
parser.add_argument("--limit", type=int, default=0, help="最多处理 N 本词书(冒烟测试)")
|
||
parser.add_argument("--sleep", type=float, default=0.0, help="单 worker 模式下每本之间 sleep 秒数")
|
||
parser.add_argument("--workers", type=int, default=1, help="并行下载词书的 worker 数")
|
||
args = parser.parse_args()
|
||
|
||
if not args.catalog.exists():
|
||
print(f"词书目录不存在:{args.catalog},请先运行 src/fetch-book-catalog.py", file=sys.stderr)
|
||
return 1
|
||
|
||
decrypt_db = load_decrypt_db()
|
||
books = [row for row in load_books(args.catalog) if row.get("url") != "*"]
|
||
if args.limit:
|
||
books = books[: args.limit]
|
||
|
||
total = len(books)
|
||
results: list[dict] = []
|
||
done = 0
|
||
workers = max(1, args.workers)
|
||
|
||
if workers == 1:
|
||
for book in books:
|
||
result = process_one(book, decrypt_db, args.force, args.force_decrypt)
|
||
results.append(result)
|
||
done += 1
|
||
exp = result["export"]
|
||
print(
|
||
f"[INFO] [步骤 2/3] 处理进度:current={done}/{total},percent={done / total * 100:.1f}%,"
|
||
f"code={result['code']},download={result['dl_status']},"
|
||
f"decrypt={exp['status'] if exp else '-'},words={exp['exported_count'] if exp else '-'}",
|
||
flush=True,
|
||
)
|
||
if args.sleep:
|
||
time.sleep(args.sleep)
|
||
else:
|
||
executor = ThreadPoolExecutor(max_workers=workers)
|
||
futures = [executor.submit(process_one, book, decrypt_db, args.force, args.force_decrypt) for book in books]
|
||
try:
|
||
for future in as_completed(futures):
|
||
result = future.result()
|
||
results.append(result)
|
||
done += 1
|
||
exp = result["export"]
|
||
print(
|
||
f"[INFO] [步骤 2/3] 处理进度:current={done}/{total},percent={done / total * 100:.1f}%,"
|
||
f"code={result['code']},download={result['dl_status']},"
|
||
f"decrypt={exp['status'] if exp else '-'},words={exp['exported_count'] if exp else '-'}",
|
||
flush=True,
|
||
)
|
||
except KeyboardInterrupt:
|
||
for future in futures:
|
||
future.cancel()
|
||
executor.shutdown(wait=False, cancel_futures=True)
|
||
print("\n收到 Ctrl+C,已取消未开始的词书任务。", file=sys.stderr, flush=True)
|
||
os._exit(130)
|
||
else:
|
||
executor.shutdown(wait=True)
|
||
|
||
summaries = [r["export"] for r in results if r["export"] is not None]
|
||
|
||
ok_dl = sum(1 for r in results if r["dl_status"] in {"downloaded", "skip-db-exists"})
|
||
ok_export = sum(1 for s in summaries if s["status"] == "ok")
|
||
failed_dl = len(results) - ok_dl
|
||
failed_export = len(summaries) - ok_export
|
||
exported_total = sum(int(s["exported_count"] or 0) for s in summaries)
|
||
|
||
print(f"[INFO] [步骤 2/3] 处理完成:total={len(results)},success={ok_export},failed={failed_export},words={exported_total}")
|
||
failures = []
|
||
for result in results:
|
||
export = result.get("export") or {}
|
||
if result["dl_status"] not in {"downloaded", "skip-db-exists"}:
|
||
failures.append(f"{result['code']}: 下载 {result['dl_status']}")
|
||
elif export.get("status") != "ok":
|
||
failures.append(f"{result['code']}: 解密/导出 {export.get('error') or export.get('status')}")
|
||
for failure in failures[:20]:
|
||
print(f"失败项:{failure}", file=sys.stderr)
|
||
if len(failures) > 20:
|
||
print(f"失败项:另有 {len(failures) - 20} 项未显示", file=sys.stderr)
|
||
# 单本失败不阻断后续流程;步骤 3 继续处理已成功导出的词书。
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|