#!/usr/bin/env python3 """步骤2:逐个下载所有词书,解包、解密 SQLCipher2 DB 并导出单词与 zpk 字段。 每本词书走完整链路:下载 zip -> 校验 MD5 -> 解包 -> 解密 -> 导出单词 CSV。 解密逻辑复用独立工具 decrypt-sqlcipher2-db.py(可单独调用)。 已下载的 DB 自动跳过(可用 --force 重下),已解密的 DB 按 mtime 复用(可用 --force-decrypt 重解)。 输出(默认统一到 output/): - output/databases/LangEasyLexisV3_.db 原始加密 DB(中间缓存) - output/decrypted-databases/LangEasyLexisV3_.sqlite 解密后 DB(可读) - output/wordbook-words/.csv 每本词书单词(可读) - output/all-wordbook-words.csv 全量合并词表(可读,步骤3 的输入) - output/wordbook-download-report.csv 下载报告 - output/wordbook-word-summary.csv 导出报告 """ from __future__ import annotations import argparse import base64 import csv import hashlib import importlib.util import json import os import re import sqlite3 import sys import tempfile import time import zipfile from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import Iterable from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen ROOT = Path(__file__).resolve().parents[1] OUT_DIR = ROOT / "output" DEFAULT_CATALOG = OUT_DIR / "book-catalog.json" WORDBOOK_DIR = OUT_DIR / "wordbooks" DB_DIR = WORDBOOK_DIR / "original" DECRYPTED_DIR = WORDBOOK_DIR / "decrypted" DECRYPTOR_PATH = ROOT / "src" / "decrypt-sqlcipher2-db.py" BASE_STATIC_URLS = [ "http://static.beingfine.cn", "https://hwstatic.beingfine.cn", "https://ccastatic.beingfine.cn", ] REFERER = "https://app.bbdc.cn" USER_AGENT = "LangEasyLexis/5.11.1" WORD_FIELDS = ( "book_id", "book_code", "book_name", "book_count", "book_exam", "book_version", "id", "word", "main_word", "seq_book", "seq_az", "seq_freq", "seq_rand", "zpk", ) SUMMARY_FIELDS = ( "book_id", "book_code", "book_name", "book_count", "exported_count", "status", "table", "local_db", "decrypted_db", "export_csv", "error", ) DOWNLOAD_REPORT_FIELDS = ("code", "status", "bytes", "url") def load_decrypt_db(): """加载独立解密工具 decrypt-sqlcipher2-db.py 的 decrypt_db 函数。""" spec = importlib.util.spec_from_file_location("decrypt_sqlcipher2_db", DECRYPTOR_PATH) if spec is None or spec.loader is None: raise RuntimeError(f"Cannot load decryptor: {DECRYPTOR_PATH}") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module.decrypt_db def load_books(path: Path) -> list[dict]: with path.open("r", encoding="utf-8") as f: data = json.load(f) if isinstance(data, dict) and isinstance(data.get("books"), list): return data["books"] if isinstance(data, list): return data raise ValueError(f"Unsupported catalog shape: {path}") def book_url(book: dict, base_url: str) -> str: code = book["code"] version = str(book.get("version") or 0) db_name = f"LangEasyLexisV3_{code}.db" return f"{base_url}/dicts/v3/{code}/{version}/{db_name}.zip" def request_url(url: str): req = Request(url, headers={"Referer": REFERER, "User-Agent": USER_AGENT}) return urlopen(req, timeout=45) def md5_b64(path: Path) -> str: h = hashlib.md5() with path.open("rb") as f: for chunk in iter(lambda: f.read(1024 * 1024), b""): h.update(chunk) return base64.b64encode(h.digest()).decode("ascii") def safe_code_filename(code: object) -> str: text = str(code or "UNKNOWN") return re.sub(r"[^A-Za-z0-9_.-]+", "_", text).strip("._") or "UNKNOWN" def quote_identifier(name: str) -> str: return '"' + name.replace('"', '""') + '"' def list_tables(conn: sqlite3.Connection) -> list[str]: return [ row[0] for row in conn.execute( "select name from sqlite_master where type='table' and name not like 'sqlite_%' order by name" ) ] def table_columns(conn: sqlite3.Connection, table: str) -> set[str]: return {row[1] for row in conn.execute(f"pragma table_info({quote_identifier(table)})")} def choose_word_table(conn: sqlite3.Connection, code: str) -> str: tables = list_tables(conn) preferred = f"LangEasyLexisV3_{code}" if preferred in tables: return preferred for table in tables: if {"id", "word"}.issubset(table_columns(conn, table)): return table raise RuntimeError(f"no word table found; tables={tables}") def iter_words(conn: sqlite3.Connection, table: str) -> Iterable[sqlite3.Row]: conn.row_factory = sqlite3.Row cols = table_columns(conn, table) required = {"id", "word", "main_word", "seq_book", "seq_az", "seq_freq", "seq_rand", "zpk"} missing = sorted(required - cols) if missing: raise RuntimeError(f"table {table} missing columns: {missing}") query = ( f"select id, word, main_word, seq_book, seq_az, seq_freq, seq_rand, zpk " f"from {quote_identifier(table)} order by seq_book, id" ) return conn.execute(query) def download_book_db(book: dict, force: bool) -> dict: """下载单本词书 DB zip 并解包,返回下载状态 dict。""" code = book["code"] db_path = DB_DIR / f"LangEasyLexisV3_{code}.db" if db_path.exists() and not force: return {"status": "skip-db-exists", "url": "", "bytes": db_path.stat().st_size} last_error = "" for base in BASE_STATIC_URLS: url = book_url(book, base) try: with request_url(url) as resp: status = getattr(resp, "status", 200) if status != 200: last_error = f"http-{status}" continue tmp_file = tempfile.NamedTemporaryFile(prefix=f"lexis-{code}-", suffix=".zip", delete=False) tmp_path = Path(tmp_file.name) with tmp_file as out: while True: chunk = resp.read(1024 * 1024) if not chunk: break out.write(chunk) content_md5 = resp.headers.get("Content-MD5") if content_md5 and md5_b64(tmp_path) != content_md5: tmp_path.unlink(missing_ok=True) last_error = "md5-mismatch" continue DB_DIR.mkdir(parents=True, exist_ok=True) with zipfile.ZipFile(tmp_path) as zf: names = zf.namelist() wanted = f"LangEasyLexisV3_{code}.db" member = wanted if wanted in names else names[0] zf.extract(member, DB_DIR) extracted = DB_DIR / member if extracted != db_path: extracted.replace(db_path) tmp_path.unlink(missing_ok=True) return {"status": "downloaded", "url": url, "bytes": db_path.stat().st_size} except HTTPError as e: last_error = f"http-{e.code}" except (URLError, TimeoutError, zipfile.BadZipFile, OSError) as e: last_error = type(e).__name__ finally: if "tmp_path" in locals(): tmp_path.unlink(missing_ok=True) return {"status": f"failed:{last_error}", "url": "", "bytes": 0} def export_book_words(row: dict, decrypt_db, force_decrypt: bool) -> dict: """解密并导出一本词书的单词,返回导出 summary dict。""" code = str(row.get("code") or "") src = DB_DIR / f"LangEasyLexisV3_{code}.db" if not src.exists(): return summary(row, "missing_db", "", src, Path(), Path(), 0, f"not found: {src}") export_csv = DECRYPTED_DIR / f"LangEasyLexisV3_{safe_code_filename(code)}.csv" DECRYPTED_DIR.mkdir(parents=True, exist_ok=True) # 解密 DB:已存在且比源新则复用 # SQLite 仅作为临时解密载体,不作为 output 产物。 # Windows 可能在 SQLite 连接关闭后仍短暂持有文件句柄;清理失败不能影响导出结果。 with tempfile.TemporaryDirectory(prefix="lexis-decrypted-", ignore_cleanup_errors=True) as temp_dir: decrypted = Path(temp_dir) / src.with_suffix(".sqlite").name decrypt_db(src, decrypted) with sqlite3.connect(decrypted) as conn: table = choose_word_table(conn, code) count = 0 with export_csv.open("w", encoding="utf-8-sig", newline="") as fh: writer = csv.DictWriter(fh, fieldnames=WORD_FIELDS) writer.writeheader() for word_row in iter_words(conn, table): writer.writerow( { "book_id": row.get("id", ""), "book_code": code, "book_name": row.get("name", ""), "book_count": row.get("count", ""), "book_exam": row.get("exam", ""), "book_version": row.get("version", ""), "id": word_row["id"], "word": word_row["word"], "main_word": word_row["main_word"], "seq_book": word_row["seq_book"], "seq_az": word_row["seq_az"], "seq_freq": word_row["seq_freq"], "seq_rand": word_row["seq_rand"], "zpk": word_row["zpk"], } ) count += 1 return summary(row, "ok", table, src, decrypted, export_csv, count, "") def summary( row: dict, status: str, table: str, src: Path, decrypted: Path, export_csv: Path, exported_count: int, error: str, ) -> dict: return { "book_id": row.get("id", ""), "book_code": row.get("code", ""), "book_name": row.get("name", ""), "book_count": row.get("count", ""), "exported_count": exported_count, "status": status, "table": table, "local_db": str(src) if src else "", "decrypted_db": str(decrypted) if decrypted else "", "export_csv": str(export_csv) if export_csv else "", "error": error, } def process_one(book: dict, decrypt_db, force_download: bool, force_decrypt: bool) -> dict: """处理一本词书:下载 + 解密 + 导出,返回合并结果 dict。""" code = book["code"] result = { "code": code, "dl_status": "", "dl_url": "", "dl_bytes": 0, "export": None, } if book.get("url") == "*": result["dl_status"] = "skip-virtual-book" return result dl = download_book_db(book, force_download) result["dl_status"] = dl["status"] result["dl_url"] = dl["url"] result["dl_bytes"] = dl["bytes"] if dl["status"] not in {"downloaded", "skip-db-exists"}: result["export"] = summary(book, "download_failed", "", DB_DIR / f"LangEasyLexisV3_{code}.db", Path(), Path(), 0, dl["status"]) return result result["export"] = export_book_words(book, decrypt_db, force_decrypt) return result def write_download_report(results: list[dict], path: Path) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", newline="", encoding="utf-8-sig") as f: writer = csv.DictWriter(f, fieldnames=DOWNLOAD_REPORT_FIELDS) writer.writeheader() for r in results: writer.writerow( {"code": r["code"], "status": r["dl_status"], "bytes": r["dl_bytes"], "url": r["dl_url"]} ) def write_summary(summaries: list[dict], path: Path) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8-sig", newline="") as fh: writer = csv.DictWriter(fh, fieldnames=SUMMARY_FIELDS) writer.writeheader() writer.writerows(summaries) def write_combined(summaries: list[dict], path: Path) -> int: total = 0 path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8-sig", newline="") as out_fh: writer = csv.DictWriter(out_fh, fieldnames=WORD_FIELDS) writer.writeheader() for item in summaries: if item["status"] != "ok": continue with Path(item["export_csv"]).open("r", encoding="utf-8-sig", newline="") as in_fh: reader = csv.DictReader(in_fh) for row in reader: writer.writerow(row) total += 1 return total def main() -> int: parser = argparse.ArgumentParser(description="逐个下载所有词书:下载 -> 解密 -> 导出单词") parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) parser.add_argument("--force", action="store_true", help="重新下载已存在的词书 DB") parser.add_argument("--force-decrypt", action="store_true", help="重新解密已存在的解密 DB") parser.add_argument("--limit", type=int, default=0, help="最多处理 N 本词书(冒烟测试)") parser.add_argument("--sleep", type=float, default=0.0, help="单 worker 模式下每本之间 sleep 秒数") parser.add_argument("--workers", type=int, default=1, help="并行下载词书的 worker 数") args = parser.parse_args() if not args.catalog.exists(): print(f"词书目录不存在:{args.catalog},请先运行 src/fetch-book-catalog.py", file=sys.stderr) return 1 decrypt_db = load_decrypt_db() books = [row for row in load_books(args.catalog) if row.get("url") != "*"] if args.limit: books = books[: args.limit] total = len(books) results: list[dict] = [] done = 0 workers = max(1, args.workers) if workers == 1: for book in books: result = process_one(book, decrypt_db, args.force, args.force_decrypt) results.append(result) done += 1 exp = result["export"] print( f"[INFO] [步骤 2/3] 处理进度:current={done}/{total},percent={done / total * 100:.1f}%," f"code={result['code']},download={result['dl_status']}," f"decrypt={exp['status'] if exp else '-'},words={exp['exported_count'] if exp else '-'}", flush=True, ) if args.sleep: time.sleep(args.sleep) else: executor = ThreadPoolExecutor(max_workers=workers) futures = [executor.submit(process_one, book, decrypt_db, args.force, args.force_decrypt) for book in books] try: for future in as_completed(futures): result = future.result() results.append(result) done += 1 exp = result["export"] print( f"[INFO] [步骤 2/3] 处理进度:current={done}/{total},percent={done / total * 100:.1f}%," f"code={result['code']},download={result['dl_status']}," f"decrypt={exp['status'] if exp else '-'},words={exp['exported_count'] if exp else '-'}", flush=True, ) except KeyboardInterrupt: for future in futures: future.cancel() executor.shutdown(wait=False, cancel_futures=True) print("\n收到 Ctrl+C,已取消未开始的词书任务。", file=sys.stderr, flush=True) os._exit(130) else: executor.shutdown(wait=True) summaries = [r["export"] for r in results if r["export"] is not None] ok_dl = sum(1 for r in results if r["dl_status"] in {"downloaded", "skip-db-exists"}) ok_export = sum(1 for s in summaries if s["status"] == "ok") failed_dl = len(results) - ok_dl failed_export = len(summaries) - ok_export exported_total = sum(int(s["exported_count"] or 0) for s in summaries) print(f"[INFO] [步骤 2/3] 处理完成:total={len(results)},success={ok_export},failed={failed_export},words={exported_total}") failures = [] for result in results: export = result.get("export") or {} if result["dl_status"] not in {"downloaded", "skip-db-exists"}: failures.append(f"{result['code']}: 下载 {result['dl_status']}") elif export.get("status") != "ok": failures.append(f"{result['code']}: 解密/导出 {export.get('error') or export.get('status')}") for failure in failures[:20]: print(f"失败项:{failure}", file=sys.stderr) if len(failures) > 20: print(f"失败项:另有 {len(failures) - 20} 项未显示", file=sys.stderr) # 单本失败不阻断后续流程;步骤 3 继续处理已成功导出的词书。 return 0 if __name__ == "__main__": sys.exit(main())