feat(importer): 添加 Lexis 词书导出工具链
三步主流程(下载目录 -> 下载词书并解密导出 -> 下载并解密 ZPK), 以及可独立调用的 SQLCipher2 / ZPK 解密工具与词根、近义词导出脚本。 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
a39ba9081e
commit
f030334243
9 files changed
+3517
No files matched your search
@@ -0,0 +1,459 @@
|
||||
#!/usr/bin/env python3
|
||||
"""步骤2:逐个下载所有词书,解包、解密 SQLCipher2 DB 并导出单词与 zpk 字段。
|
||||
|
||||
每本词书走完整链路:下载 zip -> 校验 MD5 -> 解包 -> 解密 -> 导出单词 CSV。
|
||||
解密逻辑复用独立工具 decrypt-sqlcipher2-db.py(可单独调用)。
|
||||
|
||||
已下载的 DB 自动跳过(可用 --force 重下),已解密的 DB 按 mtime 复用(可用 --force-decrypt 重解)。
|
||||
|
||||
输出(默认统一到 output/):
|
||||
- output/databases/LangEasyLexisV3_<code>.db 原始加密 DB(中间缓存)
|
||||
- output/decrypted-databases/LangEasyLexisV3_<code>.sqlite 解密后 DB(可读)
|
||||
- output/wordbook-words/<code>.csv 每本词书单词(可读)
|
||||
- output/all-wordbook-words.csv 全量合并词表(可读,步骤3 的输入)
|
||||
- output/wordbook-download-report.csv 下载报告
|
||||
- output/wordbook-word-summary.csv 导出报告
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import csv
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import zipfile
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT_DIR = ROOT / "output"
|
||||
DEFAULT_CATALOG = OUT_DIR / "book-catalog.json"
|
||||
WORDBOOK_DIR = OUT_DIR / "wordbooks"
|
||||
DB_DIR = WORDBOOK_DIR / "original"
|
||||
DECRYPTED_DIR = WORDBOOK_DIR / "decrypted"
|
||||
|
||||
DECRYPTOR_PATH = ROOT / "src" / "decrypt-sqlcipher2-db.py"
|
||||
|
||||
BASE_STATIC_URLS = [
|
||||
"http://static.beingfine.cn",
|
||||
"https://hwstatic.beingfine.cn",
|
||||
"https://ccastatic.beingfine.cn",
|
||||
]
|
||||
REFERER = "https://app.bbdc.cn"
|
||||
USER_AGENT = "LangEasyLexis/5.11.1"
|
||||
|
||||
WORD_FIELDS = (
|
||||
"book_id",
|
||||
"book_code",
|
||||
"book_name",
|
||||
"book_count",
|
||||
"book_exam",
|
||||
"book_version",
|
||||
"id",
|
||||
"word",
|
||||
"main_word",
|
||||
"seq_book",
|
||||
"seq_az",
|
||||
"seq_freq",
|
||||
"seq_rand",
|
||||
"zpk",
|
||||
)
|
||||
|
||||
SUMMARY_FIELDS = (
|
||||
"book_id",
|
||||
"book_code",
|
||||
"book_name",
|
||||
"book_count",
|
||||
"exported_count",
|
||||
"status",
|
||||
"table",
|
||||
"local_db",
|
||||
"decrypted_db",
|
||||
"export_csv",
|
||||
"error",
|
||||
)
|
||||
|
||||
DOWNLOAD_REPORT_FIELDS = ("code", "status", "bytes", "url")
|
||||
|
||||
|
||||
def load_decrypt_db():
|
||||
"""加载独立解密工具 decrypt-sqlcipher2-db.py 的 decrypt_db 函数。"""
|
||||
spec = importlib.util.spec_from_file_location("decrypt_sqlcipher2_db", DECRYPTOR_PATH)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError(f"Cannot load decryptor: {DECRYPTOR_PATH}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module.decrypt_db
|
||||
|
||||
|
||||
def load_books(path: Path) -> list[dict]:
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, dict) and isinstance(data.get("books"), list):
|
||||
return data["books"]
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
raise ValueError(f"Unsupported catalog shape: {path}")
|
||||
|
||||
|
||||
def book_url(book: dict, base_url: str) -> str:
|
||||
code = book["code"]
|
||||
version = str(book.get("version") or 0)
|
||||
db_name = f"LangEasyLexisV3_{code}.db"
|
||||
return f"{base_url}/dicts/v3/{code}/{version}/{db_name}.zip"
|
||||
|
||||
|
||||
def request_url(url: str):
|
||||
req = Request(url, headers={"Referer": REFERER, "User-Agent": USER_AGENT})
|
||||
return urlopen(req, timeout=45)
|
||||
|
||||
|
||||
def md5_b64(path: Path) -> str:
|
||||
h = hashlib.md5()
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return base64.b64encode(h.digest()).decode("ascii")
|
||||
|
||||
|
||||
def safe_code_filename(code: object) -> str:
|
||||
text = str(code or "UNKNOWN")
|
||||
return re.sub(r"[^A-Za-z0-9_.-]+", "_", text).strip("._") or "UNKNOWN"
|
||||
|
||||
|
||||
def quote_identifier(name: str) -> str:
|
||||
return '"' + name.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def list_tables(conn: sqlite3.Connection) -> list[str]:
|
||||
return [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"select name from sqlite_master where type='table' and name not like 'sqlite_%' order by name"
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def table_columns(conn: sqlite3.Connection, table: str) -> set[str]:
|
||||
return {row[1] for row in conn.execute(f"pragma table_info({quote_identifier(table)})")}
|
||||
|
||||
|
||||
def choose_word_table(conn: sqlite3.Connection, code: str) -> str:
|
||||
tables = list_tables(conn)
|
||||
preferred = f"LangEasyLexisV3_{code}"
|
||||
if preferred in tables:
|
||||
return preferred
|
||||
|
||||
for table in tables:
|
||||
if {"id", "word"}.issubset(table_columns(conn, table)):
|
||||
return table
|
||||
|
||||
raise RuntimeError(f"no word table found; tables={tables}")
|
||||
|
||||
|
||||
def iter_words(conn: sqlite3.Connection, table: str) -> Iterable[sqlite3.Row]:
|
||||
conn.row_factory = sqlite3.Row
|
||||
cols = table_columns(conn, table)
|
||||
required = {"id", "word", "main_word", "seq_book", "seq_az", "seq_freq", "seq_rand", "zpk"}
|
||||
missing = sorted(required - cols)
|
||||
if missing:
|
||||
raise RuntimeError(f"table {table} missing columns: {missing}")
|
||||
|
||||
query = (
|
||||
f"select id, word, main_word, seq_book, seq_az, seq_freq, seq_rand, zpk "
|
||||
f"from {quote_identifier(table)} order by seq_book, id"
|
||||
)
|
||||
return conn.execute(query)
|
||||
|
||||
|
||||
def download_book_db(book: dict, force: bool) -> dict:
|
||||
"""下载单本词书 DB zip 并解包,返回下载状态 dict。"""
|
||||
code = book["code"]
|
||||
db_path = DB_DIR / f"LangEasyLexisV3_{code}.db"
|
||||
|
||||
if db_path.exists() and not force:
|
||||
return {"status": "skip-db-exists", "url": "", "bytes": db_path.stat().st_size}
|
||||
|
||||
last_error = ""
|
||||
for base in BASE_STATIC_URLS:
|
||||
url = book_url(book, base)
|
||||
try:
|
||||
with request_url(url) as resp:
|
||||
status = getattr(resp, "status", 200)
|
||||
if status != 200:
|
||||
last_error = f"http-{status}"
|
||||
continue
|
||||
|
||||
tmp_file = tempfile.NamedTemporaryFile(prefix=f"lexis-{code}-", suffix=".zip", delete=False)
|
||||
tmp_path = Path(tmp_file.name)
|
||||
with tmp_file as out:
|
||||
while True:
|
||||
chunk = resp.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
out.write(chunk)
|
||||
|
||||
content_md5 = resp.headers.get("Content-MD5")
|
||||
if content_md5 and md5_b64(tmp_path) != content_md5:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
last_error = "md5-mismatch"
|
||||
continue
|
||||
|
||||
DB_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(tmp_path) as zf:
|
||||
names = zf.namelist()
|
||||
wanted = f"LangEasyLexisV3_{code}.db"
|
||||
member = wanted if wanted in names else names[0]
|
||||
zf.extract(member, DB_DIR)
|
||||
extracted = DB_DIR / member
|
||||
if extracted != db_path:
|
||||
extracted.replace(db_path)
|
||||
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
return {"status": "downloaded", "url": url, "bytes": db_path.stat().st_size}
|
||||
except HTTPError as e:
|
||||
last_error = f"http-{e.code}"
|
||||
except (URLError, TimeoutError, zipfile.BadZipFile, OSError) as e:
|
||||
last_error = type(e).__name__
|
||||
finally:
|
||||
if "tmp_path" in locals():
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
return {"status": f"failed:{last_error}", "url": "", "bytes": 0}
|
||||
|
||||
|
||||
def export_book_words(row: dict, decrypt_db, force_decrypt: bool) -> dict:
|
||||
"""解密并导出一本词书的单词,返回导出 summary dict。"""
|
||||
code = str(row.get("code") or "")
|
||||
src = DB_DIR / f"LangEasyLexisV3_{code}.db"
|
||||
if not src.exists():
|
||||
return summary(row, "missing_db", "", src, Path(), Path(), 0, f"not found: {src}")
|
||||
|
||||
export_csv = DECRYPTED_DIR / f"LangEasyLexisV3_{safe_code_filename(code)}.csv"
|
||||
DECRYPTED_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 解密 DB:已存在且比源新则复用
|
||||
# SQLite 仅作为临时解密载体,不作为 output 产物。
|
||||
# Windows 可能在 SQLite 连接关闭后仍短暂持有文件句柄;清理失败不能影响导出结果。
|
||||
with tempfile.TemporaryDirectory(prefix="lexis-decrypted-", ignore_cleanup_errors=True) as temp_dir:
|
||||
decrypted = Path(temp_dir) / src.with_suffix(".sqlite").name
|
||||
decrypt_db(src, decrypted)
|
||||
with sqlite3.connect(decrypted) as conn:
|
||||
table = choose_word_table(conn, code)
|
||||
count = 0
|
||||
with export_csv.open("w", encoding="utf-8-sig", newline="") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=WORD_FIELDS)
|
||||
writer.writeheader()
|
||||
for word_row in iter_words(conn, table):
|
||||
writer.writerow(
|
||||
{
|
||||
"book_id": row.get("id", ""),
|
||||
"book_code": code,
|
||||
"book_name": row.get("name", ""),
|
||||
"book_count": row.get("count", ""),
|
||||
"book_exam": row.get("exam", ""),
|
||||
"book_version": row.get("version", ""),
|
||||
"id": word_row["id"],
|
||||
"word": word_row["word"],
|
||||
"main_word": word_row["main_word"],
|
||||
"seq_book": word_row["seq_book"],
|
||||
"seq_az": word_row["seq_az"],
|
||||
"seq_freq": word_row["seq_freq"],
|
||||
"seq_rand": word_row["seq_rand"],
|
||||
"zpk": word_row["zpk"],
|
||||
}
|
||||
)
|
||||
count += 1
|
||||
|
||||
return summary(row, "ok", table, src, decrypted, export_csv, count, "")
|
||||
|
||||
|
||||
def summary(
|
||||
row: dict,
|
||||
status: str,
|
||||
table: str,
|
||||
src: Path,
|
||||
decrypted: Path,
|
||||
export_csv: Path,
|
||||
exported_count: int,
|
||||
error: str,
|
||||
) -> dict:
|
||||
return {
|
||||
"book_id": row.get("id", ""),
|
||||
"book_code": row.get("code", ""),
|
||||
"book_name": row.get("name", ""),
|
||||
"book_count": row.get("count", ""),
|
||||
"exported_count": exported_count,
|
||||
"status": status,
|
||||
"table": table,
|
||||
"local_db": str(src) if src else "",
|
||||
"decrypted_db": str(decrypted) if decrypted else "",
|
||||
"export_csv": str(export_csv) if export_csv else "",
|
||||
"error": error,
|
||||
}
|
||||
|
||||
|
||||
def process_one(book: dict, decrypt_db, force_download: bool, force_decrypt: bool) -> dict:
|
||||
"""处理一本词书:下载 + 解密 + 导出,返回合并结果 dict。"""
|
||||
code = book["code"]
|
||||
result = {
|
||||
"code": code,
|
||||
"dl_status": "",
|
||||
"dl_url": "",
|
||||
"dl_bytes": 0,
|
||||
"export": None,
|
||||
}
|
||||
|
||||
if book.get("url") == "*":
|
||||
result["dl_status"] = "skip-virtual-book"
|
||||
return result
|
||||
|
||||
dl = download_book_db(book, force_download)
|
||||
result["dl_status"] = dl["status"]
|
||||
result["dl_url"] = dl["url"]
|
||||
result["dl_bytes"] = dl["bytes"]
|
||||
|
||||
if dl["status"] not in {"downloaded", "skip-db-exists"}:
|
||||
result["export"] = summary(book, "download_failed", "", DB_DIR / f"LangEasyLexisV3_{code}.db", Path(), Path(), 0, dl["status"])
|
||||
return result
|
||||
|
||||
result["export"] = export_book_words(book, decrypt_db, force_decrypt)
|
||||
return result
|
||||
|
||||
|
||||
def write_download_report(results: list[dict], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", newline="", encoding="utf-8-sig") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=DOWNLOAD_REPORT_FIELDS)
|
||||
writer.writeheader()
|
||||
for r in results:
|
||||
writer.writerow(
|
||||
{"code": r["code"], "status": r["dl_status"], "bytes": r["dl_bytes"], "url": r["dl_url"]}
|
||||
)
|
||||
|
||||
|
||||
def write_summary(summaries: list[dict], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=SUMMARY_FIELDS)
|
||||
writer.writeheader()
|
||||
writer.writerows(summaries)
|
||||
|
||||
|
||||
def write_combined(summaries: list[dict], path: Path) -> int:
|
||||
total = 0
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as out_fh:
|
||||
writer = csv.DictWriter(out_fh, fieldnames=WORD_FIELDS)
|
||||
writer.writeheader()
|
||||
for item in summaries:
|
||||
if item["status"] != "ok":
|
||||
continue
|
||||
with Path(item["export_csv"]).open("r", encoding="utf-8-sig", newline="") as in_fh:
|
||||
reader = csv.DictReader(in_fh)
|
||||
for row in reader:
|
||||
writer.writerow(row)
|
||||
total += 1
|
||||
return total
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="逐个下载所有词书:下载 -> 解密 -> 导出单词")
|
||||
parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
|
||||
parser.add_argument("--force", action="store_true", help="重新下载已存在的词书 DB")
|
||||
parser.add_argument("--force-decrypt", action="store_true", help="重新解密已存在的解密 DB")
|
||||
parser.add_argument("--limit", type=int, default=0, help="最多处理 N 本词书(冒烟测试)")
|
||||
parser.add_argument("--sleep", type=float, default=0.0, help="单 worker 模式下每本之间 sleep 秒数")
|
||||
parser.add_argument("--workers", type=int, default=1, help="并行下载词书的 worker 数")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.catalog.exists():
|
||||
print(f"词书目录不存在:{args.catalog},请先运行 src/fetch-book-catalog.py", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
decrypt_db = load_decrypt_db()
|
||||
books = [row for row in load_books(args.catalog) if row.get("url") != "*"]
|
||||
if args.limit:
|
||||
books = books[: args.limit]
|
||||
|
||||
total = len(books)
|
||||
results: list[dict] = []
|
||||
done = 0
|
||||
workers = max(1, args.workers)
|
||||
|
||||
if workers == 1:
|
||||
for book in books:
|
||||
result = process_one(book, decrypt_db, args.force, args.force_decrypt)
|
||||
results.append(result)
|
||||
done += 1
|
||||
exp = result["export"]
|
||||
print(
|
||||
f"[INFO] [步骤 2/3] 处理进度:current={done}/{total},percent={done / total * 100:.1f}%,"
|
||||
f"code={result['code']},download={result['dl_status']},"
|
||||
f"decrypt={exp['status'] if exp else '-'},words={exp['exported_count'] if exp else '-'}",
|
||||
flush=True,
|
||||
)
|
||||
if args.sleep:
|
||||
time.sleep(args.sleep)
|
||||
else:
|
||||
executor = ThreadPoolExecutor(max_workers=workers)
|
||||
futures = [executor.submit(process_one, book, decrypt_db, args.force, args.force_decrypt) for book in books]
|
||||
try:
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
results.append(result)
|
||||
done += 1
|
||||
exp = result["export"]
|
||||
print(
|
||||
f"[INFO] [步骤 2/3] 处理进度:current={done}/{total},percent={done / total * 100:.1f}%,"
|
||||
f"code={result['code']},download={result['dl_status']},"
|
||||
f"decrypt={exp['status'] if exp else '-'},words={exp['exported_count'] if exp else '-'}",
|
||||
flush=True,
|
||||
)
|
||||
except KeyboardInterrupt:
|
||||
for future in futures:
|
||||
future.cancel()
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
print("\n收到 Ctrl+C,已取消未开始的词书任务。", file=sys.stderr, flush=True)
|
||||
os._exit(130)
|
||||
else:
|
||||
executor.shutdown(wait=True)
|
||||
|
||||
summaries = [r["export"] for r in results if r["export"] is not None]
|
||||
|
||||
ok_dl = sum(1 for r in results if r["dl_status"] in {"downloaded", "skip-db-exists"})
|
||||
ok_export = sum(1 for s in summaries if s["status"] == "ok")
|
||||
failed_dl = len(results) - ok_dl
|
||||
failed_export = len(summaries) - ok_export
|
||||
exported_total = sum(int(s["exported_count"] or 0) for s in summaries)
|
||||
|
||||
print(f"[INFO] [步骤 2/3] 处理完成:total={len(results)},success={ok_export},failed={failed_export},words={exported_total}")
|
||||
failures = []
|
||||
for result in results:
|
||||
export = result.get("export") or {}
|
||||
if result["dl_status"] not in {"downloaded", "skip-db-exists"}:
|
||||
failures.append(f"{result['code']}: 下载 {result['dl_status']}")
|
||||
elif export.get("status") != "ok":
|
||||
failures.append(f"{result['code']}: 解密/导出 {export.get('error') or export.get('status')}")
|
||||
for failure in failures[:20]:
|
||||
print(f"失败项:{failure}", file=sys.stderr)
|
||||
if len(failures) > 20:
|
||||
print(f"失败项:另有 {len(failures) - 20} 项未显示", file=sys.stderr)
|
||||
# 单本失败不阻断后续流程;步骤 3 继续处理已成功导出的词书。
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user