feat(importer): 添加 Lexis 词书导出工具链

三步主流程(下载目录 -> 下载词书并解密导出 -> 下载并解密 ZPK),
以及可独立调用的 SQLCipher2 / ZPK 解密工具与词根、近义词导出脚本。

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-08 21:15:43 +08:00
1 parent a39ba9081e
commit f030334243
9 files changed
+3517

No files matched your search

+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python3
import argparse
import hashlib
from pathlib import Path
from Crypto.Cipher import AES
KEY = b"arsenal iscool"
PAGE_SIZE = 1024
RESERVE = 48
KDF_ITER = 4000
SQLITE_HEADER = b"SQLite format 3\x00"
def derive_key(salt: bytes) -> bytes:
return hashlib.pbkdf2_hmac("sha1", KEY, salt, KDF_ITER, 32)
def decrypt_db(src: Path, dst: Path) -> None:
blob = src.read_bytes()
if len(blob) % PAGE_SIZE:
raise ValueError(f"input size is not aligned to {PAGE_SIZE}: {src}")
key = derive_key(blob[:16])
out = bytearray()
pages = len(blob) // PAGE_SIZE
for page_no in range(pages):
page = blob[page_no * PAGE_SIZE : (page_no + 1) * PAGE_SIZE]
iv = page[PAGE_SIZE - RESERVE : PAGE_SIZE - RESERVE + 16]
if page_no == 0:
enc_start = 16
prefix = SQLITE_HEADER
else:
enc_start = 0
prefix = b""
enc_end = PAGE_SIZE - RESERVE
dec = AES.new(key, AES.MODE_CBC, iv).decrypt(page[enc_start:enc_end])
plain = prefix + dec
if len(plain) > PAGE_SIZE:
raise ValueError("decrypted page overflow")
out.extend(plain)
out.extend(b"\x00" * (PAGE_SIZE - len(plain)))
dst.parent.mkdir(parents=True, exist_ok=True)
dst.write_bytes(out)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("src", type=Path)
parser.add_argument("dst", type=Path)
args = parser.parse_args()
decrypt_db(args.src, args.dst)
print(args.dst)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+129
View File
@@ -0,0 +1,129 @@
#!/usr/bin/env python3
"""Decrypt Lexis .zpk files offline.
ZPK format:
outer protobuf: field 1 length-delimited AES ciphertext
cipher: AES-256-CBC + PKCS7/PKCS5 padding
key: md5(word).hexdigest().encode("ascii") # 32 bytes
iv: b"IscoolInitVector"
The word -> zpk mapping can be supplied by output/unique-zpk-index.csv.
"""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
from pathlib import Path
from Crypto.Cipher import AES
from Crypto.Util.Padding import unpad
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_INDEX = ROOT / "output" / "unique-zpk-index.csv"
DEFAULT_ZPK_DIR = ROOT / "output" / "zpk" / "original"
DEFAULT_OUTPUT = ROOT / "output" / "zpk" / "decrypted" / "zpk-decrypted.jsonl"
IV = b"IscoolInitVector"
def read_varint(data: bytes, offset: int) -> tuple[int, int]:
value = 0
shift = 0
while offset < len(data):
b = data[offset]
offset += 1
value |= (b & 0x7F) << shift
if b < 0x80:
return value, offset
shift += 7
raise ValueError("unterminated varint")
def read_field1_payload(data: bytes) -> bytes:
if not data or data[0] != 0x0A:
raise ValueError("bad zpk outer protobuf header")
size, offset = read_varint(data, 1)
payload = data[offset : offset + size]
if len(payload) != size:
raise ValueError(f"truncated payload: expected={size} actual={len(payload)}")
return payload
def decrypt_zpk_bytes(raw: bytes, word: str) -> dict:
ciphertext = read_field1_payload(raw)
key = hashlib.md5(word.encode("utf-8")).hexdigest().encode("ascii")
plaintext = AES.new(key, AES.MODE_CBC, IV).decrypt(ciphertext)
plaintext = unpad(plaintext, 16)
inner = read_field1_payload(plaintext)
return json.loads(inner.decode("utf-8"))
def load_index(path: Path) -> list[dict[str, str]]:
with path.open("r", encoding="utf-8-sig", newline="") as fh:
return [row for row in csv.DictReader(fh) if row.get("zpk") and row.get("first_word")]
def flatten(obj: dict) -> dict:
return {
"word": obj.get("word", ""),
"us_pron": obj.get("us_pron", ""),
"uk_pron": obj.get("uk_pron", ""),
"interpret": obj.get("interpret", ""),
"definition_count": sum(len(g.get("d") or []) for g in (obj.get("interpret_v2") or []) if isinstance(g, dict)),
"phrase_count": sum(len(g.get("p") or []) for g in (obj.get("phrase") or []) if isinstance(g, dict)),
"raw": obj,
}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
parser.add_argument("--zpk-dir", type=Path, default=DEFAULT_ZPK_DIR)
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
parser.add_argument("--limit", type=int, default=0)
parser.add_argument("--raw", action="store_true", help="include full raw JSON object")
args = parser.parse_args()
rows = load_index(args.index)
if args.limit:
rows = rows[: args.limit]
args.output.parent.mkdir(parents=True, exist_ok=True)
ok = missing = failed = 0
with args.output.open("w", encoding="utf-8", newline="\n") as out:
for row in rows:
zpk = row["zpk"]
word = row["first_word"]
path = args.zpk_dir / f"{zpk}.zpk"
if not path.exists():
missing += 1
continue
try:
obj = decrypt_zpk_bytes(path.read_bytes(), word)
rec = dict(obj)
rec.update(
{
"zpk": zpk,
"zpk_path": str(path),
"index_word": word,
"book_occurrences": row.get("book_occurrences", ""),
}
)
if args.raw:
rec["raw"] = obj
out.write(json.dumps(rec, ensure_ascii=False, separators=(",", ":")) + "\n")
ok += 1
except Exception as exc:
failed += 1
if failed <= 20:
print(f"[fail] {zpk} word={word}: {exc}")
print(f"ok={ok} missing={missing} failed={failed} -> {args.output}")
return 0 if failed == 0 else 1
if __name__ == "__main__":
raise SystemExit(main())
+459
View File
@@ -0,0 +1,459 @@
#!/usr/bin/env python3
"""步骤2:逐个下载所有词书,解包、解密 SQLCipher2 DB 并导出单词与 zpk 字段。
每本词书走完整链路:下载 zip -> 校验 MD5 -> 解包 -> 解密 -> 导出单词 CSV。
解密逻辑复用独立工具 decrypt-sqlcipher2-db.py(可单独调用)。
已下载的 DB 自动跳过(可用 --force 重下),已解密的 DB 按 mtime 复用(可用 --force-decrypt 重解)。
输出(默认统一到 output/):
- output/databases/LangEasyLexisV3_<code>.db 原始加密 DB(中间缓存)
- output/decrypted-databases/LangEasyLexisV3_<code>.sqlite 解密后 DB(可读)
- output/wordbook-words/<code>.csv 每本词书单词(可读)
- output/all-wordbook-words.csv 全量合并词表(可读,步骤3 的输入)
- output/wordbook-download-report.csv 下载报告
- output/wordbook-word-summary.csv 导出报告
"""
from __future__ import annotations
import argparse
import base64
import csv
import hashlib
import importlib.util
import json
import os
import re
import sqlite3
import sys
import tempfile
import time
import zipfile
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Iterable
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parents[1]
OUT_DIR = ROOT / "output"
DEFAULT_CATALOG = OUT_DIR / "book-catalog.json"
WORDBOOK_DIR = OUT_DIR / "wordbooks"
DB_DIR = WORDBOOK_DIR / "original"
DECRYPTED_DIR = WORDBOOK_DIR / "decrypted"
DECRYPTOR_PATH = ROOT / "src" / "decrypt-sqlcipher2-db.py"
BASE_STATIC_URLS = [
"http://static.beingfine.cn",
"https://hwstatic.beingfine.cn",
"https://ccastatic.beingfine.cn",
]
REFERER = "https://app.bbdc.cn"
USER_AGENT = "LangEasyLexis/5.11.1"
WORD_FIELDS = (
"book_id",
"book_code",
"book_name",
"book_count",
"book_exam",
"book_version",
"id",
"word",
"main_word",
"seq_book",
"seq_az",
"seq_freq",
"seq_rand",
"zpk",
)
SUMMARY_FIELDS = (
"book_id",
"book_code",
"book_name",
"book_count",
"exported_count",
"status",
"table",
"local_db",
"decrypted_db",
"export_csv",
"error",
)
DOWNLOAD_REPORT_FIELDS = ("code", "status", "bytes", "url")
def load_decrypt_db():
"""加载独立解密工具 decrypt-sqlcipher2-db.py 的 decrypt_db 函数。"""
spec = importlib.util.spec_from_file_location("decrypt_sqlcipher2_db", DECRYPTOR_PATH)
if spec is None or spec.loader is None:
raise RuntimeError(f"Cannot load decryptor: {DECRYPTOR_PATH}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module.decrypt_db
def load_books(path: Path) -> list[dict]:
with path.open("r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, dict) and isinstance(data.get("books"), list):
return data["books"]
if isinstance(data, list):
return data
raise ValueError(f"Unsupported catalog shape: {path}")
def book_url(book: dict, base_url: str) -> str:
code = book["code"]
version = str(book.get("version") or 0)
db_name = f"LangEasyLexisV3_{code}.db"
return f"{base_url}/dicts/v3/{code}/{version}/{db_name}.zip"
def request_url(url: str):
req = Request(url, headers={"Referer": REFERER, "User-Agent": USER_AGENT})
return urlopen(req, timeout=45)
def md5_b64(path: Path) -> str:
h = hashlib.md5()
with path.open("rb") as f:
for chunk in iter(lambda: f.read(1024 * 1024), b""):
h.update(chunk)
return base64.b64encode(h.digest()).decode("ascii")
def safe_code_filename(code: object) -> str:
text = str(code or "UNKNOWN")
return re.sub(r"[^A-Za-z0-9_.-]+", "_", text).strip("._") or "UNKNOWN"
def quote_identifier(name: str) -> str:
return '"' + name.replace('"', '""') + '"'
def list_tables(conn: sqlite3.Connection) -> list[str]:
return [
row[0]
for row in conn.execute(
"select name from sqlite_master where type='table' and name not like 'sqlite_%' order by name"
)
]
def table_columns(conn: sqlite3.Connection, table: str) -> set[str]:
return {row[1] for row in conn.execute(f"pragma table_info({quote_identifier(table)})")}
def choose_word_table(conn: sqlite3.Connection, code: str) -> str:
tables = list_tables(conn)
preferred = f"LangEasyLexisV3_{code}"
if preferred in tables:
return preferred
for table in tables:
if {"id", "word"}.issubset(table_columns(conn, table)):
return table
raise RuntimeError(f"no word table found; tables={tables}")
def iter_words(conn: sqlite3.Connection, table: str) -> Iterable[sqlite3.Row]:
conn.row_factory = sqlite3.Row
cols = table_columns(conn, table)
required = {"id", "word", "main_word", "seq_book", "seq_az", "seq_freq", "seq_rand", "zpk"}
missing = sorted(required - cols)
if missing:
raise RuntimeError(f"table {table} missing columns: {missing}")
query = (
f"select id, word, main_word, seq_book, seq_az, seq_freq, seq_rand, zpk "
f"from {quote_identifier(table)} order by seq_book, id"
)
return conn.execute(query)
def download_book_db(book: dict, force: bool) -> dict:
"""下载单本词书 DB zip 并解包,返回下载状态 dict。"""
code = book["code"]
db_path = DB_DIR / f"LangEasyLexisV3_{code}.db"
if db_path.exists() and not force:
return {"status": "skip-db-exists", "url": "", "bytes": db_path.stat().st_size}
last_error = ""
for base in BASE_STATIC_URLS:
url = book_url(book, base)
try:
with request_url(url) as resp:
status = getattr(resp, "status", 200)
if status != 200:
last_error = f"http-{status}"
continue
tmp_file = tempfile.NamedTemporaryFile(prefix=f"lexis-{code}-", suffix=".zip", delete=False)
tmp_path = Path(tmp_file.name)
with tmp_file as out:
while True:
chunk = resp.read(1024 * 1024)
if not chunk:
break
out.write(chunk)
content_md5 = resp.headers.get("Content-MD5")
if content_md5 and md5_b64(tmp_path) != content_md5:
tmp_path.unlink(missing_ok=True)
last_error = "md5-mismatch"
continue
DB_DIR.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(tmp_path) as zf:
names = zf.namelist()
wanted = f"LangEasyLexisV3_{code}.db"
member = wanted if wanted in names else names[0]
zf.extract(member, DB_DIR)
extracted = DB_DIR / member
if extracted != db_path:
extracted.replace(db_path)
tmp_path.unlink(missing_ok=True)
return {"status": "downloaded", "url": url, "bytes": db_path.stat().st_size}
except HTTPError as e:
last_error = f"http-{e.code}"
except (URLError, TimeoutError, zipfile.BadZipFile, OSError) as e:
last_error = type(e).__name__
finally:
if "tmp_path" in locals():
tmp_path.unlink(missing_ok=True)
return {"status": f"failed:{last_error}", "url": "", "bytes": 0}
def export_book_words(row: dict, decrypt_db, force_decrypt: bool) -> dict:
"""解密并导出一本词书的单词,返回导出 summary dict。"""
code = str(row.get("code") or "")
src = DB_DIR / f"LangEasyLexisV3_{code}.db"
if not src.exists():
return summary(row, "missing_db", "", src, Path(), Path(), 0, f"not found: {src}")
export_csv = DECRYPTED_DIR / f"LangEasyLexisV3_{safe_code_filename(code)}.csv"
DECRYPTED_DIR.mkdir(parents=True, exist_ok=True)
# 解密 DB:已存在且比源新则复用
# SQLite 仅作为临时解密载体,不作为 output 产物。
# Windows 可能在 SQLite 连接关闭后仍短暂持有文件句柄;清理失败不能影响导出结果。
with tempfile.TemporaryDirectory(prefix="lexis-decrypted-", ignore_cleanup_errors=True) as temp_dir:
decrypted = Path(temp_dir) / src.with_suffix(".sqlite").name
decrypt_db(src, decrypted)
with sqlite3.connect(decrypted) as conn:
table = choose_word_table(conn, code)
count = 0
with export_csv.open("w", encoding="utf-8-sig", newline="") as fh:
writer = csv.DictWriter(fh, fieldnames=WORD_FIELDS)
writer.writeheader()
for word_row in iter_words(conn, table):
writer.writerow(
{
"book_id": row.get("id", ""),
"book_code": code,
"book_name": row.get("name", ""),
"book_count": row.get("count", ""),
"book_exam": row.get("exam", ""),
"book_version": row.get("version", ""),
"id": word_row["id"],
"word": word_row["word"],
"main_word": word_row["main_word"],
"seq_book": word_row["seq_book"],
"seq_az": word_row["seq_az"],
"seq_freq": word_row["seq_freq"],
"seq_rand": word_row["seq_rand"],
"zpk": word_row["zpk"],
}
)
count += 1
return summary(row, "ok", table, src, decrypted, export_csv, count, "")
def summary(
row: dict,
status: str,
table: str,
src: Path,
decrypted: Path,
export_csv: Path,
exported_count: int,
error: str,
) -> dict:
return {
"book_id": row.get("id", ""),
"book_code": row.get("code", ""),
"book_name": row.get("name", ""),
"book_count": row.get("count", ""),
"exported_count": exported_count,
"status": status,
"table": table,
"local_db": str(src) if src else "",
"decrypted_db": str(decrypted) if decrypted else "",
"export_csv": str(export_csv) if export_csv else "",
"error": error,
}
def process_one(book: dict, decrypt_db, force_download: bool, force_decrypt: bool) -> dict:
"""处理一本词书:下载 + 解密 + 导出,返回合并结果 dict。"""
code = book["code"]
result = {
"code": code,
"dl_status": "",
"dl_url": "",
"dl_bytes": 0,
"export": None,
}
if book.get("url") == "*":
result["dl_status"] = "skip-virtual-book"
return result
dl = download_book_db(book, force_download)
result["dl_status"] = dl["status"]
result["dl_url"] = dl["url"]
result["dl_bytes"] = dl["bytes"]
if dl["status"] not in {"downloaded", "skip-db-exists"}:
result["export"] = summary(book, "download_failed", "", DB_DIR / f"LangEasyLexisV3_{code}.db", Path(), Path(), 0, dl["status"])
return result
result["export"] = export_book_words(book, decrypt_db, force_decrypt)
return result
def write_download_report(results: list[dict], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", newline="", encoding="utf-8-sig") as f:
writer = csv.DictWriter(f, fieldnames=DOWNLOAD_REPORT_FIELDS)
writer.writeheader()
for r in results:
writer.writerow(
{"code": r["code"], "status": r["dl_status"], "bytes": r["dl_bytes"], "url": r["dl_url"]}
)
def write_summary(summaries: list[dict], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8-sig", newline="") as fh:
writer = csv.DictWriter(fh, fieldnames=SUMMARY_FIELDS)
writer.writeheader()
writer.writerows(summaries)
def write_combined(summaries: list[dict], path: Path) -> int:
total = 0
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8-sig", newline="") as out_fh:
writer = csv.DictWriter(out_fh, fieldnames=WORD_FIELDS)
writer.writeheader()
for item in summaries:
if item["status"] != "ok":
continue
with Path(item["export_csv"]).open("r", encoding="utf-8-sig", newline="") as in_fh:
reader = csv.DictReader(in_fh)
for row in reader:
writer.writerow(row)
total += 1
return total
def main() -> int:
parser = argparse.ArgumentParser(description="逐个下载所有词书:下载 -> 解密 -> 导出单词")
parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
parser.add_argument("--force", action="store_true", help="重新下载已存在的词书 DB")
parser.add_argument("--force-decrypt", action="store_true", help="重新解密已存在的解密 DB")
parser.add_argument("--limit", type=int, default=0, help="最多处理 N 本词书(冒烟测试)")
parser.add_argument("--sleep", type=float, default=0.0, help="单 worker 模式下每本之间 sleep 秒数")
parser.add_argument("--workers", type=int, default=1, help="并行下载词书的 worker 数")
args = parser.parse_args()
if not args.catalog.exists():
print(f"词书目录不存在:{args.catalog},请先运行 src/fetch-book-catalog.py", file=sys.stderr)
return 1
decrypt_db = load_decrypt_db()
books = [row for row in load_books(args.catalog) if row.get("url") != "*"]
if args.limit:
books = books[: args.limit]
total = len(books)
results: list[dict] = []
done = 0
workers = max(1, args.workers)
if workers == 1:
for book in books:
result = process_one(book, decrypt_db, args.force, args.force_decrypt)
results.append(result)
done += 1
exp = result["export"]
print(
f"[INFO] [步骤 2/3] 处理进度:current={done}/{total},percent={done / total * 100:.1f}%,"
f"code={result['code']},download={result['dl_status']},"
f"decrypt={exp['status'] if exp else '-'},words={exp['exported_count'] if exp else '-'}",
flush=True,
)
if args.sleep:
time.sleep(args.sleep)
else:
executor = ThreadPoolExecutor(max_workers=workers)
futures = [executor.submit(process_one, book, decrypt_db, args.force, args.force_decrypt) for book in books]
try:
for future in as_completed(futures):
result = future.result()
results.append(result)
done += 1
exp = result["export"]
print(
f"[INFO] [步骤 2/3] 处理进度:current={done}/{total},percent={done / total * 100:.1f}%,"
f"code={result['code']},download={result['dl_status']},"
f"decrypt={exp['status'] if exp else '-'},words={exp['exported_count'] if exp else '-'}",
flush=True,
)
except KeyboardInterrupt:
for future in futures:
future.cancel()
executor.shutdown(wait=False, cancel_futures=True)
print("\n收到 Ctrl+C,已取消未开始的词书任务。", file=sys.stderr, flush=True)
os._exit(130)
else:
executor.shutdown(wait=True)
summaries = [r["export"] for r in results if r["export"] is not None]
ok_dl = sum(1 for r in results if r["dl_status"] in {"downloaded", "skip-db-exists"})
ok_export = sum(1 for s in summaries if s["status"] == "ok")
failed_dl = len(results) - ok_dl
failed_export = len(summaries) - ok_export
exported_total = sum(int(s["exported_count"] or 0) for s in summaries)
print(f"[INFO] [步骤 2/3] 处理完成:total={len(results)},success={ok_export},failed={failed_export},words={exported_total}")
failures = []
for result in results:
export = result.get("export") or {}
if result["dl_status"] not in {"downloaded", "skip-db-exists"}:
failures.append(f"{result['code']}: 下载 {result['dl_status']}")
elif export.get("status") != "ok":
failures.append(f"{result['code']}: 解密/导出 {export.get('error') or export.get('status')}")
for failure in failures[:20]:
print(f"失败项:{failure}", file=sys.stderr)
if len(failures) > 20:
print(f"失败项:另有 {len(failures) - 20} 项未显示", file=sys.stderr)
# 单本失败不阻断后续流程;步骤 3 继续处理已成功导出的词书。
return 0
if __name__ == "__main__":
sys.exit(main())
+507
View File
@@ -0,0 +1,507 @@
#!/usr/bin/env python3
"""步骤3:从全量词表去重后,逐个下载所有词书所有单词的 ZPK 并解密为可读 JSONL。
解密逻辑复用独立工具 decrypt-zpk-aes.py(可单独调用)。
输入:output/all-wordbook-words.csv(步骤2 产出,含 word、zpk 字段)
去重:同一 zpk 名只下载一次,记录首个出现的 word 作为 first_word(ZPK 解密 key = md5(word))。
CDN:{base}/r/{zpkName}.zpk
输出(默认统一到 output/):
- output/unique-zpk-index.csv 去重后的 zpk 索引(可读)
- output/zpk-cache/0/{zpkName}.zpk 加密 ZPK(中间缓存)
- output/zpk-jsonl/zpk-decrypted.jsonl 解密后 JSONL(可读,最终产出)
- output/zpk-download-report.csv 下载/解密报告
已解密的 zpk(记录在 zpk-decrypted.jsonl 中)自动跳过;已下载的 zpk 自动跳过下载,仅重新解密。
"""
from __future__ import annotations
import argparse
import base64
import csv
import hashlib
import http.client
import importlib.util
import json
import queue
import sys
import threading
import time
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urlparse
ROOT = Path(__file__).resolve().parents[1]
OUT_DIR = ROOT / "output"
WORDS_CSV = OUT_DIR / "wordbooks" / "decrypted"
ZPK_DIR = OUT_DIR / "zpk" / "original"
ZPK_JSONL = OUT_DIR / "zpk" / "decrypted"
DECRYPTOR_PATH = ROOT / "src" / "decrypt-zpk-aes.py"
BASE_STATIC_URLS = [
"http://static.beingfine.cn",
"https://hwstatic.beingfine.cn",
"https://ccastatic.beingfine.cn",
]
REFERER = "https://app.bbdc.cn"
USER_AGENT = "LangEasyLexis/5.11.1"
REQUEST_TIMEOUT = 8.0
STOP_EVENT = threading.Event()
UNIQUE_FIELDS = (
"zpk",
"zpk_bid",
"zpk_word_id",
"zpk_version",
"zpk_updated_unix",
"zpk_updated_utc",
"first_word",
"book_occurrences",
)
@dataclass(frozen=True)
class CdnEndpoint:
base_url: str
scheme: str
host: str
port: int | None
@classmethod
def parse(cls, base_url: str) -> "CdnEndpoint":
parsed = urlparse(base_url)
return cls(base_url=base_url, scheme=parsed.scheme, host=parsed.hostname or "", port=parsed.port)
class CdnClient:
def __init__(self, endpoints: list[CdnEndpoint], timeout: float) -> None:
self.endpoints = endpoints
self.timeout = timeout
self.connections: dict[str, http.client.HTTPConnection] = {}
def close(self) -> None:
for conn in self.connections.values():
conn.close()
self.connections.clear()
def _conn(self, endpoint: CdnEndpoint) -> http.client.HTTPConnection:
key = endpoint.base_url
conn = self.connections.get(key)
if conn is not None:
return conn
conn_cls = http.client.HTTPSConnection if endpoint.scheme == "https" else http.client.HTTPConnection
conn = conn_cls(endpoint.host, endpoint.port, timeout=self.timeout)
self.connections[key] = conn
return conn
def get(self, endpoint: CdnEndpoint, name: str) -> tuple[str, http.client.HTTPResponse]:
url_path = f"/r/{name}.zpk"
headers = {"Connection": "keep-alive", "Referer": REFERER, "User-Agent": USER_AGENT}
conn = self._conn(endpoint)
try:
conn.request("GET", url_path, headers=headers)
return f"{endpoint.base_url}{url_path}", conn.getresponse()
except (http.client.HTTPException, OSError):
conn.close()
self.connections.pop(endpoint.base_url, None)
raise
def drop(self, endpoint: CdnEndpoint) -> None:
conn = self.connections.pop(endpoint.base_url, None)
if conn is not None:
conn.close()
class JsonlSink:
"""线程安全地为每个 ZPK 写一个独立 JSON 文件。"""
def __init__(self, path: Path) -> None:
self.lock = threading.Lock()
path.parent.mkdir(parents=True, exist_ok=True)
self.dir = path
self.dir.mkdir(parents=True, exist_ok=True)
def write(self, name: str, line: str) -> None:
with self.lock:
target = self.dir / f"{name}.json"
target.write_text(line + "\n", encoding="utf-8")
def close(self) -> None:
return
def load_decrypt_zpk():
"""加载独立解密工具 decrypt-zpk-aes.py 的解密函数。"""
spec = importlib.util.spec_from_file_location("decrypt_zpk_aes", DECRYPTOR_PATH)
if spec is None or spec.loader is None:
raise RuntimeError(f"Cannot load decryptor: {DECRYPTOR_PATH}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module.decrypt_zpk_bytes, module.flatten
def build_endpoints(cdn_mode: str, worker_index: int = 0) -> list[CdnEndpoint]:
base_urls = BASE_STATIC_URLS
if cdn_mode == "spread" and base_urls:
offset = worker_index % len(base_urls)
base_urls = base_urls[offset:] + base_urls[:offset]
return [CdnEndpoint.parse(base) for base in base_urls]
def parse_zpk(zpk: str) -> dict[str, str]:
parts = (zpk or "").split("_")
if len(parts) != 4 or not all(part.isdigit() for part in parts):
return {
"zpk_bid": "",
"zpk_word_id": "",
"zpk_version": "",
"zpk_updated_unix": "",
"zpk_updated_utc": "",
}
updated = int(parts[3])
try:
updated_utc = datetime.fromtimestamp(updated, tz=timezone.utc).isoformat().replace("+00:00", "Z")
except (OverflowError, OSError, ValueError):
updated_utc = ""
return {
"zpk_bid": parts[0],
"zpk_word_id": parts[1],
"zpk_version": parts[2],
"zpk_updated_unix": parts[3],
"zpk_updated_utc": updated_utc,
}
def build_unique_index(words_path: Path) -> list[dict[str, str]]:
"""从全量词表在内存中去重 ZPK 名,不生成索引文件。"""
unique: dict[str, dict[str, str | int]] = {}
paths = [words_path] if words_path.is_file() else sorted(words_path.glob("*.csv"))
for path in paths:
with path.open("r", encoding="utf-8-sig", newline="") as fh:
reader = csv.DictReader(fh)
for row in reader:
zpk = row.get("zpk", "")
if not zpk:
continue
if zpk not in unique:
unique[zpk] = {
"zpk": zpk,
**parse_zpk(zpk),
"first_word": row.get("word", ""),
"book_occurrences": 0,
}
unique[zpk]["book_occurrences"] = int(unique[zpk]["book_occurrences"]) + 1
return list(unique.values())
def load_already_decrypted(path: Path) -> set[str]:
"""读取现有 JSONL 中已解密的 zpk 名集合,用于断点续传跳过。"""
if not path.exists():
return set()
if path.is_dir():
return {p.stem for p in path.glob("*.json")}
seen = set()
with path.open("r", encoding="utf-8", newline="") as fh:
for line in fh:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
zpk = obj.get("zpk")
if zpk:
seen.add(zpk)
return seen
def read_and_save_response(resp: http.client.HTTPResponse, dest: Path) -> int:
tmp = dest.with_name(f"{dest.name}.{threading.get_ident()}.tmp")
h = hashlib.md5()
size = 0
content_md5 = resp.getheader("Content-MD5")
dest.parent.mkdir(parents=True, exist_ok=True)
with tmp.open("wb") as out:
while not STOP_EVENT.is_set():
chunk = resp.read(256 * 1024)
if not chunk:
break
out.write(chunk)
h.update(chunk)
size += len(chunk)
if STOP_EVENT.is_set():
tmp.unlink(missing_ok=True)
raise InterruptedError
if content_md5 and base64.b64encode(h.digest()).decode("ascii") != content_md5:
tmp.unlink(missing_ok=True)
raise ValueError("md5-mismatch")
tmp.replace(dest)
return size
def ensure_downloaded(name: str, zpk_dir: Path, client: CdnClient) -> dict:
"""确保 zpk 文件已下载,返回 (status, url, bytes)。"""
dest = zpk_dir / f"{name}.zpk"
if dest.exists() and dest.stat().st_size > 0:
return {"status": "skip-download", "url": "", "bytes": dest.stat().st_size}
last_error = ""
for endpoint in client.endpoints:
if STOP_EVENT.is_set():
return {"status": "cancelled", "url": "", "bytes": 0}
try:
url, resp = client.get(endpoint, name)
try:
if resp.status != 200:
last_error = f"http-{resp.status}"
resp.read()
if resp.status >= 500:
client.drop(endpoint)
continue
size = read_and_save_response(resp, dest)
return {"status": "downloaded", "url": url, "bytes": size}
finally:
resp.close()
except InterruptedError:
return {"status": "cancelled", "url": "", "bytes": 0}
except ValueError as e:
last_error = str(e)
client.drop(endpoint)
except (http.client.HTTPException, OSError, TimeoutError) as e:
last_error = type(e).__name__
client.drop(endpoint)
return {"status": f"failed:{last_error}", "url": "", "bytes": 0}
def process_one(
row: dict,
zpk_dir: Path,
client: CdnClient,
sink: JsonlSink,
decrypt_zpk_bytes,
flatten,
include_raw: bool = True,
) -> dict:
"""下载 + 解密单个 zpk,解密结果写入 JSONL。"""
name = row["zpk"]
word = row.get("first_word", "")
dl = ensure_downloaded(name, zpk_dir, client)
if dl["status"].startswith("failed:"):
return {"name": name, "status": dl["status"], "bytes": dl["bytes"], "url": dl["url"]}
if dl["status"] == "cancelled":
return {"name": name, "status": "cancelled", "bytes": 0, "url": ""}
# 下载成功或已存在,开始解密
path = zpk_dir / f"{name}.zpk"
try:
obj = decrypt_zpk_bytes(path.read_bytes(), word)
# 解密输出必须保留 ZPK 内层 JSON 的全部字段;flatten 仅用于旧版兼容。
rec = dict(obj)
rec.update(
{
"zpk": name,
"zpk_path": str(path),
"index_word": word,
"book_occurrences": row.get("book_occurrences", ""),
}
)
if include_raw:
rec["raw"] = obj
sink.write(name, json.dumps(rec, ensure_ascii=False, separators=(",", ":")))
return {"name": name, "status": "decrypted", "bytes": dl["bytes"], "url": dl["url"]}
except Exception as exc:
return {"name": name, "status": f"failed:decrypt-{type(exc).__name__}", "bytes": dl["bytes"], "url": dl["url"]}
def progress_line(done: int, total: int, result: dict, started_at: float, bytes_done: int) -> str:
elapsed = max(0.001, time.monotonic() - started_at)
files_per_sec = done / elapsed
mib_per_sec = bytes_done / 1024 / 1024 / elapsed
return (
f"[INFO] [步骤 3/3] 处理进度:current={done}/{total},percent={done / total * 100:.1f}%,"
f"name={result['name']},status={result['status']},rate={files_per_sec:.1f}/s,speed={mib_per_sec:.2f}MiB/s"
)
def run_parallel(
rows: list[dict],
zpk_dir: Path,
sink: JsonlSink,
workers: int,
progress_every: int,
cdn_mode: str,
decrypt_zpk_bytes,
flatten,
include_raw: bool,
) -> list[dict]:
work: queue.Queue[dict | None] = queue.Queue(maxsize=max(1, workers * 4))
results: list[dict] = []
results_lock = threading.Lock()
started_at = time.monotonic()
done = 0
bytes_done = 0
total = len(rows)
def record(result: dict) -> None:
nonlocal done, bytes_done
with results_lock:
results.append(result)
done += 1
if result["status"] in {"decrypted"}:
bytes_done += int(result["bytes"])
if done % progress_every == 0 or done == total or STOP_EVENT.is_set():
print(progress_line(done, total, result, started_at, bytes_done), flush=True)
def worker(worker_index: int) -> None:
client = CdnClient(build_endpoints(cdn_mode, worker_index), REQUEST_TIMEOUT)
try:
while not STOP_EVENT.is_set():
try:
row = work.get(timeout=0.25)
except queue.Empty:
continue
try:
if row is None:
return
record(process_one(row, zpk_dir, client, sink, decrypt_zpk_bytes, flatten, include_raw))
finally:
work.task_done()
finally:
client.close()
threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(workers)]
for thread in threads:
thread.start()
try:
for row in rows:
while not STOP_EVENT.is_set():
try:
work.put(row, timeout=0.25)
break
except queue.Full:
continue
if STOP_EVENT.is_set():
break
for _ in threads:
while not STOP_EVENT.is_set():
try:
work.put(None, timeout=0.25)
break
except queue.Full:
continue
work.join()
except KeyboardInterrupt:
STOP_EVENT.set()
print(f"\n[中断] ZPK 已处理 {done}/{total},已保留已完成结果。", flush=True)
finally:
for thread in threads:
thread.join(timeout=REQUEST_TIMEOUT + 1)
return results
def write_report(rows: list[dict], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", newline="", encoding="utf-8-sig") as f:
writer = csv.DictWriter(f, fieldnames=["name", "status", "bytes", "url"])
writer.writeheader()
writer.writerows(rows)
def main() -> int:
global REQUEST_TIMEOUT
parser = argparse.ArgumentParser(description="从全量词表去重下载所有 ZPK 并解密为 JSONL。")
parser.add_argument("--words", type=Path, default=WORDS_CSV, help="词书解密目录或 CSV 文件")
parser.add_argument("--out-dir", type=Path, default=ZPK_DIR)
parser.add_argument("--jsonl", type=Path, default=ZPK_JSONL, help="解密输出目录,每个 ZPK 一个 JSON")
parser.add_argument("--workers", type=int, default=64)
parser.add_argument("--timeout", type=float, default=8, help="单个 CDN 请求超时秒数;Ctrl+C 最多等待活跃请求到此超时")
parser.add_argument("--progress-every", type=int, default=500)
parser.add_argument(
"--cdn-mode",
choices=("spread", "primary"),
default="primary",
help="primary 会始终优先第一个 CDN;spread 会把 worker 分散到 3 个 CDN",
)
parser.add_argument("--limit", type=int, default=0, help="最多处理 N 个(冒烟测试)")
parser.add_argument("--force", action="store_true", help="已解密也重新下载解密")
parser.add_argument("--raw", action="store_true", default=True, help="保留完整原始 JSON 对象(默认开启)")
args = parser.parse_args()
REQUEST_TIMEOUT = max(1.0, args.timeout)
zpk_dir = args.out_dir
if not args.words.exists():
print(f"词表不存在:{args.words},请先运行 src/download-wordbooks.py", file=sys.stderr)
return 1
decrypt_zpk_bytes, flatten = load_decrypt_zpk()
index_rows = build_unique_index(args.words)
print(f"[INFO] [步骤 3/3] 索引准备完成:total={len(index_rows)}", flush=True)
already_decrypted = set() if args.force else load_already_decrypted(args.jsonl)
rows = [r for r in index_rows if r["zpk"] not in already_decrypted]
print(f"[INFO] [步骤 3/3] 断点状态:completed={len(already_decrypted)},pending={len(rows)}", flush=True)
if args.limit:
rows = rows[: args.limit]
total = len(rows)
print(f"[INFO] [步骤 3/3] 开始下载并解密:total={total}", flush=True)
sink = JsonlSink(args.jsonl)
workers = max(1, args.workers)
try:
if workers == 1:
endpoint_client = CdnClient(build_endpoints(args.cdn_mode), REQUEST_TIMEOUT)
results = []
started_at = time.monotonic()
bytes_done = 0
try:
for done, row in enumerate(rows, 1):
r = process_one(row, zpk_dir, endpoint_client, sink, decrypt_zpk_bytes, flatten, args.raw)
results.append(r)
if r["status"] == "decrypted":
bytes_done += int(r["bytes"])
print(progress_line(done, total, r, started_at, bytes_done), flush=True)
except KeyboardInterrupt:
STOP_EVENT.set()
print(f"\n[中断] ZPK 已处理 {len(results)}/{total},已保留已完成结果。", flush=True)
finally:
endpoint_client.close()
else:
results = run_parallel(
rows, zpk_dir, sink, workers, max(1, args.progress_every), args.cdn_mode,
decrypt_zpk_bytes, flatten, args.raw,
)
finally:
sink.close()
ok = sum(1 for r in results if r["status"] == "decrypted")
failed = len(results) - ok
cancelled = sum(1 for r in results if r["status"] == "cancelled")
print(f"[INFO] [步骤 3/3] 处理完成:success={ok},failed={failed},skipped={cancelled}", flush=True)
if STOP_EVENT.is_set():
return 130
return 0 if failed == 0 else 2
if __name__ == "__main__":
sys.exit(main())
+98
View File
@@ -0,0 +1,98 @@
#!/usr/bin/env python3
"""Export synonym data from bbdb and derivative/family data from wordroot."""
from __future__ import annotations
import csv
import argparse
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "output" / "synonym-derivative-export"
PACKAGE = "cn.com.langeasy.LangEasyLexis"
KEYS = {
"bbdb": "North_London_bbdb_Forever_20230710",
"wordroot": "North_wordroot_London_Forever_20220601",
}
def connect(path: Path, key: str):
try:
import sqlcipher3.dbapi2 as dbapi
except ImportError as exc:
raise SystemExit("请先安装 sqlcipher3:python -m pip install sqlcipher3") from exc
conn = dbapi.connect(str(path))
conn.execute(f"PRAGMA key='{key}'")
conn.execute("SELECT count(*) FROM sqlite_master").fetchone()
return conn
def adb_export(serial: str | None, remote_name: str, local_path: Path) -> None:
target = ["adb"] + (["-s", serial] if serial else [])
target += ["exec-out", "cat", f"/data/data/{PACKAGE}/databases/{remote_name}"]
with local_path.open("wb") as fh:
result = subprocess.run(target, stdout=fh, stderr=subprocess.PIPE)
if result.returncode != 0:
target = ["adb"] + (["-s", serial] if serial else [])
target += ["exec-out", "run-as", PACKAGE, "cat", f"databases/{remote_name}"]
with local_path.open("wb") as fh:
result = subprocess.run(target, stdout=fh, stderr=subprocess.PIPE)
if result.returncode != 0 or local_path.stat().st_size == 0:
detail = result.stderr.decode(errors="replace").strip()
raise SystemExit(f"ADB 导出失败: {remote_name}: {detail}")
def dump_query(conn, sql: str, name: str) -> int:
cur = conn.execute(sql)
path = OUT / name
with path.open("w", newline="", encoding="utf-8-sig") as fh:
writer = csv.writer(fh)
writer.writerow([d[0] for d in cur.description])
rows = cur.fetchall()
writer.writerows(rows)
return len(rows)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--serial", help="adb 设备序列号;默认使用当前设备")
args = parser.parse_args()
OUT.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="lexis-adb-export-") as tmp:
bb_path = Path(tmp) / "bbdb.db"
wr_path = Path(tmp) / "wordroot.db"
adb_export(args.serial, "bbdb.db", bb_path)
adb_export(args.serial, "wordroot.db", wr_path)
bb = connect(bb_path, KEYS["bbdb"])
wr = connect(wr_path, KEYS["wordroot"])
try:
counts = {}
counts["synonym_words.csv"] = dump_query(bb, "SELECT id, word FROM t_synonym_words ORDER BY id", "synonym_words.csv")
counts["synonym_data.csv"] = dump_query(bb, "SELECT * FROM t_synonym_datas ORDER BY word_id, pos, id", "synonym_data.csv")
counts["synonyms-joined.csv"] = dump_query(bb, """SELECT w.word, d.pos, d.interpret, d.zk, d.gk, d.cet4, d.cet6, d.ky, d.ielts, d.zsb, d.other, d.toefl FROM t_synonym_words w JOIN t_synonym_datas d ON d.word_id=w.id ORDER BY w.word, d.pos, d.id""", "synonyms-joined.csv")
counts["derivative-family.csv"] = dump_query(wr, """SELECT id, word, top_id, grade, interpret, prefix, suffix, group1, group2, group3, json FROM word_family ORDER BY top_id, id""", "derivative-family.csv")
counts["derivative-mini-groups.csv"] = dump_query(wr, "SELECT id, word_ids FROM word_family_mini_group ORDER BY id", "derivative-mini-groups.csv")
finally:
bb.close()
wr.close()
(OUT / "synonym-derivative-summary.txt").write_text(
"近义词与派生串记导出\n"
f"近义词单词: {counts['synonym_words.csv']}\n"
f"近义词释义记录: {counts['synonym_data.csv']}\n"
f"近义词关联记录: {counts['synonyms-joined.csv']}\n"
f"派生词族记录: {counts['derivative-family.csv']}\n"
f"派生小组记录: {counts['derivative-mini-groups.csv']}\n",
encoding="utf-8",
)
print("已完成近义词和派生串记导出")
for name, count in counts.items():
print(f"{name}: {count}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+120
View File
@@ -0,0 +1,120 @@
#!/usr/bin/env python3
"""Export the encrypted word-root database into CSV files.
The script supports pysqlcipher3/sqlcipher3 and the sqlcipher CLI. The
database is copied to a temporary file before opening so the source remains
unchanged.
"""
from __future__ import annotations
import argparse
import csv
import shutil
import sqlite3
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_DB = ROOT / "assets" / "wordroot.db"
DEFAULT_OUT = ROOT / "output" / "wordroot-export"
DB_KEY = "North_wordroot_London_Forever_20220601"
LOCAL_RUNTIME = ROOT / ".runtime" / "sqlcipher3"
if LOCAL_RUNTIME.exists():
sys.path.insert(0, str(LOCAL_RUNTIME))
def open_python_cipher(db: Path):
for module_name in ("pysqlcipher3.dbapi2", "sqlcipher3.dbapi2"):
try:
module = __import__(module_name, fromlist=["connect"])
except ImportError:
continue
conn = module.connect(str(db))
conn.execute(f"PRAGMA key='{DB_KEY}'")
conn.execute("PRAGMA cipher_compatibility=4")
conn.execute("SELECT count(*) FROM sqlite_master").fetchone()
return conn
return None
def export_with_cli(db: Path, out_dir: Path) -> None:
sqlcipher = shutil.which("sqlcipher")
if not sqlcipher:
raise RuntimeError(
"未找到 SQLCipher。请安装 pysqlcipher3/sqlcipher3,或将 sqlcipher 加入 PATH。"
)
out_dir.mkdir(parents=True, exist_ok=True)
script = (
f"PRAGMA key='{DB_KEY}';\n"
"PRAGMA cipher_compatibility=4;\n"
".mode csv\n"
".headers on\n"
)
tables = subprocess.run(
[sqlcipher, str(db)], input=script + "SELECT name FROM sqlite_master WHERE type='table';\n",
text=True, capture_output=True, check=True,
).stdout.splitlines()
for table in (line.strip() for line in tables):
if not table or table == "name" or table.startswith("---"):
continue
safe = "".join(c if c.isalnum() or c in "-_" else "_" for c in table)
query = script + f'.output "{out_dir / (safe + ".csv")}"\nSELECT * FROM "{table}";\n.output stdout\n'
subprocess.run([sqlcipher, str(db)], input=query, text=True, check=True)
def export_with_python(conn, out_dir: Path) -> list[str]:
out_dir.mkdir(parents=True, exist_ok=True)
names = [row[0] for row in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
)]
for table in names:
safe = "".join(c if c.isalnum() or c in "-_" else "_" for c in table)
cur = conn.execute(f'SELECT * FROM "{table}"')
with (out_dir / f"{safe}.csv").open("w", newline="", encoding="utf-8-sig") as fh:
writer = csv.writer(fh)
writer.writerow([col[0] for col in cur.description])
writer.writerows(cur)
return names
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--db", type=Path, default=DEFAULT_DB)
parser.add_argument("--output", type=Path, default=DEFAULT_OUT)
args = parser.parse_args()
db = args.db.resolve()
if not db.exists():
raise SystemExit(f"数据库不存在: {db}")
with tempfile.TemporaryDirectory(prefix="lexis-wordroot-") as tmp:
local_db = Path(tmp) / "wordroot.db"
shutil.copy2(db, local_db)
conn = open_python_cipher(local_db)
if conn is not None:
try:
tables = export_with_python(conn, args.output.resolve())
finally:
conn.close()
else:
export_with_cli(local_db, args.output.resolve())
tables = [p.stem for p in args.output.resolve().glob("*.csv")]
manifest = args.output.resolve() / "README.txt"
manifest.write_text(
"wordroot.db 导出结果\n"
f"源数据库: {db}\n"
f"表数量: {len(tables)}\n"
"CSV 使用 UTF-8-BOM 编码。\n",
encoding="utf-8",
)
print(f"已导出 {len(tables)} 张表到: {args.output.resolve()}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+102
View File
@@ -0,0 +1,102 @@
"""从服务端词书列表接口拉取词书目录,替代从设备偏好文件提取。
接口:GET https://api.beingfine.cn/v3/3/bb/wordbooks/for-guest/{timestamp}
该接口是游客态全量词书列表,无需登录,响应 data_body 为明文 JSON。
"""
import argparse
import json
import sys
import time
import urllib.parse
import urllib.request
from pathlib import Path
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
API_PATH = "https://api.beingfine.cn/v3/3/bb/wordbooks/for-guest"
APP_ID = "600000001"
APP_VERSION = "5.11.1"
CHANNEL = "isCool"
TOKEN = "tk85bf4d7d70a4d5028187ba6dfb3649"
REFERER = "https://app.bbdc.cn"
USER_AGENT = "LangEasyLexis/5.11.1"
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "output" / "book-catalog.json"
FIELDS = ("id", "code", "name", "count", "exam", "url", "version", "bookVersion", "purchased", "forSale")
def compact_book(book):
return {key: book.get(key) for key in FIELDS if key in book}
def fetch_catalog(timeout=20):
"""拉取游客态词书列表,返回 (books, raw_body)。"""
ts = int(time.time() * 1000)
url = f"{API_PATH}/{ts}"
params = {
"type": "3/bb/wordbooks/for-guest",
"app_id": APP_ID,
"udid": "",
"umid": "",
"version": APP_VERSION,
"user_id": "0",
"channel": CHANNEL,
"timestamp": str(ts),
"timezone": "480",
"pad": "0",
"token": TOKEN,
}
qs = "&".join(f"{k}={urllib.parse.quote(str(v), safe='')}" for k, v in params.items())
req = urllib.request.Request(
url + "?" + qs,
headers={"Referer": REFERER, "User-Agent": USER_AGENT},
)
with urllib.request.urlopen(req, timeout=timeout) as resp:
raw = resp.read().decode("utf-8")
outer = json.loads(raw)
if outer.get("result_code") != 200:
raise RuntimeError(f"result_code={outer.get('result_code')} body={raw[:200]}")
data_body = outer.get("data_body")
if isinstance(data_body, str):
data_body = json.loads(data_body)
if not isinstance(data_body, dict):
raise RuntimeError(f"data_body 缺失或格式异常: {raw[:200]}")
books = data_body.get("books") or []
return books, data_body
def main():
parser = argparse.ArgumentParser(description="从服务端拉取 Lexis 词书目录")
parser.add_argument("--output", default=OUT, help="输出词书目录 JSON 路径")
args = parser.parse_args()
print("[INFO] [步骤 1/3] 开始获取词书目录", flush=True)
started = time.monotonic()
books, full = fetch_catalog()
catalog = sorted(
books,
key=lambda item: (str(item.get("exam", "")), str(item.get("code", ""))),
)
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
with open(args.output, "w", encoding="utf-8") as f:
json.dump(catalog, f, ensure_ascii=False, indent=2)
f.write("\n")
categories = {}
for item in catalog:
category = str(item.get("exam") or "未分类")
categories[category] = categories.get(category, 0) + 1
elapsed = time.monotonic() - started
category_text = ", ".join(f"{name}: {count}" for name, count in sorted(categories.items()))
print(f"[INFO] [步骤 1/3] 词书目录获取完成:数量={len(catalog)},耗时={elapsed:.1f}s")
print(f"[INFO] [步骤 1/3] 分类统计:{category_text}")
print(f"[INFO] [步骤 1/3] 输出文件:{args.output}")
if __name__ == "__main__":
main()