feat(importer): 添加 Lexis 词书导出工具链
三步主流程(下载目录 -> 下载词书并解密导出 -> 下载并解密 ZPK), 以及可独立调用的 SQLCipher2 / ZPK 解密工具与词根、近义词导出脚本。 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
a39ba9081e
commit
f030334243
9 files changed
+3517
No files matched your search
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
from Crypto.Cipher import AES
|
||||
|
||||
|
||||
KEY = b"arsenal iscool"
|
||||
PAGE_SIZE = 1024
|
||||
RESERVE = 48
|
||||
KDF_ITER = 4000
|
||||
SQLITE_HEADER = b"SQLite format 3\x00"
|
||||
|
||||
|
||||
def derive_key(salt: bytes) -> bytes:
|
||||
return hashlib.pbkdf2_hmac("sha1", KEY, salt, KDF_ITER, 32)
|
||||
|
||||
|
||||
def decrypt_db(src: Path, dst: Path) -> None:
|
||||
blob = src.read_bytes()
|
||||
if len(blob) % PAGE_SIZE:
|
||||
raise ValueError(f"input size is not aligned to {PAGE_SIZE}: {src}")
|
||||
|
||||
key = derive_key(blob[:16])
|
||||
out = bytearray()
|
||||
pages = len(blob) // PAGE_SIZE
|
||||
for page_no in range(pages):
|
||||
page = blob[page_no * PAGE_SIZE : (page_no + 1) * PAGE_SIZE]
|
||||
iv = page[PAGE_SIZE - RESERVE : PAGE_SIZE - RESERVE + 16]
|
||||
if page_no == 0:
|
||||
enc_start = 16
|
||||
prefix = SQLITE_HEADER
|
||||
else:
|
||||
enc_start = 0
|
||||
prefix = b""
|
||||
enc_end = PAGE_SIZE - RESERVE
|
||||
dec = AES.new(key, AES.MODE_CBC, iv).decrypt(page[enc_start:enc_end])
|
||||
plain = prefix + dec
|
||||
if len(plain) > PAGE_SIZE:
|
||||
raise ValueError("decrypted page overflow")
|
||||
out.extend(plain)
|
||||
out.extend(b"\x00" * (PAGE_SIZE - len(plain)))
|
||||
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
dst.write_bytes(out)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("src", type=Path)
|
||||
parser.add_argument("dst", type=Path)
|
||||
args = parser.parse_args()
|
||||
decrypt_db(args.src, args.dst)
|
||||
print(args.dst)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Decrypt Lexis .zpk files offline.
|
||||
|
||||
ZPK format:
|
||||
outer protobuf: field 1 length-delimited AES ciphertext
|
||||
cipher: AES-256-CBC + PKCS7/PKCS5 padding
|
||||
key: md5(word).hexdigest().encode("ascii") # 32 bytes
|
||||
iv: b"IscoolInitVector"
|
||||
|
||||
The word -> zpk mapping can be supplied by output/unique-zpk-index.csv.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from Crypto.Cipher import AES
|
||||
from Crypto.Util.Padding import unpad
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_INDEX = ROOT / "output" / "unique-zpk-index.csv"
|
||||
DEFAULT_ZPK_DIR = ROOT / "output" / "zpk" / "original"
|
||||
DEFAULT_OUTPUT = ROOT / "output" / "zpk" / "decrypted" / "zpk-decrypted.jsonl"
|
||||
IV = b"IscoolInitVector"
|
||||
|
||||
|
||||
def read_varint(data: bytes, offset: int) -> tuple[int, int]:
|
||||
value = 0
|
||||
shift = 0
|
||||
while offset < len(data):
|
||||
b = data[offset]
|
||||
offset += 1
|
||||
value |= (b & 0x7F) << shift
|
||||
if b < 0x80:
|
||||
return value, offset
|
||||
shift += 7
|
||||
raise ValueError("unterminated varint")
|
||||
|
||||
|
||||
def read_field1_payload(data: bytes) -> bytes:
|
||||
if not data or data[0] != 0x0A:
|
||||
raise ValueError("bad zpk outer protobuf header")
|
||||
size, offset = read_varint(data, 1)
|
||||
payload = data[offset : offset + size]
|
||||
if len(payload) != size:
|
||||
raise ValueError(f"truncated payload: expected={size} actual={len(payload)}")
|
||||
return payload
|
||||
|
||||
|
||||
def decrypt_zpk_bytes(raw: bytes, word: str) -> dict:
|
||||
ciphertext = read_field1_payload(raw)
|
||||
key = hashlib.md5(word.encode("utf-8")).hexdigest().encode("ascii")
|
||||
plaintext = AES.new(key, AES.MODE_CBC, IV).decrypt(ciphertext)
|
||||
plaintext = unpad(plaintext, 16)
|
||||
inner = read_field1_payload(plaintext)
|
||||
return json.loads(inner.decode("utf-8"))
|
||||
|
||||
|
||||
def load_index(path: Path) -> list[dict[str, str]]:
|
||||
with path.open("r", encoding="utf-8-sig", newline="") as fh:
|
||||
return [row for row in csv.DictReader(fh) if row.get("zpk") and row.get("first_word")]
|
||||
|
||||
|
||||
def flatten(obj: dict) -> dict:
|
||||
return {
|
||||
"word": obj.get("word", ""),
|
||||
"us_pron": obj.get("us_pron", ""),
|
||||
"uk_pron": obj.get("uk_pron", ""),
|
||||
"interpret": obj.get("interpret", ""),
|
||||
"definition_count": sum(len(g.get("d") or []) for g in (obj.get("interpret_v2") or []) if isinstance(g, dict)),
|
||||
"phrase_count": sum(len(g.get("p") or []) for g in (obj.get("phrase") or []) if isinstance(g, dict)),
|
||||
"raw": obj,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
|
||||
parser.add_argument("--zpk-dir", type=Path, default=DEFAULT_ZPK_DIR)
|
||||
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||||
parser.add_argument("--limit", type=int, default=0)
|
||||
parser.add_argument("--raw", action="store_true", help="include full raw JSON object")
|
||||
args = parser.parse_args()
|
||||
|
||||
rows = load_index(args.index)
|
||||
if args.limit:
|
||||
rows = rows[: args.limit]
|
||||
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
ok = missing = failed = 0
|
||||
with args.output.open("w", encoding="utf-8", newline="\n") as out:
|
||||
for row in rows:
|
||||
zpk = row["zpk"]
|
||||
word = row["first_word"]
|
||||
path = args.zpk_dir / f"{zpk}.zpk"
|
||||
if not path.exists():
|
||||
missing += 1
|
||||
continue
|
||||
try:
|
||||
obj = decrypt_zpk_bytes(path.read_bytes(), word)
|
||||
rec = dict(obj)
|
||||
rec.update(
|
||||
{
|
||||
"zpk": zpk,
|
||||
"zpk_path": str(path),
|
||||
"index_word": word,
|
||||
"book_occurrences": row.get("book_occurrences", ""),
|
||||
}
|
||||
)
|
||||
if args.raw:
|
||||
rec["raw"] = obj
|
||||
out.write(json.dumps(rec, ensure_ascii=False, separators=(",", ":")) + "\n")
|
||||
ok += 1
|
||||
except Exception as exc:
|
||||
failed += 1
|
||||
if failed <= 20:
|
||||
print(f"[fail] {zpk} word={word}: {exc}")
|
||||
|
||||
print(f"ok={ok} missing={missing} failed={failed} -> {args.output}")
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,459 @@
|
||||
#!/usr/bin/env python3
|
||||
"""步骤2:逐个下载所有词书,解包、解密 SQLCipher2 DB 并导出单词与 zpk 字段。
|
||||
|
||||
每本词书走完整链路:下载 zip -> 校验 MD5 -> 解包 -> 解密 -> 导出单词 CSV。
|
||||
解密逻辑复用独立工具 decrypt-sqlcipher2-db.py(可单独调用)。
|
||||
|
||||
已下载的 DB 自动跳过(可用 --force 重下),已解密的 DB 按 mtime 复用(可用 --force-decrypt 重解)。
|
||||
|
||||
输出(默认统一到 output/):
|
||||
- output/databases/LangEasyLexisV3_<code>.db 原始加密 DB(中间缓存)
|
||||
- output/decrypted-databases/LangEasyLexisV3_<code>.sqlite 解密后 DB(可读)
|
||||
- output/wordbook-words/<code>.csv 每本词书单词(可读)
|
||||
- output/all-wordbook-words.csv 全量合并词表(可读,步骤3 的输入)
|
||||
- output/wordbook-download-report.csv 下载报告
|
||||
- output/wordbook-word-summary.csv 导出报告
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import csv
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import zipfile
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT_DIR = ROOT / "output"
|
||||
DEFAULT_CATALOG = OUT_DIR / "book-catalog.json"
|
||||
WORDBOOK_DIR = OUT_DIR / "wordbooks"
|
||||
DB_DIR = WORDBOOK_DIR / "original"
|
||||
DECRYPTED_DIR = WORDBOOK_DIR / "decrypted"
|
||||
|
||||
DECRYPTOR_PATH = ROOT / "src" / "decrypt-sqlcipher2-db.py"
|
||||
|
||||
BASE_STATIC_URLS = [
|
||||
"http://static.beingfine.cn",
|
||||
"https://hwstatic.beingfine.cn",
|
||||
"https://ccastatic.beingfine.cn",
|
||||
]
|
||||
REFERER = "https://app.bbdc.cn"
|
||||
USER_AGENT = "LangEasyLexis/5.11.1"
|
||||
|
||||
WORD_FIELDS = (
|
||||
"book_id",
|
||||
"book_code",
|
||||
"book_name",
|
||||
"book_count",
|
||||
"book_exam",
|
||||
"book_version",
|
||||
"id",
|
||||
"word",
|
||||
"main_word",
|
||||
"seq_book",
|
||||
"seq_az",
|
||||
"seq_freq",
|
||||
"seq_rand",
|
||||
"zpk",
|
||||
)
|
||||
|
||||
SUMMARY_FIELDS = (
|
||||
"book_id",
|
||||
"book_code",
|
||||
"book_name",
|
||||
"book_count",
|
||||
"exported_count",
|
||||
"status",
|
||||
"table",
|
||||
"local_db",
|
||||
"decrypted_db",
|
||||
"export_csv",
|
||||
"error",
|
||||
)
|
||||
|
||||
DOWNLOAD_REPORT_FIELDS = ("code", "status", "bytes", "url")
|
||||
|
||||
|
||||
def load_decrypt_db():
|
||||
"""加载独立解密工具 decrypt-sqlcipher2-db.py 的 decrypt_db 函数。"""
|
||||
spec = importlib.util.spec_from_file_location("decrypt_sqlcipher2_db", DECRYPTOR_PATH)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError(f"Cannot load decryptor: {DECRYPTOR_PATH}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module.decrypt_db
|
||||
|
||||
|
||||
def load_books(path: Path) -> list[dict]:
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, dict) and isinstance(data.get("books"), list):
|
||||
return data["books"]
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
raise ValueError(f"Unsupported catalog shape: {path}")
|
||||
|
||||
|
||||
def book_url(book: dict, base_url: str) -> str:
|
||||
code = book["code"]
|
||||
version = str(book.get("version") or 0)
|
||||
db_name = f"LangEasyLexisV3_{code}.db"
|
||||
return f"{base_url}/dicts/v3/{code}/{version}/{db_name}.zip"
|
||||
|
||||
|
||||
def request_url(url: str):
|
||||
req = Request(url, headers={"Referer": REFERER, "User-Agent": USER_AGENT})
|
||||
return urlopen(req, timeout=45)
|
||||
|
||||
|
||||
def md5_b64(path: Path) -> str:
|
||||
h = hashlib.md5()
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return base64.b64encode(h.digest()).decode("ascii")
|
||||
|
||||
|
||||
def safe_code_filename(code: object) -> str:
|
||||
text = str(code or "UNKNOWN")
|
||||
return re.sub(r"[^A-Za-z0-9_.-]+", "_", text).strip("._") or "UNKNOWN"
|
||||
|
||||
|
||||
def quote_identifier(name: str) -> str:
|
||||
return '"' + name.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def list_tables(conn: sqlite3.Connection) -> list[str]:
|
||||
return [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"select name from sqlite_master where type='table' and name not like 'sqlite_%' order by name"
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def table_columns(conn: sqlite3.Connection, table: str) -> set[str]:
|
||||
return {row[1] for row in conn.execute(f"pragma table_info({quote_identifier(table)})")}
|
||||
|
||||
|
||||
def choose_word_table(conn: sqlite3.Connection, code: str) -> str:
|
||||
tables = list_tables(conn)
|
||||
preferred = f"LangEasyLexisV3_{code}"
|
||||
if preferred in tables:
|
||||
return preferred
|
||||
|
||||
for table in tables:
|
||||
if {"id", "word"}.issubset(table_columns(conn, table)):
|
||||
return table
|
||||
|
||||
raise RuntimeError(f"no word table found; tables={tables}")
|
||||
|
||||
|
||||
def iter_words(conn: sqlite3.Connection, table: str) -> Iterable[sqlite3.Row]:
|
||||
conn.row_factory = sqlite3.Row
|
||||
cols = table_columns(conn, table)
|
||||
required = {"id", "word", "main_word", "seq_book", "seq_az", "seq_freq", "seq_rand", "zpk"}
|
||||
missing = sorted(required - cols)
|
||||
if missing:
|
||||
raise RuntimeError(f"table {table} missing columns: {missing}")
|
||||
|
||||
query = (
|
||||
f"select id, word, main_word, seq_book, seq_az, seq_freq, seq_rand, zpk "
|
||||
f"from {quote_identifier(table)} order by seq_book, id"
|
||||
)
|
||||
return conn.execute(query)
|
||||
|
||||
|
||||
def download_book_db(book: dict, force: bool) -> dict:
|
||||
"""下载单本词书 DB zip 并解包,返回下载状态 dict。"""
|
||||
code = book["code"]
|
||||
db_path = DB_DIR / f"LangEasyLexisV3_{code}.db"
|
||||
|
||||
if db_path.exists() and not force:
|
||||
return {"status": "skip-db-exists", "url": "", "bytes": db_path.stat().st_size}
|
||||
|
||||
last_error = ""
|
||||
for base in BASE_STATIC_URLS:
|
||||
url = book_url(book, base)
|
||||
try:
|
||||
with request_url(url) as resp:
|
||||
status = getattr(resp, "status", 200)
|
||||
if status != 200:
|
||||
last_error = f"http-{status}"
|
||||
continue
|
||||
|
||||
tmp_file = tempfile.NamedTemporaryFile(prefix=f"lexis-{code}-", suffix=".zip", delete=False)
|
||||
tmp_path = Path(tmp_file.name)
|
||||
with tmp_file as out:
|
||||
while True:
|
||||
chunk = resp.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
out.write(chunk)
|
||||
|
||||
content_md5 = resp.headers.get("Content-MD5")
|
||||
if content_md5 and md5_b64(tmp_path) != content_md5:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
last_error = "md5-mismatch"
|
||||
continue
|
||||
|
||||
DB_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(tmp_path) as zf:
|
||||
names = zf.namelist()
|
||||
wanted = f"LangEasyLexisV3_{code}.db"
|
||||
member = wanted if wanted in names else names[0]
|
||||
zf.extract(member, DB_DIR)
|
||||
extracted = DB_DIR / member
|
||||
if extracted != db_path:
|
||||
extracted.replace(db_path)
|
||||
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
return {"status": "downloaded", "url": url, "bytes": db_path.stat().st_size}
|
||||
except HTTPError as e:
|
||||
last_error = f"http-{e.code}"
|
||||
except (URLError, TimeoutError, zipfile.BadZipFile, OSError) as e:
|
||||
last_error = type(e).__name__
|
||||
finally:
|
||||
if "tmp_path" in locals():
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
return {"status": f"failed:{last_error}", "url": "", "bytes": 0}
|
||||
|
||||
|
||||
def export_book_words(row: dict, decrypt_db, force_decrypt: bool) -> dict:
|
||||
"""解密并导出一本词书的单词,返回导出 summary dict。"""
|
||||
code = str(row.get("code") or "")
|
||||
src = DB_DIR / f"LangEasyLexisV3_{code}.db"
|
||||
if not src.exists():
|
||||
return summary(row, "missing_db", "", src, Path(), Path(), 0, f"not found: {src}")
|
||||
|
||||
export_csv = DECRYPTED_DIR / f"LangEasyLexisV3_{safe_code_filename(code)}.csv"
|
||||
DECRYPTED_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 解密 DB:已存在且比源新则复用
|
||||
# SQLite 仅作为临时解密载体,不作为 output 产物。
|
||||
# Windows 可能在 SQLite 连接关闭后仍短暂持有文件句柄;清理失败不能影响导出结果。
|
||||
with tempfile.TemporaryDirectory(prefix="lexis-decrypted-", ignore_cleanup_errors=True) as temp_dir:
|
||||
decrypted = Path(temp_dir) / src.with_suffix(".sqlite").name
|
||||
decrypt_db(src, decrypted)
|
||||
with sqlite3.connect(decrypted) as conn:
|
||||
table = choose_word_table(conn, code)
|
||||
count = 0
|
||||
with export_csv.open("w", encoding="utf-8-sig", newline="") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=WORD_FIELDS)
|
||||
writer.writeheader()
|
||||
for word_row in iter_words(conn, table):
|
||||
writer.writerow(
|
||||
{
|
||||
"book_id": row.get("id", ""),
|
||||
"book_code": code,
|
||||
"book_name": row.get("name", ""),
|
||||
"book_count": row.get("count", ""),
|
||||
"book_exam": row.get("exam", ""),
|
||||
"book_version": row.get("version", ""),
|
||||
"id": word_row["id"],
|
||||
"word": word_row["word"],
|
||||
"main_word": word_row["main_word"],
|
||||
"seq_book": word_row["seq_book"],
|
||||
"seq_az": word_row["seq_az"],
|
||||
"seq_freq": word_row["seq_freq"],
|
||||
"seq_rand": word_row["seq_rand"],
|
||||
"zpk": word_row["zpk"],
|
||||
}
|
||||
)
|
||||
count += 1
|
||||
|
||||
return summary(row, "ok", table, src, decrypted, export_csv, count, "")
|
||||
|
||||
|
||||
def summary(
|
||||
row: dict,
|
||||
status: str,
|
||||
table: str,
|
||||
src: Path,
|
||||
decrypted: Path,
|
||||
export_csv: Path,
|
||||
exported_count: int,
|
||||
error: str,
|
||||
) -> dict:
|
||||
return {
|
||||
"book_id": row.get("id", ""),
|
||||
"book_code": row.get("code", ""),
|
||||
"book_name": row.get("name", ""),
|
||||
"book_count": row.get("count", ""),
|
||||
"exported_count": exported_count,
|
||||
"status": status,
|
||||
"table": table,
|
||||
"local_db": str(src) if src else "",
|
||||
"decrypted_db": str(decrypted) if decrypted else "",
|
||||
"export_csv": str(export_csv) if export_csv else "",
|
||||
"error": error,
|
||||
}
|
||||
|
||||
|
||||
def process_one(book: dict, decrypt_db, force_download: bool, force_decrypt: bool) -> dict:
|
||||
"""处理一本词书:下载 + 解密 + 导出,返回合并结果 dict。"""
|
||||
code = book["code"]
|
||||
result = {
|
||||
"code": code,
|
||||
"dl_status": "",
|
||||
"dl_url": "",
|
||||
"dl_bytes": 0,
|
||||
"export": None,
|
||||
}
|
||||
|
||||
if book.get("url") == "*":
|
||||
result["dl_status"] = "skip-virtual-book"
|
||||
return result
|
||||
|
||||
dl = download_book_db(book, force_download)
|
||||
result["dl_status"] = dl["status"]
|
||||
result["dl_url"] = dl["url"]
|
||||
result["dl_bytes"] = dl["bytes"]
|
||||
|
||||
if dl["status"] not in {"downloaded", "skip-db-exists"}:
|
||||
result["export"] = summary(book, "download_failed", "", DB_DIR / f"LangEasyLexisV3_{code}.db", Path(), Path(), 0, dl["status"])
|
||||
return result
|
||||
|
||||
result["export"] = export_book_words(book, decrypt_db, force_decrypt)
|
||||
return result
|
||||
|
||||
|
||||
def write_download_report(results: list[dict], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", newline="", encoding="utf-8-sig") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=DOWNLOAD_REPORT_FIELDS)
|
||||
writer.writeheader()
|
||||
for r in results:
|
||||
writer.writerow(
|
||||
{"code": r["code"], "status": r["dl_status"], "bytes": r["dl_bytes"], "url": r["dl_url"]}
|
||||
)
|
||||
|
||||
|
||||
def write_summary(summaries: list[dict], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=SUMMARY_FIELDS)
|
||||
writer.writeheader()
|
||||
writer.writerows(summaries)
|
||||
|
||||
|
||||
def write_combined(summaries: list[dict], path: Path) -> int:
|
||||
total = 0
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as out_fh:
|
||||
writer = csv.DictWriter(out_fh, fieldnames=WORD_FIELDS)
|
||||
writer.writeheader()
|
||||
for item in summaries:
|
||||
if item["status"] != "ok":
|
||||
continue
|
||||
with Path(item["export_csv"]).open("r", encoding="utf-8-sig", newline="") as in_fh:
|
||||
reader = csv.DictReader(in_fh)
|
||||
for row in reader:
|
||||
writer.writerow(row)
|
||||
total += 1
|
||||
return total
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="逐个下载所有词书:下载 -> 解密 -> 导出单词")
|
||||
parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
|
||||
parser.add_argument("--force", action="store_true", help="重新下载已存在的词书 DB")
|
||||
parser.add_argument("--force-decrypt", action="store_true", help="重新解密已存在的解密 DB")
|
||||
parser.add_argument("--limit", type=int, default=0, help="最多处理 N 本词书(冒烟测试)")
|
||||
parser.add_argument("--sleep", type=float, default=0.0, help="单 worker 模式下每本之间 sleep 秒数")
|
||||
parser.add_argument("--workers", type=int, default=1, help="并行下载词书的 worker 数")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.catalog.exists():
|
||||
print(f"词书目录不存在:{args.catalog},请先运行 src/fetch-book-catalog.py", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
decrypt_db = load_decrypt_db()
|
||||
books = [row for row in load_books(args.catalog) if row.get("url") != "*"]
|
||||
if args.limit:
|
||||
books = books[: args.limit]
|
||||
|
||||
total = len(books)
|
||||
results: list[dict] = []
|
||||
done = 0
|
||||
workers = max(1, args.workers)
|
||||
|
||||
if workers == 1:
|
||||
for book in books:
|
||||
result = process_one(book, decrypt_db, args.force, args.force_decrypt)
|
||||
results.append(result)
|
||||
done += 1
|
||||
exp = result["export"]
|
||||
print(
|
||||
f"[INFO] [步骤 2/3] 处理进度:current={done}/{total},percent={done / total * 100:.1f}%,"
|
||||
f"code={result['code']},download={result['dl_status']},"
|
||||
f"decrypt={exp['status'] if exp else '-'},words={exp['exported_count'] if exp else '-'}",
|
||||
flush=True,
|
||||
)
|
||||
if args.sleep:
|
||||
time.sleep(args.sleep)
|
||||
else:
|
||||
executor = ThreadPoolExecutor(max_workers=workers)
|
||||
futures = [executor.submit(process_one, book, decrypt_db, args.force, args.force_decrypt) for book in books]
|
||||
try:
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
results.append(result)
|
||||
done += 1
|
||||
exp = result["export"]
|
||||
print(
|
||||
f"[INFO] [步骤 2/3] 处理进度:current={done}/{total},percent={done / total * 100:.1f}%,"
|
||||
f"code={result['code']},download={result['dl_status']},"
|
||||
f"decrypt={exp['status'] if exp else '-'},words={exp['exported_count'] if exp else '-'}",
|
||||
flush=True,
|
||||
)
|
||||
except KeyboardInterrupt:
|
||||
for future in futures:
|
||||
future.cancel()
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
print("\n收到 Ctrl+C,已取消未开始的词书任务。", file=sys.stderr, flush=True)
|
||||
os._exit(130)
|
||||
else:
|
||||
executor.shutdown(wait=True)
|
||||
|
||||
summaries = [r["export"] for r in results if r["export"] is not None]
|
||||
|
||||
ok_dl = sum(1 for r in results if r["dl_status"] in {"downloaded", "skip-db-exists"})
|
||||
ok_export = sum(1 for s in summaries if s["status"] == "ok")
|
||||
failed_dl = len(results) - ok_dl
|
||||
failed_export = len(summaries) - ok_export
|
||||
exported_total = sum(int(s["exported_count"] or 0) for s in summaries)
|
||||
|
||||
print(f"[INFO] [步骤 2/3] 处理完成:total={len(results)},success={ok_export},failed={failed_export},words={exported_total}")
|
||||
failures = []
|
||||
for result in results:
|
||||
export = result.get("export") or {}
|
||||
if result["dl_status"] not in {"downloaded", "skip-db-exists"}:
|
||||
failures.append(f"{result['code']}: 下载 {result['dl_status']}")
|
||||
elif export.get("status") != "ok":
|
||||
failures.append(f"{result['code']}: 解密/导出 {export.get('error') or export.get('status')}")
|
||||
for failure in failures[:20]:
|
||||
print(f"失败项:{failure}", file=sys.stderr)
|
||||
if len(failures) > 20:
|
||||
print(f"失败项:另有 {len(failures) - 20} 项未显示", file=sys.stderr)
|
||||
# 单本失败不阻断后续流程;步骤 3 继续处理已成功导出的词书。
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,507 @@
|
||||
#!/usr/bin/env python3
|
||||
"""步骤3:从全量词表去重后,逐个下载所有词书所有单词的 ZPK 并解密为可读 JSONL。
|
||||
|
||||
解密逻辑复用独立工具 decrypt-zpk-aes.py(可单独调用)。
|
||||
|
||||
输入:output/all-wordbook-words.csv(步骤2 产出,含 word、zpk 字段)
|
||||
去重:同一 zpk 名只下载一次,记录首个出现的 word 作为 first_word(ZPK 解密 key = md5(word))。
|
||||
|
||||
CDN:{base}/r/{zpkName}.zpk
|
||||
|
||||
输出(默认统一到 output/):
|
||||
- output/unique-zpk-index.csv 去重后的 zpk 索引(可读)
|
||||
- output/zpk-cache/0/{zpkName}.zpk 加密 ZPK(中间缓存)
|
||||
- output/zpk-jsonl/zpk-decrypted.jsonl 解密后 JSONL(可读,最终产出)
|
||||
- output/zpk-download-report.csv 下载/解密报告
|
||||
|
||||
已解密的 zpk(记录在 zpk-decrypted.jsonl 中)自动跳过;已下载的 zpk 自动跳过下载,仅重新解密。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import csv
|
||||
import hashlib
|
||||
import http.client
|
||||
import importlib.util
|
||||
import json
|
||||
import queue
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT_DIR = ROOT / "output"
|
||||
WORDS_CSV = OUT_DIR / "wordbooks" / "decrypted"
|
||||
ZPK_DIR = OUT_DIR / "zpk" / "original"
|
||||
ZPK_JSONL = OUT_DIR / "zpk" / "decrypted"
|
||||
|
||||
DECRYPTOR_PATH = ROOT / "src" / "decrypt-zpk-aes.py"
|
||||
|
||||
BASE_STATIC_URLS = [
|
||||
"http://static.beingfine.cn",
|
||||
"https://hwstatic.beingfine.cn",
|
||||
"https://ccastatic.beingfine.cn",
|
||||
]
|
||||
REFERER = "https://app.bbdc.cn"
|
||||
USER_AGENT = "LangEasyLexis/5.11.1"
|
||||
REQUEST_TIMEOUT = 8.0
|
||||
STOP_EVENT = threading.Event()
|
||||
|
||||
UNIQUE_FIELDS = (
|
||||
"zpk",
|
||||
"zpk_bid",
|
||||
"zpk_word_id",
|
||||
"zpk_version",
|
||||
"zpk_updated_unix",
|
||||
"zpk_updated_utc",
|
||||
"first_word",
|
||||
"book_occurrences",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CdnEndpoint:
|
||||
base_url: str
|
||||
scheme: str
|
||||
host: str
|
||||
port: int | None
|
||||
|
||||
@classmethod
|
||||
def parse(cls, base_url: str) -> "CdnEndpoint":
|
||||
parsed = urlparse(base_url)
|
||||
return cls(base_url=base_url, scheme=parsed.scheme, host=parsed.hostname or "", port=parsed.port)
|
||||
|
||||
|
||||
class CdnClient:
|
||||
def __init__(self, endpoints: list[CdnEndpoint], timeout: float) -> None:
|
||||
self.endpoints = endpoints
|
||||
self.timeout = timeout
|
||||
self.connections: dict[str, http.client.HTTPConnection] = {}
|
||||
|
||||
def close(self) -> None:
|
||||
for conn in self.connections.values():
|
||||
conn.close()
|
||||
self.connections.clear()
|
||||
|
||||
def _conn(self, endpoint: CdnEndpoint) -> http.client.HTTPConnection:
|
||||
key = endpoint.base_url
|
||||
conn = self.connections.get(key)
|
||||
if conn is not None:
|
||||
return conn
|
||||
conn_cls = http.client.HTTPSConnection if endpoint.scheme == "https" else http.client.HTTPConnection
|
||||
conn = conn_cls(endpoint.host, endpoint.port, timeout=self.timeout)
|
||||
self.connections[key] = conn
|
||||
return conn
|
||||
|
||||
def get(self, endpoint: CdnEndpoint, name: str) -> tuple[str, http.client.HTTPResponse]:
|
||||
url_path = f"/r/{name}.zpk"
|
||||
headers = {"Connection": "keep-alive", "Referer": REFERER, "User-Agent": USER_AGENT}
|
||||
conn = self._conn(endpoint)
|
||||
try:
|
||||
conn.request("GET", url_path, headers=headers)
|
||||
return f"{endpoint.base_url}{url_path}", conn.getresponse()
|
||||
except (http.client.HTTPException, OSError):
|
||||
conn.close()
|
||||
self.connections.pop(endpoint.base_url, None)
|
||||
raise
|
||||
|
||||
def drop(self, endpoint: CdnEndpoint) -> None:
|
||||
conn = self.connections.pop(endpoint.base_url, None)
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
|
||||
|
||||
class JsonlSink:
|
||||
"""线程安全地为每个 ZPK 写一个独立 JSON 文件。"""
|
||||
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.lock = threading.Lock()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.dir = path
|
||||
self.dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def write(self, name: str, line: str) -> None:
|
||||
with self.lock:
|
||||
target = self.dir / f"{name}.json"
|
||||
target.write_text(line + "\n", encoding="utf-8")
|
||||
|
||||
def close(self) -> None:
|
||||
return
|
||||
|
||||
|
||||
def load_decrypt_zpk():
|
||||
"""加载独立解密工具 decrypt-zpk-aes.py 的解密函数。"""
|
||||
spec = importlib.util.spec_from_file_location("decrypt_zpk_aes", DECRYPTOR_PATH)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError(f"Cannot load decryptor: {DECRYPTOR_PATH}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module.decrypt_zpk_bytes, module.flatten
|
||||
|
||||
|
||||
def build_endpoints(cdn_mode: str, worker_index: int = 0) -> list[CdnEndpoint]:
|
||||
base_urls = BASE_STATIC_URLS
|
||||
if cdn_mode == "spread" and base_urls:
|
||||
offset = worker_index % len(base_urls)
|
||||
base_urls = base_urls[offset:] + base_urls[:offset]
|
||||
return [CdnEndpoint.parse(base) for base in base_urls]
|
||||
|
||||
|
||||
def parse_zpk(zpk: str) -> dict[str, str]:
|
||||
parts = (zpk or "").split("_")
|
||||
if len(parts) != 4 or not all(part.isdigit() for part in parts):
|
||||
return {
|
||||
"zpk_bid": "",
|
||||
"zpk_word_id": "",
|
||||
"zpk_version": "",
|
||||
"zpk_updated_unix": "",
|
||||
"zpk_updated_utc": "",
|
||||
}
|
||||
|
||||
updated = int(parts[3])
|
||||
try:
|
||||
updated_utc = datetime.fromtimestamp(updated, tz=timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
except (OverflowError, OSError, ValueError):
|
||||
updated_utc = ""
|
||||
|
||||
return {
|
||||
"zpk_bid": parts[0],
|
||||
"zpk_word_id": parts[1],
|
||||
"zpk_version": parts[2],
|
||||
"zpk_updated_unix": parts[3],
|
||||
"zpk_updated_utc": updated_utc,
|
||||
}
|
||||
|
||||
|
||||
def build_unique_index(words_path: Path) -> list[dict[str, str]]:
|
||||
"""从全量词表在内存中去重 ZPK 名,不生成索引文件。"""
|
||||
unique: dict[str, dict[str, str | int]] = {}
|
||||
paths = [words_path] if words_path.is_file() else sorted(words_path.glob("*.csv"))
|
||||
for path in paths:
|
||||
with path.open("r", encoding="utf-8-sig", newline="") as fh:
|
||||
reader = csv.DictReader(fh)
|
||||
for row in reader:
|
||||
zpk = row.get("zpk", "")
|
||||
if not zpk:
|
||||
continue
|
||||
if zpk not in unique:
|
||||
unique[zpk] = {
|
||||
"zpk": zpk,
|
||||
**parse_zpk(zpk),
|
||||
"first_word": row.get("word", ""),
|
||||
"book_occurrences": 0,
|
||||
}
|
||||
unique[zpk]["book_occurrences"] = int(unique[zpk]["book_occurrences"]) + 1
|
||||
|
||||
return list(unique.values())
|
||||
|
||||
|
||||
def load_already_decrypted(path: Path) -> set[str]:
|
||||
"""读取现有 JSONL 中已解密的 zpk 名集合,用于断点续传跳过。"""
|
||||
if not path.exists():
|
||||
return set()
|
||||
if path.is_dir():
|
||||
return {p.stem for p in path.glob("*.json")}
|
||||
seen = set()
|
||||
with path.open("r", encoding="utf-8", newline="") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
zpk = obj.get("zpk")
|
||||
if zpk:
|
||||
seen.add(zpk)
|
||||
return seen
|
||||
|
||||
|
||||
def read_and_save_response(resp: http.client.HTTPResponse, dest: Path) -> int:
|
||||
tmp = dest.with_name(f"{dest.name}.{threading.get_ident()}.tmp")
|
||||
h = hashlib.md5()
|
||||
size = 0
|
||||
content_md5 = resp.getheader("Content-MD5")
|
||||
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tmp.open("wb") as out:
|
||||
while not STOP_EVENT.is_set():
|
||||
chunk = resp.read(256 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
out.write(chunk)
|
||||
h.update(chunk)
|
||||
size += len(chunk)
|
||||
|
||||
if STOP_EVENT.is_set():
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise InterruptedError
|
||||
|
||||
if content_md5 and base64.b64encode(h.digest()).decode("ascii") != content_md5:
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise ValueError("md5-mismatch")
|
||||
|
||||
tmp.replace(dest)
|
||||
return size
|
||||
|
||||
|
||||
def ensure_downloaded(name: str, zpk_dir: Path, client: CdnClient) -> dict:
|
||||
"""确保 zpk 文件已下载,返回 (status, url, bytes)。"""
|
||||
dest = zpk_dir / f"{name}.zpk"
|
||||
if dest.exists() and dest.stat().st_size > 0:
|
||||
return {"status": "skip-download", "url": "", "bytes": dest.stat().st_size}
|
||||
|
||||
last_error = ""
|
||||
for endpoint in client.endpoints:
|
||||
if STOP_EVENT.is_set():
|
||||
return {"status": "cancelled", "url": "", "bytes": 0}
|
||||
|
||||
try:
|
||||
url, resp = client.get(endpoint, name)
|
||||
try:
|
||||
if resp.status != 200:
|
||||
last_error = f"http-{resp.status}"
|
||||
resp.read()
|
||||
if resp.status >= 500:
|
||||
client.drop(endpoint)
|
||||
continue
|
||||
size = read_and_save_response(resp, dest)
|
||||
return {"status": "downloaded", "url": url, "bytes": size}
|
||||
finally:
|
||||
resp.close()
|
||||
except InterruptedError:
|
||||
return {"status": "cancelled", "url": "", "bytes": 0}
|
||||
except ValueError as e:
|
||||
last_error = str(e)
|
||||
client.drop(endpoint)
|
||||
except (http.client.HTTPException, OSError, TimeoutError) as e:
|
||||
last_error = type(e).__name__
|
||||
client.drop(endpoint)
|
||||
|
||||
return {"status": f"failed:{last_error}", "url": "", "bytes": 0}
|
||||
|
||||
|
||||
def process_one(
|
||||
row: dict,
|
||||
zpk_dir: Path,
|
||||
client: CdnClient,
|
||||
sink: JsonlSink,
|
||||
decrypt_zpk_bytes,
|
||||
flatten,
|
||||
include_raw: bool = True,
|
||||
) -> dict:
|
||||
"""下载 + 解密单个 zpk,解密结果写入 JSONL。"""
|
||||
name = row["zpk"]
|
||||
word = row.get("first_word", "")
|
||||
|
||||
dl = ensure_downloaded(name, zpk_dir, client)
|
||||
if dl["status"].startswith("failed:"):
|
||||
return {"name": name, "status": dl["status"], "bytes": dl["bytes"], "url": dl["url"]}
|
||||
if dl["status"] == "cancelled":
|
||||
return {"name": name, "status": "cancelled", "bytes": 0, "url": ""}
|
||||
|
||||
# 下载成功或已存在,开始解密
|
||||
path = zpk_dir / f"{name}.zpk"
|
||||
try:
|
||||
obj = decrypt_zpk_bytes(path.read_bytes(), word)
|
||||
# 解密输出必须保留 ZPK 内层 JSON 的全部字段;flatten 仅用于旧版兼容。
|
||||
rec = dict(obj)
|
||||
rec.update(
|
||||
{
|
||||
"zpk": name,
|
||||
"zpk_path": str(path),
|
||||
"index_word": word,
|
||||
"book_occurrences": row.get("book_occurrences", ""),
|
||||
}
|
||||
)
|
||||
if include_raw:
|
||||
rec["raw"] = obj
|
||||
sink.write(name, json.dumps(rec, ensure_ascii=False, separators=(",", ":")))
|
||||
return {"name": name, "status": "decrypted", "bytes": dl["bytes"], "url": dl["url"]}
|
||||
except Exception as exc:
|
||||
return {"name": name, "status": f"failed:decrypt-{type(exc).__name__}", "bytes": dl["bytes"], "url": dl["url"]}
|
||||
|
||||
|
||||
def progress_line(done: int, total: int, result: dict, started_at: float, bytes_done: int) -> str:
|
||||
elapsed = max(0.001, time.monotonic() - started_at)
|
||||
files_per_sec = done / elapsed
|
||||
mib_per_sec = bytes_done / 1024 / 1024 / elapsed
|
||||
return (
|
||||
f"[INFO] [步骤 3/3] 处理进度:current={done}/{total},percent={done / total * 100:.1f}%,"
|
||||
f"name={result['name']},status={result['status']},rate={files_per_sec:.1f}/s,speed={mib_per_sec:.2f}MiB/s"
|
||||
)
|
||||
|
||||
|
||||
def run_parallel(
|
||||
rows: list[dict],
|
||||
zpk_dir: Path,
|
||||
sink: JsonlSink,
|
||||
workers: int,
|
||||
progress_every: int,
|
||||
cdn_mode: str,
|
||||
decrypt_zpk_bytes,
|
||||
flatten,
|
||||
include_raw: bool,
|
||||
) -> list[dict]:
|
||||
work: queue.Queue[dict | None] = queue.Queue(maxsize=max(1, workers * 4))
|
||||
results: list[dict] = []
|
||||
results_lock = threading.Lock()
|
||||
started_at = time.monotonic()
|
||||
done = 0
|
||||
bytes_done = 0
|
||||
total = len(rows)
|
||||
|
||||
def record(result: dict) -> None:
|
||||
nonlocal done, bytes_done
|
||||
with results_lock:
|
||||
results.append(result)
|
||||
done += 1
|
||||
if result["status"] in {"decrypted"}:
|
||||
bytes_done += int(result["bytes"])
|
||||
if done % progress_every == 0 or done == total or STOP_EVENT.is_set():
|
||||
print(progress_line(done, total, result, started_at, bytes_done), flush=True)
|
||||
|
||||
def worker(worker_index: int) -> None:
|
||||
client = CdnClient(build_endpoints(cdn_mode, worker_index), REQUEST_TIMEOUT)
|
||||
try:
|
||||
while not STOP_EVENT.is_set():
|
||||
try:
|
||||
row = work.get(timeout=0.25)
|
||||
except queue.Empty:
|
||||
continue
|
||||
try:
|
||||
if row is None:
|
||||
return
|
||||
record(process_one(row, zpk_dir, client, sink, decrypt_zpk_bytes, flatten, include_raw))
|
||||
finally:
|
||||
work.task_done()
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(workers)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
|
||||
try:
|
||||
for row in rows:
|
||||
while not STOP_EVENT.is_set():
|
||||
try:
|
||||
work.put(row, timeout=0.25)
|
||||
break
|
||||
except queue.Full:
|
||||
continue
|
||||
if STOP_EVENT.is_set():
|
||||
break
|
||||
for _ in threads:
|
||||
while not STOP_EVENT.is_set():
|
||||
try:
|
||||
work.put(None, timeout=0.25)
|
||||
break
|
||||
except queue.Full:
|
||||
continue
|
||||
work.join()
|
||||
except KeyboardInterrupt:
|
||||
STOP_EVENT.set()
|
||||
print(f"\n[中断] ZPK 已处理 {done}/{total},已保留已完成结果。", flush=True)
|
||||
finally:
|
||||
for thread in threads:
|
||||
thread.join(timeout=REQUEST_TIMEOUT + 1)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def write_report(rows: list[dict], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", newline="", encoding="utf-8-sig") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=["name", "status", "bytes", "url"])
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
global REQUEST_TIMEOUT
|
||||
parser = argparse.ArgumentParser(description="从全量词表去重下载所有 ZPK 并解密为 JSONL。")
|
||||
parser.add_argument("--words", type=Path, default=WORDS_CSV, help="词书解密目录或 CSV 文件")
|
||||
parser.add_argument("--out-dir", type=Path, default=ZPK_DIR)
|
||||
parser.add_argument("--jsonl", type=Path, default=ZPK_JSONL, help="解密输出目录,每个 ZPK 一个 JSON")
|
||||
parser.add_argument("--workers", type=int, default=64)
|
||||
parser.add_argument("--timeout", type=float, default=8, help="单个 CDN 请求超时秒数;Ctrl+C 最多等待活跃请求到此超时")
|
||||
parser.add_argument("--progress-every", type=int, default=500)
|
||||
parser.add_argument(
|
||||
"--cdn-mode",
|
||||
choices=("spread", "primary"),
|
||||
default="primary",
|
||||
help="primary 会始终优先第一个 CDN;spread 会把 worker 分散到 3 个 CDN",
|
||||
)
|
||||
parser.add_argument("--limit", type=int, default=0, help="最多处理 N 个(冒烟测试)")
|
||||
parser.add_argument("--force", action="store_true", help="已解密也重新下载解密")
|
||||
parser.add_argument("--raw", action="store_true", default=True, help="保留完整原始 JSON 对象(默认开启)")
|
||||
args = parser.parse_args()
|
||||
|
||||
REQUEST_TIMEOUT = max(1.0, args.timeout)
|
||||
zpk_dir = args.out_dir
|
||||
|
||||
if not args.words.exists():
|
||||
print(f"词表不存在:{args.words},请先运行 src/download-wordbooks.py", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
decrypt_zpk_bytes, flatten = load_decrypt_zpk()
|
||||
index_rows = build_unique_index(args.words)
|
||||
print(f"[INFO] [步骤 3/3] 索引准备完成:total={len(index_rows)}", flush=True)
|
||||
|
||||
already_decrypted = set() if args.force else load_already_decrypted(args.jsonl)
|
||||
rows = [r for r in index_rows if r["zpk"] not in already_decrypted]
|
||||
print(f"[INFO] [步骤 3/3] 断点状态:completed={len(already_decrypted)},pending={len(rows)}", flush=True)
|
||||
|
||||
if args.limit:
|
||||
rows = rows[: args.limit]
|
||||
|
||||
total = len(rows)
|
||||
print(f"[INFO] [步骤 3/3] 开始下载并解密:total={total}", flush=True)
|
||||
|
||||
sink = JsonlSink(args.jsonl)
|
||||
workers = max(1, args.workers)
|
||||
try:
|
||||
if workers == 1:
|
||||
endpoint_client = CdnClient(build_endpoints(args.cdn_mode), REQUEST_TIMEOUT)
|
||||
results = []
|
||||
started_at = time.monotonic()
|
||||
bytes_done = 0
|
||||
try:
|
||||
for done, row in enumerate(rows, 1):
|
||||
r = process_one(row, zpk_dir, endpoint_client, sink, decrypt_zpk_bytes, flatten, args.raw)
|
||||
results.append(r)
|
||||
if r["status"] == "decrypted":
|
||||
bytes_done += int(r["bytes"])
|
||||
print(progress_line(done, total, r, started_at, bytes_done), flush=True)
|
||||
except KeyboardInterrupt:
|
||||
STOP_EVENT.set()
|
||||
print(f"\n[中断] ZPK 已处理 {len(results)}/{total},已保留已完成结果。", flush=True)
|
||||
finally:
|
||||
endpoint_client.close()
|
||||
else:
|
||||
results = run_parallel(
|
||||
rows, zpk_dir, sink, workers, max(1, args.progress_every), args.cdn_mode,
|
||||
decrypt_zpk_bytes, flatten, args.raw,
|
||||
)
|
||||
finally:
|
||||
sink.close()
|
||||
|
||||
ok = sum(1 for r in results if r["status"] == "decrypted")
|
||||
failed = len(results) - ok
|
||||
cancelled = sum(1 for r in results if r["status"] == "cancelled")
|
||||
print(f"[INFO] [步骤 3/3] 处理完成:success={ok},failed={failed},skipped={cancelled}", flush=True)
|
||||
if STOP_EVENT.is_set():
|
||||
return 130
|
||||
return 0 if failed == 0 else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export synonym data from bbdb and derivative/family data from wordroot."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import argparse
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / "output" / "synonym-derivative-export"
|
||||
PACKAGE = "cn.com.langeasy.LangEasyLexis"
|
||||
KEYS = {
|
||||
"bbdb": "North_London_bbdb_Forever_20230710",
|
||||
"wordroot": "North_wordroot_London_Forever_20220601",
|
||||
}
|
||||
|
||||
|
||||
def connect(path: Path, key: str):
|
||||
try:
|
||||
import sqlcipher3.dbapi2 as dbapi
|
||||
except ImportError as exc:
|
||||
raise SystemExit("请先安装 sqlcipher3:python -m pip install sqlcipher3") from exc
|
||||
conn = dbapi.connect(str(path))
|
||||
conn.execute(f"PRAGMA key='{key}'")
|
||||
conn.execute("SELECT count(*) FROM sqlite_master").fetchone()
|
||||
return conn
|
||||
|
||||
|
||||
def adb_export(serial: str | None, remote_name: str, local_path: Path) -> None:
|
||||
target = ["adb"] + (["-s", serial] if serial else [])
|
||||
target += ["exec-out", "cat", f"/data/data/{PACKAGE}/databases/{remote_name}"]
|
||||
with local_path.open("wb") as fh:
|
||||
result = subprocess.run(target, stdout=fh, stderr=subprocess.PIPE)
|
||||
if result.returncode != 0:
|
||||
target = ["adb"] + (["-s", serial] if serial else [])
|
||||
target += ["exec-out", "run-as", PACKAGE, "cat", f"databases/{remote_name}"]
|
||||
with local_path.open("wb") as fh:
|
||||
result = subprocess.run(target, stdout=fh, stderr=subprocess.PIPE)
|
||||
if result.returncode != 0 or local_path.stat().st_size == 0:
|
||||
detail = result.stderr.decode(errors="replace").strip()
|
||||
raise SystemExit(f"ADB 导出失败: {remote_name}: {detail}")
|
||||
|
||||
|
||||
def dump_query(conn, sql: str, name: str) -> int:
|
||||
cur = conn.execute(sql)
|
||||
path = OUT / name
|
||||
with path.open("w", newline="", encoding="utf-8-sig") as fh:
|
||||
writer = csv.writer(fh)
|
||||
writer.writerow([d[0] for d in cur.description])
|
||||
rows = cur.fetchall()
|
||||
writer.writerows(rows)
|
||||
return len(rows)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--serial", help="adb 设备序列号;默认使用当前设备")
|
||||
args = parser.parse_args()
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix="lexis-adb-export-") as tmp:
|
||||
bb_path = Path(tmp) / "bbdb.db"
|
||||
wr_path = Path(tmp) / "wordroot.db"
|
||||
adb_export(args.serial, "bbdb.db", bb_path)
|
||||
adb_export(args.serial, "wordroot.db", wr_path)
|
||||
bb = connect(bb_path, KEYS["bbdb"])
|
||||
wr = connect(wr_path, KEYS["wordroot"])
|
||||
try:
|
||||
counts = {}
|
||||
counts["synonym_words.csv"] = dump_query(bb, "SELECT id, word FROM t_synonym_words ORDER BY id", "synonym_words.csv")
|
||||
counts["synonym_data.csv"] = dump_query(bb, "SELECT * FROM t_synonym_datas ORDER BY word_id, pos, id", "synonym_data.csv")
|
||||
counts["synonyms-joined.csv"] = dump_query(bb, """SELECT w.word, d.pos, d.interpret, d.zk, d.gk, d.cet4, d.cet6, d.ky, d.ielts, d.zsb, d.other, d.toefl FROM t_synonym_words w JOIN t_synonym_datas d ON d.word_id=w.id ORDER BY w.word, d.pos, d.id""", "synonyms-joined.csv")
|
||||
counts["derivative-family.csv"] = dump_query(wr, """SELECT id, word, top_id, grade, interpret, prefix, suffix, group1, group2, group3, json FROM word_family ORDER BY top_id, id""", "derivative-family.csv")
|
||||
counts["derivative-mini-groups.csv"] = dump_query(wr, "SELECT id, word_ids FROM word_family_mini_group ORDER BY id", "derivative-mini-groups.csv")
|
||||
finally:
|
||||
bb.close()
|
||||
wr.close()
|
||||
(OUT / "synonym-derivative-summary.txt").write_text(
|
||||
"近义词与派生串记导出\n"
|
||||
f"近义词单词: {counts['synonym_words.csv']}\n"
|
||||
f"近义词释义记录: {counts['synonym_data.csv']}\n"
|
||||
f"近义词关联记录: {counts['synonyms-joined.csv']}\n"
|
||||
f"派生词族记录: {counts['derivative-family.csv']}\n"
|
||||
f"派生小组记录: {counts['derivative-mini-groups.csv']}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print("已完成近义词和派生串记导出")
|
||||
for name, count in counts.items():
|
||||
print(f"{name}: {count}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export the encrypted word-root database into CSV files.
|
||||
|
||||
The script supports pysqlcipher3/sqlcipher3 and the sqlcipher CLI. The
|
||||
database is copied to a temporary file before opening so the source remains
|
||||
unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_DB = ROOT / "assets" / "wordroot.db"
|
||||
DEFAULT_OUT = ROOT / "output" / "wordroot-export"
|
||||
DB_KEY = "North_wordroot_London_Forever_20220601"
|
||||
|
||||
LOCAL_RUNTIME = ROOT / ".runtime" / "sqlcipher3"
|
||||
if LOCAL_RUNTIME.exists():
|
||||
sys.path.insert(0, str(LOCAL_RUNTIME))
|
||||
|
||||
|
||||
def open_python_cipher(db: Path):
|
||||
for module_name in ("pysqlcipher3.dbapi2", "sqlcipher3.dbapi2"):
|
||||
try:
|
||||
module = __import__(module_name, fromlist=["connect"])
|
||||
except ImportError:
|
||||
continue
|
||||
conn = module.connect(str(db))
|
||||
conn.execute(f"PRAGMA key='{DB_KEY}'")
|
||||
conn.execute("PRAGMA cipher_compatibility=4")
|
||||
conn.execute("SELECT count(*) FROM sqlite_master").fetchone()
|
||||
return conn
|
||||
return None
|
||||
|
||||
|
||||
def export_with_cli(db: Path, out_dir: Path) -> None:
|
||||
sqlcipher = shutil.which("sqlcipher")
|
||||
if not sqlcipher:
|
||||
raise RuntimeError(
|
||||
"未找到 SQLCipher。请安装 pysqlcipher3/sqlcipher3,或将 sqlcipher 加入 PATH。"
|
||||
)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
script = (
|
||||
f"PRAGMA key='{DB_KEY}';\n"
|
||||
"PRAGMA cipher_compatibility=4;\n"
|
||||
".mode csv\n"
|
||||
".headers on\n"
|
||||
)
|
||||
tables = subprocess.run(
|
||||
[sqlcipher, str(db)], input=script + "SELECT name FROM sqlite_master WHERE type='table';\n",
|
||||
text=True, capture_output=True, check=True,
|
||||
).stdout.splitlines()
|
||||
for table in (line.strip() for line in tables):
|
||||
if not table or table == "name" or table.startswith("---"):
|
||||
continue
|
||||
safe = "".join(c if c.isalnum() or c in "-_" else "_" for c in table)
|
||||
query = script + f'.output "{out_dir / (safe + ".csv")}"\nSELECT * FROM "{table}";\n.output stdout\n'
|
||||
subprocess.run([sqlcipher, str(db)], input=query, text=True, check=True)
|
||||
|
||||
|
||||
def export_with_python(conn, out_dir: Path) -> list[str]:
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
names = [row[0] for row in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
|
||||
)]
|
||||
for table in names:
|
||||
safe = "".join(c if c.isalnum() or c in "-_" else "_" for c in table)
|
||||
cur = conn.execute(f'SELECT * FROM "{table}"')
|
||||
with (out_dir / f"{safe}.csv").open("w", newline="", encoding="utf-8-sig") as fh:
|
||||
writer = csv.writer(fh)
|
||||
writer.writerow([col[0] for col in cur.description])
|
||||
writer.writerows(cur)
|
||||
return names
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--db", type=Path, default=DEFAULT_DB)
|
||||
parser.add_argument("--output", type=Path, default=DEFAULT_OUT)
|
||||
args = parser.parse_args()
|
||||
db = args.db.resolve()
|
||||
if not db.exists():
|
||||
raise SystemExit(f"数据库不存在: {db}")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="lexis-wordroot-") as tmp:
|
||||
local_db = Path(tmp) / "wordroot.db"
|
||||
shutil.copy2(db, local_db)
|
||||
conn = open_python_cipher(local_db)
|
||||
if conn is not None:
|
||||
try:
|
||||
tables = export_with_python(conn, args.output.resolve())
|
||||
finally:
|
||||
conn.close()
|
||||
else:
|
||||
export_with_cli(local_db, args.output.resolve())
|
||||
tables = [p.stem for p in args.output.resolve().glob("*.csv")]
|
||||
|
||||
manifest = args.output.resolve() / "README.txt"
|
||||
manifest.write_text(
|
||||
"wordroot.db 导出结果\n"
|
||||
f"源数据库: {db}\n"
|
||||
f"表数量: {len(tables)}\n"
|
||||
"CSV 使用 UTF-8-BOM 编码。\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"已导出 {len(tables)} 张表到: {args.output.resolve()}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,102 @@
|
||||
"""从服务端词书列表接口拉取词书目录,替代从设备偏好文件提取。
|
||||
|
||||
接口:GET https://api.beingfine.cn/v3/3/bb/wordbooks/for-guest/{timestamp}
|
||||
该接口是游客态全量词书列表,无需登录,响应 data_body 为明文 JSON。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
|
||||
API_PATH = "https://api.beingfine.cn/v3/3/bb/wordbooks/for-guest"
|
||||
APP_ID = "600000001"
|
||||
APP_VERSION = "5.11.1"
|
||||
CHANNEL = "isCool"
|
||||
TOKEN = "tk85bf4d7d70a4d5028187ba6dfb3649"
|
||||
REFERER = "https://app.bbdc.cn"
|
||||
USER_AGENT = "LangEasyLexis/5.11.1"
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / "output" / "book-catalog.json"
|
||||
FIELDS = ("id", "code", "name", "count", "exam", "url", "version", "bookVersion", "purchased", "forSale")
|
||||
|
||||
|
||||
def compact_book(book):
|
||||
return {key: book.get(key) for key in FIELDS if key in book}
|
||||
|
||||
|
||||
def fetch_catalog(timeout=20):
|
||||
"""拉取游客态词书列表,返回 (books, raw_body)。"""
|
||||
ts = int(time.time() * 1000)
|
||||
url = f"{API_PATH}/{ts}"
|
||||
params = {
|
||||
"type": "3/bb/wordbooks/for-guest",
|
||||
"app_id": APP_ID,
|
||||
"udid": "",
|
||||
"umid": "",
|
||||
"version": APP_VERSION,
|
||||
"user_id": "0",
|
||||
"channel": CHANNEL,
|
||||
"timestamp": str(ts),
|
||||
"timezone": "480",
|
||||
"pad": "0",
|
||||
"token": TOKEN,
|
||||
}
|
||||
qs = "&".join(f"{k}={urllib.parse.quote(str(v), safe='')}" for k, v in params.items())
|
||||
req = urllib.request.Request(
|
||||
url + "?" + qs,
|
||||
headers={"Referer": REFERER, "User-Agent": USER_AGENT},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read().decode("utf-8")
|
||||
|
||||
outer = json.loads(raw)
|
||||
if outer.get("result_code") != 200:
|
||||
raise RuntimeError(f"result_code={outer.get('result_code')} body={raw[:200]}")
|
||||
data_body = outer.get("data_body")
|
||||
if isinstance(data_body, str):
|
||||
data_body = json.loads(data_body)
|
||||
if not isinstance(data_body, dict):
|
||||
raise RuntimeError(f"data_body 缺失或格式异常: {raw[:200]}")
|
||||
books = data_body.get("books") or []
|
||||
return books, data_body
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="从服务端拉取 Lexis 词书目录")
|
||||
parser.add_argument("--output", default=OUT, help="输出词书目录 JSON 路径")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("[INFO] [步骤 1/3] 开始获取词书目录", flush=True)
|
||||
started = time.monotonic()
|
||||
books, full = fetch_catalog()
|
||||
catalog = sorted(
|
||||
books,
|
||||
key=lambda item: (str(item.get("exam", "")), str(item.get("code", ""))),
|
||||
)
|
||||
|
||||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
json.dump(catalog, f, ensure_ascii=False, indent=2)
|
||||
f.write("\n")
|
||||
|
||||
categories = {}
|
||||
for item in catalog:
|
||||
category = str(item.get("exam") or "未分类")
|
||||
categories[category] = categories.get(category, 0) + 1
|
||||
elapsed = time.monotonic() - started
|
||||
category_text = ", ".join(f"{name}: {count}" for name, count in sorted(categories.items()))
|
||||
print(f"[INFO] [步骤 1/3] 词书目录获取完成:数量={len(catalog)},耗时={elapsed:.1f}s")
|
||||
print(f"[INFO] [步骤 1/3] 分类统计:{category_text}")
|
||||
print(f"[INFO] [步骤 1/3] 输出文件:{args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user