diff --git a/importer/src/archive/build-book-download-manifest.py b/importer/src/archive/build-book-download-manifest.py new file mode 100644 index 0000000..83e2588 --- /dev/null +++ b/importer/src/archive/build-book-download-manifest.py @@ -0,0 +1,103 @@ +"""Build a sanitized remote word-book manifest from the runtime catalog.""" + +from __future__ import annotations + +import argparse +import csv +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_CATALOG = ROOT / "runtime-dump" / "book-catalog.json" +DEFAULT_DB_DIR = ROOT / "runtime-dump" / "databases" +DEFAULT_JSONL = ROOT / "runtime-dump" / "book-download-manifest.jsonl" +DEFAULT_CSV = ROOT / "runtime-dump" / "book-download-manifest.csv" +DEFAULT_HOST = "https://api.beingfine.cn" + +FIELDS = ( + "id", + "code", + "name", + "count", + "exam", + "url", + "remote_url", + "version", + "bookVersion", + "purchased", + "forSale", + "is_remote", + "local_db", + "local_size", +) + + +def load_books(path: Path) -> list[dict]: + with path.open("r", encoding="utf-8") as fh: + data = json.load(fh) + if isinstance(data, dict) and isinstance(data.get("books"), list): + return data["books"] + if isinstance(data, list): + return data + raise ValueError(f"Unsupported catalog shape: {path}") + + +def build_rows(books: list[dict], db_dir: Path, host: str) -> list[dict]: + rows = [] + for book in books: + url = str(book.get("url") or "") + filename = Path(url).name + local_path = db_dir / filename if filename else None + exists = bool(local_path and local_path.exists()) + row = {field: "" for field in FIELDS} + for field in ("id", "code", "name", "count", "exam", "url", "version", "bookVersion", "purchased", "forSale"): + row[field] = book.get(field, "") + row["remote_url"] = host.rstrip("/") + url if url.startswith("/") else url + row["is_remote"] = bool(url.startswith("/")) + row["local_db"] = str(local_path) if exists else "" + row["local_size"] = local_path.stat().st_size if exists and local_path else "" + rows.append(row) + return rows + + +def write_jsonl(rows: list[dict], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as fh: + for row in rows: + fh.write(json.dumps(row, ensure_ascii=False) + "\n") + + +def write_csv(rows: list[dict], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8-sig", newline="") as fh: + writer = csv.DictWriter(fh, fieldnames=FIELDS) + writer.writeheader() + writer.writerows(rows) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) + parser.add_argument("--db-dir", type=Path, default=DEFAULT_DB_DIR) + parser.add_argument("--host", default=DEFAULT_HOST) + parser.add_argument("--jsonl", type=Path, default=DEFAULT_JSONL) + parser.add_argument("--csv", type=Path, default=DEFAULT_CSV) + args = parser.parse_args() + + books = load_books(args.catalog) + rows = build_rows(books, args.db_dir, args.host) + write_jsonl(rows, args.jsonl) + write_csv(rows, args.csv) + + local_count = sum(1 for row in rows if row["local_db"]) + remote_count = sum(1 for row in rows if row["is_remote"]) + missing_count = sum(1 for row in rows if row["is_remote"] and not row["local_db"]) + print(f"books={len(rows)} remote={remote_count} local={local_count} missing_remote={missing_count}") + print(f"jsonl={args.jsonl}") + print(f"csv={args.csv}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/importer/src/archive/derive-lexis-db-keys.py b/importer/src/archive/derive-lexis-db-keys.py new file mode 100644 index 0000000..d456cd5 --- /dev/null +++ b/importer/src/archive/derive-lexis-db-keys.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +from pathlib import Path +import struct +import hashlib + + +LIBCOMMON_X86 = Path("jadx-out/resources/lib/x86/libcommon.so") +PACKAGE = b"cn.com.langeasy.LangEasyLexis" +PACKAGE_SHA256 = hashlib.sha256(PACKAGE).hexdigest().encode("ascii") + + +# x86 getDBKey() computes ebx as 0x2ff0 and dec(type) before jump dispatch. +# type=0 falls through to the default branch. +TABLES = { + 0: (0x2FF0 - 0x260B, 0x0E), + 1: (0x2FF0 - 0x26E3, 0x24), + 2: (0x2FF0 - 0x2652, 0x25), + 3: (0x2FF0 - 0x2699, 0x25), + 4: (0x2FF0 - 0x2674, 0x22), + 5: (0x2FF0 - 0x26BF, 0x26), + 6: (0x2FF0 - 0x262D, 0x22), +} + + +def decode(blob: bytes, offset: int, size: int) -> str: + out = bytearray() + for i, b in enumerate(blob[offset : offset + size]): + out.append(b ^ PACKAGE_SHA256[i % len(PACKAGE_SHA256)]) + return out.decode("utf-8") + + +def jump_targets(blob: bytes) -> dict[int, int]: + base = 0x2FF0 + table = base - 0x2820 + targets = {} + for selector in range(1, 7): + rel = struct.unpack_from(" int: + blob = LIBCOMMON_X86.read_bytes() + print("jump_targets:") + for selector, target in jump_targets(blob).items(): + print(f" type={selector}: 0x{target:x}") + print("keys:") + for key_type, (offset, size) in TABLES.items(): + print(f" DBKEY[{key_type}]={decode(blob, offset, size)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/importer/src/archive/download-all-wordbooks.py b/importer/src/archive/download-all-wordbooks.py new file mode 100644 index 0000000..fd084df --- /dev/null +++ b/importer/src/archive/download-all-wordbooks.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +import argparse +import base64 +import csv +import hashlib +import json +import os +import sys +import time +import zipfile +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + + +ROOT = Path(__file__).resolve().parents[1] +CATALOG = ROOT / "runtime-dump" / "book-catalog.json" +ZIP_DIR = ROOT / "runtime-dump" / "remote-zips" +DB_DIR = ROOT / "runtime-dump" / "databases" +REPORT_CSV = ROOT / "runtime-dump" / "wordbook-download-report.csv" + +BASE_STATIC_URLS = [ + "http://static.beingfine.cn", + "https://hwstatic.beingfine.cn", + "https://ccastatic.beingfine.cn", +] +REFERER = "https://app.bbdc.cn" +USER_AGENT = "LangEasyLexis/5.11.1" + + +def load_books(path: Path) -> list[dict]: + with path.open("r", encoding="utf-8") as f: + data = json.load(f) + if isinstance(data, dict) and isinstance(data.get("books"), list): + return data["books"] + if isinstance(data, list): + return data + raise ValueError(f"Unsupported catalog shape: {path}") + + +def book_url(book: dict, base_url: str) -> str: + code = book["code"] + version = str(book.get("version") or 0) + db_name = f"LangEasyLexisV3_{code}.db" + return f"{base_url}/dicts/v3/{code}/{version}/{db_name}.zip" + + +def request_url(url: str): + req = Request( + url, + headers={ + "Referer": REFERER, + "User-Agent": USER_AGENT, + }, + ) + return urlopen(req, timeout=45) + + +def md5_b64(path: Path) -> str: + h = hashlib.md5() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return base64.b64encode(h.digest()).decode("ascii") + + +def download_one(book: dict, force: bool = False) -> dict: + code = book["code"] + if book.get("url") == "*": + return {"code": code, "status": "skip-virtual-book", "url": "", "bytes": 0} + + zip_path = ZIP_DIR / f"LangEasyLexisV3_{code}.db.zip" + db_path = DB_DIR / f"LangEasyLexisV3_{code}.db" + + if db_path.exists() and not force: + return {"code": code, "status": "skip-db-exists", "url": "", "bytes": db_path.stat().st_size} + + last_error = "" + for base in BASE_STATIC_URLS: + url = book_url(book, base) + try: + with request_url(url) as resp: + status = getattr(resp, "status", 200) + if status != 200: + last_error = f"http-{status}" + continue + + ZIP_DIR.mkdir(parents=True, exist_ok=True) + tmp_path = zip_path.with_suffix(zip_path.suffix + ".tmp") + with tmp_path.open("wb") as out: + while True: + chunk = resp.read(1024 * 1024) + if not chunk: + break + out.write(chunk) + + content_md5 = resp.headers.get("Content-MD5") + if content_md5 and md5_b64(tmp_path) != content_md5: + tmp_path.unlink(missing_ok=True) + last_error = "md5-mismatch" + continue + + tmp_path.replace(zip_path) + DB_DIR.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(zip_path) as zf: + names = zf.namelist() + wanted = f"LangEasyLexisV3_{code}.db" + member = wanted if wanted in names else names[0] + zf.extract(member, DB_DIR) + extracted = DB_DIR / member + if extracted != db_path: + extracted.replace(db_path) + + return {"code": code, "status": "downloaded", "url": url, "bytes": db_path.stat().st_size} + except HTTPError as e: + last_error = f"http-{e.code}" + except (URLError, TimeoutError, zipfile.BadZipFile, OSError) as e: + last_error = type(e).__name__ + + return {"code": code, "status": f"failed:{last_error}", "url": "", "bytes": 0} + + +def write_report(rows: list[dict]) -> None: + REPORT_CSV.parent.mkdir(parents=True, exist_ok=True) + with REPORT_CSV.open("w", newline="", encoding="utf-8-sig") as f: + writer = csv.DictWriter(f, fieldnames=["code", "status", "bytes", "url"]) + writer.writeheader() + writer.writerows(rows) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Download all Lexis V3 wordbook DB zips from the app's CDN interface.") + parser.add_argument("--catalog", type=Path, default=CATALOG) + parser.add_argument("--force", action="store_true", help="re-download books even if the extracted DB already exists") + parser.add_argument("--limit", type=int, default=0, help="download at most N books, useful for smoke tests") + parser.add_argument("--sleep", type=float, default=0.15, help="seconds to sleep between downloads in single-worker mode") + parser.add_argument("--workers", type=int, default=1, help="parallel downloads") + parser.add_argument("--include-existing", action="store_true", help="include already cached local DBs in the run") + args = parser.parse_args() + + books = load_books(args.catalog) + if not args.include_existing and not args.force: + books = [b for b in books if not (DB_DIR / f"LangEasyLexisV3_{b['code']}.db").exists()] + if args.limit: + books = books[: args.limit] + + total = len(books) + results = [] + done = 0 + workers = max(1, args.workers) + if workers == 1: + for book in books: + result = download_one(book, force=args.force) + results.append(result) + done += 1 + print(f"[{done}/{total}] {result['code']} {result['status']} {result['bytes']}", flush=True) + if args.sleep: + time.sleep(args.sleep) + else: + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = [executor.submit(download_one, book, args.force) for book in books] + for future in as_completed(futures): + result = future.result() + results.append(result) + done += 1 + print(f"[{done}/{total}] {result['code']} {result['status']} {result['bytes']}", flush=True) + + write_report(results) + ok = sum(1 for r in results if r["status"] in {"downloaded", "skip-db-exists", "skip-virtual-book"}) + failed = len(results) - ok + print(f"done ok={ok} failed={failed} report={REPORT_CSV}", flush=True) + return 0 if failed == 0 else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/importer/src/archive/download-all-zpk.py b/importer/src/archive/download-all-zpk.py new file mode 100644 index 0000000..565dedc --- /dev/null +++ b/importer/src/archive/download-all-zpk.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +"""批量下载所有 zpk 文件。 + +来源:unique-zpk-index.csv 中的唯一 zpk 名(115104 个)。 +CDN:{base}/r/{zpkName}.zpk,Referer 同词书下载。 +落地:runtime-dump/zpk-cache/0/{zpkName}.zpk(与设备目录结构一致)。 +已存在的文件自动跳过,可断点续传。 +""" + +from __future__ import annotations + +import argparse +import base64 +import csv +import hashlib +import http.client +import queue +import sys +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlparse + +ROOT = Path(__file__).resolve().parents[1] +INDEX_CSV = ROOT / "runtime-dump" / "unique-zpk-index.csv" +ZPK_DIR = ROOT / "runtime-dump" / "zpk-cache" / "0" +REPORT_CSV = ROOT / "runtime-dump" / "zpk-download-report.csv" + +BASE_STATIC_URLS = [ + "http://static.beingfine.cn", + "https://hwstatic.beingfine.cn", + "https://ccastatic.beingfine.cn", +] +REFERER = "https://app.bbdc.cn" +USER_AGENT = "LangEasyLexis/5.11.1" +REQUEST_TIMEOUT = 8.0 +STOP_EVENT = threading.Event() + + +@dataclass(frozen=True) +class CdnEndpoint: + base_url: str + scheme: str + host: str + port: int | None + + @classmethod + def parse(cls, base_url: str) -> "CdnEndpoint": + parsed = urlparse(base_url) + return cls(base_url=base_url, scheme=parsed.scheme, host=parsed.hostname or "", port=parsed.port) + + +class CdnClient: + def __init__(self, endpoints: list[CdnEndpoint], timeout: float) -> None: + self.endpoints = endpoints + self.timeout = timeout + self.connections: dict[str, http.client.HTTPConnection] = {} + + def close(self) -> None: + for conn in self.connections.values(): + conn.close() + self.connections.clear() + + def _conn(self, endpoint: CdnEndpoint) -> http.client.HTTPConnection: + key = endpoint.base_url + conn = self.connections.get(key) + if conn is not None: + return conn + conn_cls = http.client.HTTPSConnection if endpoint.scheme == "https" else http.client.HTTPConnection + conn = conn_cls(endpoint.host, endpoint.port, timeout=self.timeout) + self.connections[key] = conn + return conn + + def get(self, endpoint: CdnEndpoint, name: str) -> tuple[str, http.client.HTTPResponse]: + url_path = f"/r/{name}.zpk" + headers = { + "Connection": "keep-alive", + "Referer": REFERER, + "User-Agent": USER_AGENT, + } + conn = self._conn(endpoint) + try: + conn.request("GET", url_path, headers=headers) + return f"{endpoint.base_url}{url_path}", conn.getresponse() + except (http.client.HTTPException, OSError): + conn.close() + self.connections.pop(endpoint.base_url, None) + raise + + def drop(self, endpoint: CdnEndpoint) -> None: + conn = self.connections.pop(endpoint.base_url, None) + if conn is not None: + conn.close() + + +def build_endpoints(cdn_mode: str, worker_index: int = 0) -> list[CdnEndpoint]: + base_urls = BASE_STATIC_URLS + if cdn_mode == "spread" and base_urls: + offset = worker_index % len(base_urls) + base_urls = base_urls[offset:] + base_urls[:offset] + return [CdnEndpoint.parse(base) for base in base_urls] + + +def load_names(path: Path) -> list[str]: + names = [] + seen = set() + with path.open("r", encoding="utf-8-sig", newline="") as f: + for row in csv.DictReader(f): + name = row.get("zpk") + if name and name not in seen: + seen.add(name) + names.append(name) + return names + + +def read_and_save_response(resp: http.client.HTTPResponse, dest: Path) -> int: + tmp = dest.with_name(f"{dest.name}.{threading.get_ident()}.tmp") + h = hashlib.md5() + size = 0 + content_md5 = resp.getheader("Content-MD5") + + dest.parent.mkdir(parents=True, exist_ok=True) + with tmp.open("wb") as out: + while not STOP_EVENT.is_set(): + chunk = resp.read(256 * 1024) + if not chunk: + break + out.write(chunk) + h.update(chunk) + size += len(chunk) + + if STOP_EVENT.is_set(): + tmp.unlink(missing_ok=True) + raise InterruptedError + + if content_md5 and base64.b64encode(h.digest()).decode("ascii") != content_md5: + tmp.unlink(missing_ok=True) + raise ValueError("md5-mismatch") + + tmp.replace(dest) + return size + + +def download_one(name: str, zpk_dir: Path, client: CdnClient) -> dict: + dest = zpk_dir / f"{name}.zpk" + if dest.exists() and dest.stat().st_size > 0: + return {"name": name, "status": "skip-exists", "bytes": dest.stat().st_size, "url": ""} + + last_error = "" + for endpoint in client.endpoints: + if STOP_EVENT.is_set(): + return {"name": name, "status": "cancelled", "bytes": 0, "url": ""} + + try: + url, resp = client.get(endpoint, name) + try: + if resp.status != 200: + last_error = f"http-{resp.status}" + resp.read() + if resp.status >= 500: + client.drop(endpoint) + continue + size = read_and_save_response(resp, dest) + return {"name": name, "status": "downloaded", "bytes": size, "url": url} + finally: + resp.close() + except InterruptedError: + return {"name": name, "status": "cancelled", "bytes": 0, "url": ""} + except ValueError as e: + last_error = str(e) + client.drop(endpoint) + except (http.client.HTTPException, OSError, TimeoutError) as e: + last_error = type(e).__name__ + client.drop(endpoint) + + return {"name": name, "status": f"failed:{last_error}", "bytes": 0, "url": ""} + + +def progress_line(done: int, total: int, result: dict, started_at: float, bytes_done: int) -> str: + elapsed = max(0.001, time.monotonic() - started_at) + files_per_sec = done / elapsed + mib_per_sec = bytes_done / 1024 / 1024 / elapsed + return ( + f"[{done}/{total}] {result['name']} {result['status']} {result['bytes']} " + f"rate={files_per_sec:.1f}/s {mib_per_sec:.2f}MiB/s" + ) + + +def run_parallel(names: list[str], zpk_dir: Path, workers: int, progress_every: int, cdn_mode: str) -> list[dict]: + work: queue.Queue[str | None] = queue.Queue(maxsize=max(1, workers * 4)) + results: list[dict] = [] + results_lock = threading.Lock() + started_at = time.monotonic() + done = 0 + bytes_done = 0 + total = len(names) + + def record(result: dict) -> None: + nonlocal done, bytes_done + with results_lock: + results.append(result) + done += 1 + if result["status"] in {"downloaded", "skip-exists"}: + bytes_done += int(result["bytes"]) + if done % progress_every == 0 or done == total or STOP_EVENT.is_set(): + print(progress_line(done, total, result, started_at, bytes_done), flush=True) + + def worker(worker_index: int) -> None: + client = CdnClient(build_endpoints(cdn_mode, worker_index), REQUEST_TIMEOUT) + try: + while not STOP_EVENT.is_set(): + try: + name = work.get(timeout=0.25) + except queue.Empty: + continue + try: + if name is None: + return + record(download_one(name, zpk_dir, client)) + finally: + work.task_done() + finally: + client.close() + + threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(workers)] + for thread in threads: + thread.start() + + try: + for name in names: + while not STOP_EVENT.is_set(): + try: + work.put(name, timeout=0.25) + break + except queue.Full: + continue + if STOP_EVENT.is_set(): + break + for _ in threads: + work.put(None) + work.join() + except KeyboardInterrupt: + STOP_EVENT.set() + print(f"\ninterrupted done={done}/{total}; writing partial report...", flush=True) + finally: + for thread in threads: + thread.join(timeout=REQUEST_TIMEOUT + 1) + + return results + + +def write_report(rows: list[dict], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="", encoding="utf-8-sig") as f: + writer = csv.DictWriter(f, fieldnames=["name", "status", "bytes", "url"]) + writer.writeheader() + writer.writerows(rows) + + +def main() -> int: + global REQUEST_TIMEOUT + parser = argparse.ArgumentParser(description="批量下载所有 zpk 文件。") + parser.add_argument("--index", type=Path, default=INDEX_CSV) + parser.add_argument("--out-dir", type=Path, default=ZPK_DIR) + parser.add_argument("--report", type=Path, default=REPORT_CSV) + parser.add_argument("--workers", type=int, default=64) + parser.add_argument("--timeout", type=float, default=8, help="单个 CDN 请求超时秒数;Ctrl+C 最多等待活跃请求到此超时") + parser.add_argument("--progress-every", type=int, default=500) + parser.add_argument( + "--cdn-mode", + choices=("spread", "primary"), + default="primary", + help="primary 会始终优先第一个 CDN;spread 会把 worker 分散到 3 个 CDN", + ) + parser.add_argument("--limit", type=int, default=0, help="最多下载 N 个(冒烟测试)") + parser.add_argument("--force", action="store_true", help="已存在也重新下载") + args = parser.parse_args() + + REQUEST_TIMEOUT = max(1.0, args.timeout) + zpk_dir = args.out_dir + + names = load_names(args.index) + if not args.force: + names = [n for n in names if not (zpk_dir / f"{n}.zpk").exists()] + if args.limit: + names = names[: args.limit] + + total = len(names) + print(f"total_to_download={total} out_dir={zpk_dir}", flush=True) + + workers = max(1, args.workers) + if workers == 1: + endpoint_client = CdnClient(build_endpoints(args.cdn_mode), REQUEST_TIMEOUT) + results = [] + started_at = time.monotonic() + bytes_done = 0 + try: + for done, name in enumerate(names, 1): + r = download_one(name, zpk_dir, endpoint_client) + results.append(r) + if r["status"] in {"downloaded", "skip-exists"}: + bytes_done += int(r["bytes"]) + print(progress_line(done, total, r, started_at, bytes_done), flush=True) + except KeyboardInterrupt: + STOP_EVENT.set() + print(f"\ninterrupted done={len(results)}/{total}; writing partial report...", flush=True) + finally: + endpoint_client.close() + else: + results = run_parallel(names, zpk_dir, workers, max(1, args.progress_every), args.cdn_mode) + + write_report(results, args.report) + ok = sum(1 for r in results if r["status"] in {"downloaded", "skip-exists"}) + failed = len(results) - ok + cancelled = sum(1 for r in results if r["status"] == "cancelled") + print(f"done ok={ok} failed={failed} cancelled={cancelled} report={args.report}", flush=True) + if STOP_EVENT.is_set(): + return 130 + return 0 if failed == 0 else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/importer/src/archive/export-word-zpk-index.py b/importer/src/archive/export-word-zpk-index.py new file mode 100644 index 0000000..0a616cb --- /dev/null +++ b/importer/src/archive/export-word-zpk-index.py @@ -0,0 +1,121 @@ +"""Expand each exported word zpk name into structured index fields.""" + +from __future__ import annotations + +import argparse +import csv +from datetime import datetime, timezone +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_WORDS = ROOT / "runtime-dump" / "all-wordbook-words.csv" +DEFAULT_OUT = ROOT / "runtime-dump" / "all-wordbook-zpk-index.csv" +DEFAULT_UNIQUE_OUT = ROOT / "runtime-dump" / "unique-zpk-index.csv" + +OUT_FIELDS = ( + "book_id", + "book_code", + "book_name", + "word", + "word_id", + "zpk", + "zpk_bid", + "zpk_word_id", + "zpk_version", + "zpk_updated_unix", + "zpk_updated_utc", +) + +UNIQUE_FIELDS = ( + "zpk", + "zpk_bid", + "zpk_word_id", + "zpk_version", + "zpk_updated_unix", + "zpk_updated_utc", + "first_word", + "book_occurrences", +) + + +def parse_zpk(zpk: str) -> dict[str, str]: + parts = (zpk or "").split("_") + if len(parts) != 4 or not all(part.isdigit() for part in parts): + return { + "zpk_bid": "", + "zpk_word_id": "", + "zpk_version": "", + "zpk_updated_unix": "", + "zpk_updated_utc": "", + } + + updated = int(parts[3]) + try: + updated_utc = datetime.fromtimestamp(updated, tz=timezone.utc).isoformat().replace("+00:00", "Z") + except (OverflowError, OSError, ValueError): + updated_utc = "" + + return { + "zpk_bid": parts[0], + "zpk_word_id": parts[1], + "zpk_version": parts[2], + "zpk_updated_unix": parts[3], + "zpk_updated_utc": updated_utc, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--words", type=Path, default=DEFAULT_WORDS) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument("--unique-out", type=Path, default=DEFAULT_UNIQUE_OUT) + args = parser.parse_args() + + unique: dict[str, dict[str, str | int]] = {} + total = 0 + args.out.parent.mkdir(parents=True, exist_ok=True) + with args.words.open("r", encoding="utf-8-sig", newline="") as in_fh, args.out.open( + "w", encoding="utf-8-sig", newline="" + ) as out_fh: + reader = csv.DictReader(in_fh) + writer = csv.DictWriter(out_fh, fieldnames=OUT_FIELDS) + writer.writeheader() + for row in reader: + zpk = row.get("zpk", "") + parsed = parse_zpk(zpk) + out_row = { + "book_id": row.get("book_id", ""), + "book_code": row.get("book_code", ""), + "book_name": row.get("book_name", ""), + "word": row.get("word", ""), + "word_id": row.get("id", ""), + "zpk": zpk, + **parsed, + } + writer.writerow(out_row) + total += 1 + + if zpk not in unique: + unique[zpk] = { + "zpk": zpk, + **parsed, + "first_word": row.get("word", ""), + "book_occurrences": 0, + } + unique[zpk]["book_occurrences"] = int(unique[zpk]["book_occurrences"]) + 1 + + with args.unique_out.open("w", encoding="utf-8-sig", newline="") as fh: + writer = csv.DictWriter(fh, fieldnames=UNIQUE_FIELDS) + writer.writeheader() + writer.writerows(unique.values()) + + mismatches = sum(1 for zpk, item in unique.items() if item["zpk_word_id"] and item["zpk_word_id"] != "") + print(f"rows={total} unique_zpk={len(unique)} parsed_unique={mismatches}") + print(f"out={args.out}") + print(f"unique_out={args.unique_out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/importer/src/archive/export-wordbook-words.py b/importer/src/archive/export-wordbook-words.py new file mode 100644 index 0000000..ac30612 --- /dev/null +++ b/importer/src/archive/export-wordbook-words.py @@ -0,0 +1,282 @@ +"""Export words from every downloaded LangEasy Lexis V3 word-book DB.""" + +from __future__ import annotations + +import argparse +import csv +import importlib.util +import json +import re +import sqlite3 +from pathlib import Path +from typing import Iterable + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_CATALOG = ROOT / "runtime-dump" / "book-catalog.json" +DEFAULT_DB_DIR = ROOT / "runtime-dump" / "databases" +DEFAULT_DECRYPTED_DIR = ROOT / "runtime-dump" / "decrypted-databases" +DEFAULT_PER_BOOK_DIR = ROOT / "runtime-dump" / "wordbook-words" +DEFAULT_COMBINED_CSV = ROOT / "runtime-dump" / "all-wordbook-words.csv" +DEFAULT_SUMMARY_CSV = ROOT / "runtime-dump" / "wordbook-word-summary.csv" +DECRYPTOR_PATH = ROOT / "tools" / "decrypt-sqlcipher2-db.py" + +WORD_FIELDS = ( + "book_id", + "book_code", + "book_name", + "book_count", + "book_exam", + "book_version", + "id", + "word", + "main_word", + "seq_book", + "seq_az", + "seq_freq", + "seq_rand", + "zpk", +) + +SUMMARY_FIELDS = ( + "book_id", + "book_code", + "book_name", + "book_count", + "exported_count", + "status", + "table", + "local_db", + "decrypted_db", + "export_csv", + "error", +) + + +def load_decrypt_db(): + spec = importlib.util.spec_from_file_location("decrypt_sqlcipher2_db", DECRYPTOR_PATH) + if spec is None or spec.loader is None: + raise RuntimeError(f"Cannot load decryptor: {DECRYPTOR_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.decrypt_db + + +def load_books(path: Path) -> list[dict]: + with path.open("r", encoding="utf-8") as fh: + data = json.load(fh) + if isinstance(data, dict) and isinstance(data.get("books"), list): + return data["books"] + if isinstance(data, list): + return data + raise ValueError(f"Unsupported catalog shape: {path}") + + +def safe_code_filename(code: object) -> str: + text = str(code or "UNKNOWN") + return re.sub(r"[^A-Za-z0-9_.-]+", "_", text).strip("._") or "UNKNOWN" + + +def quote_identifier(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +def list_tables(conn: sqlite3.Connection) -> list[str]: + return [ + row[0] + for row in conn.execute( + "select name from sqlite_master where type='table' and name not like 'sqlite_%' order by name" + ) + ] + + +def table_columns(conn: sqlite3.Connection, table: str) -> set[str]: + return {row[1] for row in conn.execute(f"pragma table_info({quote_identifier(table)})")} + + +def choose_word_table(conn: sqlite3.Connection, code: str) -> str: + tables = list_tables(conn) + preferred = f"LangEasyLexisV3_{code}" + if preferred in tables: + return preferred + + for table in tables: + cols = table_columns(conn, table) + if {"id", "word"}.issubset(cols): + return table + + raise RuntimeError(f"no word table found; tables={tables}") + + +def ensure_decrypted(src: Path, dst: Path, decrypt_db, force: bool) -> None: + if ( + not force + and dst.exists() + and dst.stat().st_size > 0 + and dst.stat().st_mtime >= src.stat().st_mtime + ): + return + decrypt_db(src, dst) + + +def iter_words(conn: sqlite3.Connection, table: str) -> Iterable[sqlite3.Row]: + conn.row_factory = sqlite3.Row + cols = table_columns(conn, table) + required = {"id", "word", "main_word", "seq_book", "seq_az", "seq_freq", "seq_rand", "zpk"} + missing = sorted(required - cols) + if missing: + raise RuntimeError(f"table {table} missing columns: {missing}") + + query = ( + f"select id, word, main_word, seq_book, seq_az, seq_freq, seq_rand, zpk " + f"from {quote_identifier(table)} order by seq_book, id" + ) + return conn.execute(query) + + +def export_one( + row: dict, + db_dir: Path, + decrypted_dir: Path, + per_book_dir: Path, + decrypt_db, + force_decrypt: bool, +) -> dict: + code = str(row.get("code") or "") + src = db_dir / f"LangEasyLexisV3_{code}.db" + if not src.exists(): + return summary(row, "missing_db", "", src, Path(), Path(), 0, f"not found: {src}") + + db_name = src.with_suffix(".sqlite").name + decrypted = decrypted_dir / db_name + export_csv = per_book_dir / f"{safe_code_filename(code)}.csv" + + ensure_decrypted(src, decrypted, decrypt_db, force_decrypt) + per_book_dir.mkdir(parents=True, exist_ok=True) + + with sqlite3.connect(decrypted) as conn: + table = choose_word_table(conn, code) + count = 0 + with export_csv.open("w", encoding="utf-8-sig", newline="") as fh: + writer = csv.DictWriter(fh, fieldnames=WORD_FIELDS) + writer.writeheader() + for word_row in iter_words(conn, table): + writer.writerow( + { + "book_id": row.get("id", ""), + "book_code": code, + "book_name": row.get("name", ""), + "book_count": row.get("count", ""), + "book_exam": row.get("exam", ""), + "book_version": row.get("version", ""), + "id": word_row["id"], + "word": word_row["word"], + "main_word": word_row["main_word"], + "seq_book": word_row["seq_book"], + "seq_az": word_row["seq_az"], + "seq_freq": word_row["seq_freq"], + "seq_rand": word_row["seq_rand"], + "zpk": word_row["zpk"], + } + ) + count += 1 + + return summary(row, "ok", table, src, decrypted, export_csv, count, "") + + +def summary( + row: dict, + status: str, + table: str, + src: Path, + decrypted: Path, + export_csv: Path, + exported_count: int, + error: str, +) -> dict: + return { + "book_id": row.get("id", ""), + "book_code": row.get("code", ""), + "book_name": row.get("name", ""), + "book_count": row.get("count", ""), + "exported_count": exported_count, + "status": status, + "table": table, + "local_db": str(src) if src else "", + "decrypted_db": str(decrypted) if decrypted else "", + "export_csv": str(export_csv) if export_csv else "", + "error": error, + } + + +def write_summary(rows: list[dict], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8-sig", newline="") as fh: + writer = csv.DictWriter(fh, fieldnames=SUMMARY_FIELDS) + writer.writeheader() + writer.writerows(rows) + + +def write_combined(summary_rows: list[dict], path: Path) -> int: + total = 0 + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8-sig", newline="") as out_fh: + writer = csv.DictWriter(out_fh, fieldnames=WORD_FIELDS) + writer.writeheader() + for item in summary_rows: + if item["status"] != "ok": + continue + with Path(item["export_csv"]).open("r", encoding="utf-8-sig", newline="") as in_fh: + reader = csv.DictReader(in_fh) + for row in reader: + writer.writerow(row) + total += 1 + return total + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) + parser.add_argument("--db-dir", type=Path, default=DEFAULT_DB_DIR) + parser.add_argument("--decrypted-dir", type=Path, default=DEFAULT_DECRYPTED_DIR) + parser.add_argument("--per-book-dir", type=Path, default=DEFAULT_PER_BOOK_DIR) + parser.add_argument("--combined-csv", type=Path, default=DEFAULT_COMBINED_CSV) + parser.add_argument("--summary-csv", type=Path, default=DEFAULT_SUMMARY_CSV) + parser.add_argument("--limit", type=int, default=0) + parser.add_argument("--force-decrypt", action="store_true") + parser.add_argument("--skip-combined", action="store_true") + parser.add_argument("--progress-every", type=int, default=25) + args = parser.parse_args() + + decrypt_db = load_decrypt_db() + books = [row for row in load_books(args.catalog) if row.get("url") != "*"] + if args.limit: + books = books[: args.limit] + + summaries = [] + for index, row in enumerate(books, start=1): + try: + item = export_one(row, args.db_dir, args.decrypted_dir, args.per_book_dir, decrypt_db, args.force_decrypt) + except Exception as exc: # Keep the batch moving and surface failures in the summary. + item = summary(row, "error", "", args.db_dir / f"LangEasyLexisV3_{row.get('code')}.db", Path(), Path(), 0, repr(exc)) + summaries.append(item) + if args.progress_every and (index % args.progress_every == 0 or index == len(books)): + ok = sum(1 for entry in summaries if entry["status"] == "ok") + print(f"progress={index}/{len(books)} ok={ok}") + + write_summary(summaries, args.summary_csv) + combined_total = 0 if args.skip_combined else write_combined(summaries, args.combined_csv) + ok_count = sum(1 for item in summaries if item["status"] == "ok") + failed_count = len(summaries) - ok_count + exported_total = sum(int(item["exported_count"] or 0) for item in summaries) + + print(f"books={len(summaries)} ok={ok_count} failed={failed_count} words={exported_total}") + print(f"summary={args.summary_csv}") + print(f"per_book_dir={args.per_book_dir}") + if not args.skip_combined: + print(f"combined={args.combined_csv} combined_words={combined_total}") + return 0 if failed_count == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/importer/src/archive/extract-book-catalog.py b/importer/src/archive/extract-book-catalog.py new file mode 100644 index 0000000..70a9d89 --- /dev/null +++ b/importer/src/archive/extract-book-catalog.py @@ -0,0 +1,69 @@ +import html +import json +import os +import sys +import xml.etree.ElementTree as ET + + +if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + +PREF = sys.argv[1] if len(sys.argv) > 1 else "runtime-dump/shared_prefs/User_Preference_344124367.xml" +OUT = sys.argv[2] if len(sys.argv) > 2 else "runtime-dump/book-catalog.json" +FIELDS = ("id", "code", "name", "count", "exam", "url", "version", "bookVersion", "purchased", "forSale") + + +def get_pref_map(path): + result = {} + root = ET.parse(path).getroot() + for el in root: + name = el.attrib.get("name") + if not name: + continue + result[name] = html.unescape(el.attrib.get("value", el.text or "")).strip() + return result + + +def compact_book(book): + return {key: book.get(key) for key in FIELDS if key in book} + + +def walk_books(value): + if isinstance(value, dict): + if "code" in value and "url" in value: + yield value + for child in value.values(): + yield from walk_books(child) + elif isinstance(value, list): + for child in value: + yield from walk_books(child) + + +def main(): + prefs = get_pref_map(PREF) + found = {} + for key in ("key_group_book_List_V3", "lexis_matched_books", "library_learning"): + raw = prefs.get(key) + if not raw: + continue + try: + value = json.loads(raw) + except json.JSONDecodeError: + continue + for book in walk_books(value): + code = book.get("code") + if code and code not in found: + found[code] = compact_book(book) + + catalog = sorted(found.values(), key=lambda item: (str(item.get("exam", "")), str(item.get("code", "")))) + os.makedirs(os.path.dirname(OUT), exist_ok=True) + with open(OUT, "w", encoding="utf-8") as f: + json.dump(catalog, f, ensure_ascii=False, indent=2) + f.write("\n") + print(f"wrote {len(catalog)} books -> {OUT}") + for item in catalog[:30]: + print(json.dumps(item, ensure_ascii=False)) + + +if __name__ == "__main__": + main()