chore(importer): 归档旧版分步工具脚本

保留早期的全量下载、词表导出、目录提取与 DB key 派生脚本,
供复核历史流程与 libcommon.so 常量时参考,日常流程不再使用。

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
voidandClaude Sonnet 4.6 committed 2026-11-08 21:15:56 +08:00
1 parent f030334243
commit f03ef43c21
7 files changed
+1130

No files matched your search

@@ -0,0 +1,103 @@
"""Build a sanitized remote word-book manifest from the runtime catalog."""
from __future__ import annotations
import argparse
import csv
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_CATALOG = ROOT / "runtime-dump" / "book-catalog.json"
DEFAULT_DB_DIR = ROOT / "runtime-dump" / "databases"
DEFAULT_JSONL = ROOT / "runtime-dump" / "book-download-manifest.jsonl"
DEFAULT_CSV = ROOT / "runtime-dump" / "book-download-manifest.csv"
DEFAULT_HOST = "https://api.beingfine.cn"
FIELDS = (
"id",
"code",
"name",
"count",
"exam",
"url",
"remote_url",
"version",
"bookVersion",
"purchased",
"forSale",
"is_remote",
"local_db",
"local_size",
)
def load_books(path: Path) -> list[dict]:
with path.open("r", encoding="utf-8") as fh:
data = json.load(fh)
if isinstance(data, dict) and isinstance(data.get("books"), list):
return data["books"]
if isinstance(data, list):
return data
raise ValueError(f"Unsupported catalog shape: {path}")
def build_rows(books: list[dict], db_dir: Path, host: str) -> list[dict]:
rows = []
for book in books:
url = str(book.get("url") or "")
filename = Path(url).name
local_path = db_dir / filename if filename else None
exists = bool(local_path and local_path.exists())
row = {field: "" for field in FIELDS}
for field in ("id", "code", "name", "count", "exam", "url", "version", "bookVersion", "purchased", "forSale"):
row[field] = book.get(field, "")
row["remote_url"] = host.rstrip("/") + url if url.startswith("/") else url
row["is_remote"] = bool(url.startswith("/"))
row["local_db"] = str(local_path) if exists else ""
row["local_size"] = local_path.stat().st_size if exists and local_path else ""
rows.append(row)
return rows
def write_jsonl(rows: list[dict], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as fh:
for row in rows:
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
def write_csv(rows: list[dict], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8-sig", newline="") as fh:
writer = csv.DictWriter(fh, fieldnames=FIELDS)
writer.writeheader()
writer.writerows(rows)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
parser.add_argument("--db-dir", type=Path, default=DEFAULT_DB_DIR)
parser.add_argument("--host", default=DEFAULT_HOST)
parser.add_argument("--jsonl", type=Path, default=DEFAULT_JSONL)
parser.add_argument("--csv", type=Path, default=DEFAULT_CSV)
args = parser.parse_args()
books = load_books(args.catalog)
rows = build_rows(books, args.db_dir, args.host)
write_jsonl(rows, args.jsonl)
write_csv(rows, args.csv)
local_count = sum(1 for row in rows if row["local_db"])
remote_count = sum(1 for row in rows if row["is_remote"])
missing_count = sum(1 for row in rows if row["is_remote"] and not row["local_db"])
print(f"books={len(rows)} remote={remote_count} local={local_count} missing_remote={missing_count}")
print(f"jsonl={args.jsonl}")
print(f"csv={args.csv}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,54 @@
#!/usr/bin/env python3
from pathlib import Path
import struct
import hashlib
LIBCOMMON_X86 = Path("jadx-out/resources/lib/x86/libcommon.so")
PACKAGE = b"cn.com.langeasy.LangEasyLexis"
PACKAGE_SHA256 = hashlib.sha256(PACKAGE).hexdigest().encode("ascii")
# x86 getDBKey() computes ebx as 0x2ff0 and dec(type) before jump dispatch.
# type=0 falls through to the default branch.
TABLES = {
0: (0x2FF0 - 0x260B, 0x0E),
1: (0x2FF0 - 0x26E3, 0x24),
2: (0x2FF0 - 0x2652, 0x25),
3: (0x2FF0 - 0x2699, 0x25),
4: (0x2FF0 - 0x2674, 0x22),
5: (0x2FF0 - 0x26BF, 0x26),
6: (0x2FF0 - 0x262D, 0x22),
}
def decode(blob: bytes, offset: int, size: int) -> str:
out = bytearray()
for i, b in enumerate(blob[offset : offset + size]):
out.append(b ^ PACKAGE_SHA256[i % len(PACKAGE_SHA256)])
return out.decode("utf-8")
def jump_targets(blob: bytes) -> dict[int, int]:
base = 0x2FF0
table = base - 0x2820
targets = {}
for selector in range(1, 7):
rel = struct.unpack_from("<i", blob, table + (selector - 1) * 4)[0]
targets[selector] = base + rel
return targets
def main() -> int:
blob = LIBCOMMON_X86.read_bytes()
print("jump_targets:")
for selector, target in jump_targets(blob).items():
print(f" type={selector}: 0x{target:x}")
print("keys:")
for key_type, (offset, size) in TABLES.items():
print(f" DBKEY[{key_type}]={decode(blob, offset, size)}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,177 @@
#!/usr/bin/env python3
import argparse
import base64
import csv
import hashlib
import json
import os
import sys
import time
import zipfile
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parents[1]
CATALOG = ROOT / "runtime-dump" / "book-catalog.json"
ZIP_DIR = ROOT / "runtime-dump" / "remote-zips"
DB_DIR = ROOT / "runtime-dump" / "databases"
REPORT_CSV = ROOT / "runtime-dump" / "wordbook-download-report.csv"
BASE_STATIC_URLS = [
"http://static.beingfine.cn",
"https://hwstatic.beingfine.cn",
"https://ccastatic.beingfine.cn",
]
REFERER = "https://app.bbdc.cn"
USER_AGENT = "LangEasyLexis/5.11.1"
def load_books(path: Path) -> list[dict]:
with path.open("r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, dict) and isinstance(data.get("books"), list):
return data["books"]
if isinstance(data, list):
return data
raise ValueError(f"Unsupported catalog shape: {path}")
def book_url(book: dict, base_url: str) -> str:
code = book["code"]
version = str(book.get("version") or 0)
db_name = f"LangEasyLexisV3_{code}.db"
return f"{base_url}/dicts/v3/{code}/{version}/{db_name}.zip"
def request_url(url: str):
req = Request(
url,
headers={
"Referer": REFERER,
"User-Agent": USER_AGENT,
},
)
return urlopen(req, timeout=45)
def md5_b64(path: Path) -> str:
h = hashlib.md5()
with path.open("rb") as f:
for chunk in iter(lambda: f.read(1024 * 1024), b""):
h.update(chunk)
return base64.b64encode(h.digest()).decode("ascii")
def download_one(book: dict, force: bool = False) -> dict:
code = book["code"]
if book.get("url") == "*":
return {"code": code, "status": "skip-virtual-book", "url": "", "bytes": 0}
zip_path = ZIP_DIR / f"LangEasyLexisV3_{code}.db.zip"
db_path = DB_DIR / f"LangEasyLexisV3_{code}.db"
if db_path.exists() and not force:
return {"code": code, "status": "skip-db-exists", "url": "", "bytes": db_path.stat().st_size}
last_error = ""
for base in BASE_STATIC_URLS:
url = book_url(book, base)
try:
with request_url(url) as resp:
status = getattr(resp, "status", 200)
if status != 200:
last_error = f"http-{status}"
continue
ZIP_DIR.mkdir(parents=True, exist_ok=True)
tmp_path = zip_path.with_suffix(zip_path.suffix + ".tmp")
with tmp_path.open("wb") as out:
while True:
chunk = resp.read(1024 * 1024)
if not chunk:
break
out.write(chunk)
content_md5 = resp.headers.get("Content-MD5")
if content_md5 and md5_b64(tmp_path) != content_md5:
tmp_path.unlink(missing_ok=True)
last_error = "md5-mismatch"
continue
tmp_path.replace(zip_path)
DB_DIR.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(zip_path) as zf:
names = zf.namelist()
wanted = f"LangEasyLexisV3_{code}.db"
member = wanted if wanted in names else names[0]
zf.extract(member, DB_DIR)
extracted = DB_DIR / member
if extracted != db_path:
extracted.replace(db_path)
return {"code": code, "status": "downloaded", "url": url, "bytes": db_path.stat().st_size}
except HTTPError as e:
last_error = f"http-{e.code}"
except (URLError, TimeoutError, zipfile.BadZipFile, OSError) as e:
last_error = type(e).__name__
return {"code": code, "status": f"failed:{last_error}", "url": "", "bytes": 0}
def write_report(rows: list[dict]) -> None:
REPORT_CSV.parent.mkdir(parents=True, exist_ok=True)
with REPORT_CSV.open("w", newline="", encoding="utf-8-sig") as f:
writer = csv.DictWriter(f, fieldnames=["code", "status", "bytes", "url"])
writer.writeheader()
writer.writerows(rows)
def main() -> int:
parser = argparse.ArgumentParser(description="Download all Lexis V3 wordbook DB zips from the app's CDN interface.")
parser.add_argument("--catalog", type=Path, default=CATALOG)
parser.add_argument("--force", action="store_true", help="re-download books even if the extracted DB already exists")
parser.add_argument("--limit", type=int, default=0, help="download at most N books, useful for smoke tests")
parser.add_argument("--sleep", type=float, default=0.15, help="seconds to sleep between downloads in single-worker mode")
parser.add_argument("--workers", type=int, default=1, help="parallel downloads")
parser.add_argument("--include-existing", action="store_true", help="include already cached local DBs in the run")
args = parser.parse_args()
books = load_books(args.catalog)
if not args.include_existing and not args.force:
books = [b for b in books if not (DB_DIR / f"LangEasyLexisV3_{b['code']}.db").exists()]
if args.limit:
books = books[: args.limit]
total = len(books)
results = []
done = 0
workers = max(1, args.workers)
if workers == 1:
for book in books:
result = download_one(book, force=args.force)
results.append(result)
done += 1
print(f"[{done}/{total}] {result['code']} {result['status']} {result['bytes']}", flush=True)
if args.sleep:
time.sleep(args.sleep)
else:
with ThreadPoolExecutor(max_workers=workers) as executor:
futures = [executor.submit(download_one, book, args.force) for book in books]
for future in as_completed(futures):
result = future.result()
results.append(result)
done += 1
print(f"[{done}/{total}] {result['code']} {result['status']} {result['bytes']}", flush=True)
write_report(results)
ok = sum(1 for r in results if r["status"] in {"downloaded", "skip-db-exists", "skip-virtual-book"})
failed = len(results) - ok
print(f"done ok={ok} failed={failed} report={REPORT_CSV}", flush=True)
return 0 if failed == 0 else 2
if __name__ == "__main__":
sys.exit(main())
+324
View File
@@ -0,0 +1,324 @@
#!/usr/bin/env python3
"""批量下载所有 zpk 文件。
来源:unique-zpk-index.csv 中的唯一 zpk 名(115104 个)。
CDN:{base}/r/{zpkName}.zpk,Referer 同词书下载。
落地:runtime-dump/zpk-cache/0/{zpkName}.zpk(与设备目录结构一致)。
已存在的文件自动跳过,可断点续传。
"""
from __future__ import annotations
import argparse
import base64
import csv
import hashlib
import http.client
import queue
import sys
import threading
import time
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import urlparse
ROOT = Path(__file__).resolve().parents[1]
INDEX_CSV = ROOT / "runtime-dump" / "unique-zpk-index.csv"
ZPK_DIR = ROOT / "runtime-dump" / "zpk-cache" / "0"
REPORT_CSV = ROOT / "runtime-dump" / "zpk-download-report.csv"
BASE_STATIC_URLS = [
"http://static.beingfine.cn",
"https://hwstatic.beingfine.cn",
"https://ccastatic.beingfine.cn",
]
REFERER = "https://app.bbdc.cn"
USER_AGENT = "LangEasyLexis/5.11.1"
REQUEST_TIMEOUT = 8.0
STOP_EVENT = threading.Event()
@dataclass(frozen=True)
class CdnEndpoint:
base_url: str
scheme: str
host: str
port: int | None
@classmethod
def parse(cls, base_url: str) -> "CdnEndpoint":
parsed = urlparse(base_url)
return cls(base_url=base_url, scheme=parsed.scheme, host=parsed.hostname or "", port=parsed.port)
class CdnClient:
def __init__(self, endpoints: list[CdnEndpoint], timeout: float) -> None:
self.endpoints = endpoints
self.timeout = timeout
self.connections: dict[str, http.client.HTTPConnection] = {}
def close(self) -> None:
for conn in self.connections.values():
conn.close()
self.connections.clear()
def _conn(self, endpoint: CdnEndpoint) -> http.client.HTTPConnection:
key = endpoint.base_url
conn = self.connections.get(key)
if conn is not None:
return conn
conn_cls = http.client.HTTPSConnection if endpoint.scheme == "https" else http.client.HTTPConnection
conn = conn_cls(endpoint.host, endpoint.port, timeout=self.timeout)
self.connections[key] = conn
return conn
def get(self, endpoint: CdnEndpoint, name: str) -> tuple[str, http.client.HTTPResponse]:
url_path = f"/r/{name}.zpk"
headers = {
"Connection": "keep-alive",
"Referer": REFERER,
"User-Agent": USER_AGENT,
}
conn = self._conn(endpoint)
try:
conn.request("GET", url_path, headers=headers)
return f"{endpoint.base_url}{url_path}", conn.getresponse()
except (http.client.HTTPException, OSError):
conn.close()
self.connections.pop(endpoint.base_url, None)
raise
def drop(self, endpoint: CdnEndpoint) -> None:
conn = self.connections.pop(endpoint.base_url, None)
if conn is not None:
conn.close()
def build_endpoints(cdn_mode: str, worker_index: int = 0) -> list[CdnEndpoint]:
base_urls = BASE_STATIC_URLS
if cdn_mode == "spread" and base_urls:
offset = worker_index % len(base_urls)
base_urls = base_urls[offset:] + base_urls[:offset]
return [CdnEndpoint.parse(base) for base in base_urls]
def load_names(path: Path) -> list[str]:
names = []
seen = set()
with path.open("r", encoding="utf-8-sig", newline="") as f:
for row in csv.DictReader(f):
name = row.get("zpk")
if name and name not in seen:
seen.add(name)
names.append(name)
return names
def read_and_save_response(resp: http.client.HTTPResponse, dest: Path) -> int:
tmp = dest.with_name(f"{dest.name}.{threading.get_ident()}.tmp")
h = hashlib.md5()
size = 0
content_md5 = resp.getheader("Content-MD5")
dest.parent.mkdir(parents=True, exist_ok=True)
with tmp.open("wb") as out:
while not STOP_EVENT.is_set():
chunk = resp.read(256 * 1024)
if not chunk:
break
out.write(chunk)
h.update(chunk)
size += len(chunk)
if STOP_EVENT.is_set():
tmp.unlink(missing_ok=True)
raise InterruptedError
if content_md5 and base64.b64encode(h.digest()).decode("ascii") != content_md5:
tmp.unlink(missing_ok=True)
raise ValueError("md5-mismatch")
tmp.replace(dest)
return size
def download_one(name: str, zpk_dir: Path, client: CdnClient) -> dict:
dest = zpk_dir / f"{name}.zpk"
if dest.exists() and dest.stat().st_size > 0:
return {"name": name, "status": "skip-exists", "bytes": dest.stat().st_size, "url": ""}
last_error = ""
for endpoint in client.endpoints:
if STOP_EVENT.is_set():
return {"name": name, "status": "cancelled", "bytes": 0, "url": ""}
try:
url, resp = client.get(endpoint, name)
try:
if resp.status != 200:
last_error = f"http-{resp.status}"
resp.read()
if resp.status >= 500:
client.drop(endpoint)
continue
size = read_and_save_response(resp, dest)
return {"name": name, "status": "downloaded", "bytes": size, "url": url}
finally:
resp.close()
except InterruptedError:
return {"name": name, "status": "cancelled", "bytes": 0, "url": ""}
except ValueError as e:
last_error = str(e)
client.drop(endpoint)
except (http.client.HTTPException, OSError, TimeoutError) as e:
last_error = type(e).__name__
client.drop(endpoint)
return {"name": name, "status": f"failed:{last_error}", "bytes": 0, "url": ""}
def progress_line(done: int, total: int, result: dict, started_at: float, bytes_done: int) -> str:
elapsed = max(0.001, time.monotonic() - started_at)
files_per_sec = done / elapsed
mib_per_sec = bytes_done / 1024 / 1024 / elapsed
return (
f"[{done}/{total}] {result['name']} {result['status']} {result['bytes']} "
f"rate={files_per_sec:.1f}/s {mib_per_sec:.2f}MiB/s"
)
def run_parallel(names: list[str], zpk_dir: Path, workers: int, progress_every: int, cdn_mode: str) -> list[dict]:
work: queue.Queue[str | None] = queue.Queue(maxsize=max(1, workers * 4))
results: list[dict] = []
results_lock = threading.Lock()
started_at = time.monotonic()
done = 0
bytes_done = 0
total = len(names)
def record(result: dict) -> None:
nonlocal done, bytes_done
with results_lock:
results.append(result)
done += 1
if result["status"] in {"downloaded", "skip-exists"}:
bytes_done += int(result["bytes"])
if done % progress_every == 0 or done == total or STOP_EVENT.is_set():
print(progress_line(done, total, result, started_at, bytes_done), flush=True)
def worker(worker_index: int) -> None:
client = CdnClient(build_endpoints(cdn_mode, worker_index), REQUEST_TIMEOUT)
try:
while not STOP_EVENT.is_set():
try:
name = work.get(timeout=0.25)
except queue.Empty:
continue
try:
if name is None:
return
record(download_one(name, zpk_dir, client))
finally:
work.task_done()
finally:
client.close()
threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(workers)]
for thread in threads:
thread.start()
try:
for name in names:
while not STOP_EVENT.is_set():
try:
work.put(name, timeout=0.25)
break
except queue.Full:
continue
if STOP_EVENT.is_set():
break
for _ in threads:
work.put(None)
work.join()
except KeyboardInterrupt:
STOP_EVENT.set()
print(f"\ninterrupted done={done}/{total}; writing partial report...", flush=True)
finally:
for thread in threads:
thread.join(timeout=REQUEST_TIMEOUT + 1)
return results
def write_report(rows: list[dict], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", newline="", encoding="utf-8-sig") as f:
writer = csv.DictWriter(f, fieldnames=["name", "status", "bytes", "url"])
writer.writeheader()
writer.writerows(rows)
def main() -> int:
global REQUEST_TIMEOUT
parser = argparse.ArgumentParser(description="批量下载所有 zpk 文件。")
parser.add_argument("--index", type=Path, default=INDEX_CSV)
parser.add_argument("--out-dir", type=Path, default=ZPK_DIR)
parser.add_argument("--report", type=Path, default=REPORT_CSV)
parser.add_argument("--workers", type=int, default=64)
parser.add_argument("--timeout", type=float, default=8, help="单个 CDN 请求超时秒数;Ctrl+C 最多等待活跃请求到此超时")
parser.add_argument("--progress-every", type=int, default=500)
parser.add_argument(
"--cdn-mode",
choices=("spread", "primary"),
default="primary",
help="primary 会始终优先第一个 CDN;spread 会把 worker 分散到 3 个 CDN",
)
parser.add_argument("--limit", type=int, default=0, help="最多下载 N 个(冒烟测试)")
parser.add_argument("--force", action="store_true", help="已存在也重新下载")
args = parser.parse_args()
REQUEST_TIMEOUT = max(1.0, args.timeout)
zpk_dir = args.out_dir
names = load_names(args.index)
if not args.force:
names = [n for n in names if not (zpk_dir / f"{n}.zpk").exists()]
if args.limit:
names = names[: args.limit]
total = len(names)
print(f"total_to_download={total} out_dir={zpk_dir}", flush=True)
workers = max(1, args.workers)
if workers == 1:
endpoint_client = CdnClient(build_endpoints(args.cdn_mode), REQUEST_TIMEOUT)
results = []
started_at = time.monotonic()
bytes_done = 0
try:
for done, name in enumerate(names, 1):
r = download_one(name, zpk_dir, endpoint_client)
results.append(r)
if r["status"] in {"downloaded", "skip-exists"}:
bytes_done += int(r["bytes"])
print(progress_line(done, total, r, started_at, bytes_done), flush=True)
except KeyboardInterrupt:
STOP_EVENT.set()
print(f"\ninterrupted done={len(results)}/{total}; writing partial report...", flush=True)
finally:
endpoint_client.close()
else:
results = run_parallel(names, zpk_dir, workers, max(1, args.progress_every), args.cdn_mode)
write_report(results, args.report)
ok = sum(1 for r in results if r["status"] in {"downloaded", "skip-exists"})
failed = len(results) - ok
cancelled = sum(1 for r in results if r["status"] == "cancelled")
print(f"done ok={ok} failed={failed} cancelled={cancelled} report={args.report}", flush=True)
if STOP_EVENT.is_set():
return 130
return 0 if failed == 0 else 2
if __name__ == "__main__":
sys.exit(main())
@@ -0,0 +1,121 @@
"""Expand each exported word zpk name into structured index fields."""
from __future__ import annotations
import argparse
import csv
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_WORDS = ROOT / "runtime-dump" / "all-wordbook-words.csv"
DEFAULT_OUT = ROOT / "runtime-dump" / "all-wordbook-zpk-index.csv"
DEFAULT_UNIQUE_OUT = ROOT / "runtime-dump" / "unique-zpk-index.csv"
OUT_FIELDS = (
"book_id",
"book_code",
"book_name",
"word",
"word_id",
"zpk",
"zpk_bid",
"zpk_word_id",
"zpk_version",
"zpk_updated_unix",
"zpk_updated_utc",
)
UNIQUE_FIELDS = (
"zpk",
"zpk_bid",
"zpk_word_id",
"zpk_version",
"zpk_updated_unix",
"zpk_updated_utc",
"first_word",
"book_occurrences",
)
def parse_zpk(zpk: str) -> dict[str, str]:
parts = (zpk or "").split("_")
if len(parts) != 4 or not all(part.isdigit() for part in parts):
return {
"zpk_bid": "",
"zpk_word_id": "",
"zpk_version": "",
"zpk_updated_unix": "",
"zpk_updated_utc": "",
}
updated = int(parts[3])
try:
updated_utc = datetime.fromtimestamp(updated, tz=timezone.utc).isoformat().replace("+00:00", "Z")
except (OverflowError, OSError, ValueError):
updated_utc = ""
return {
"zpk_bid": parts[0],
"zpk_word_id": parts[1],
"zpk_version": parts[2],
"zpk_updated_unix": parts[3],
"zpk_updated_utc": updated_utc,
}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--words", type=Path, default=DEFAULT_WORDS)
parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
parser.add_argument("--unique-out", type=Path, default=DEFAULT_UNIQUE_OUT)
args = parser.parse_args()
unique: dict[str, dict[str, str | int]] = {}
total = 0
args.out.parent.mkdir(parents=True, exist_ok=True)
with args.words.open("r", encoding="utf-8-sig", newline="") as in_fh, args.out.open(
"w", encoding="utf-8-sig", newline=""
) as out_fh:
reader = csv.DictReader(in_fh)
writer = csv.DictWriter(out_fh, fieldnames=OUT_FIELDS)
writer.writeheader()
for row in reader:
zpk = row.get("zpk", "")
parsed = parse_zpk(zpk)
out_row = {
"book_id": row.get("book_id", ""),
"book_code": row.get("book_code", ""),
"book_name": row.get("book_name", ""),
"word": row.get("word", ""),
"word_id": row.get("id", ""),
"zpk": zpk,
**parsed,
}
writer.writerow(out_row)
total += 1
if zpk not in unique:
unique[zpk] = {
"zpk": zpk,
**parsed,
"first_word": row.get("word", ""),
"book_occurrences": 0,
}
unique[zpk]["book_occurrences"] = int(unique[zpk]["book_occurrences"]) + 1
with args.unique_out.open("w", encoding="utf-8-sig", newline="") as fh:
writer = csv.DictWriter(fh, fieldnames=UNIQUE_FIELDS)
writer.writeheader()
writer.writerows(unique.values())
mismatches = sum(1 for zpk, item in unique.items() if item["zpk_word_id"] and item["zpk_word_id"] != "")
print(f"rows={total} unique_zpk={len(unique)} parsed_unique={mismatches}")
print(f"out={args.out}")
print(f"unique_out={args.unique_out}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,282 @@
"""Export words from every downloaded LangEasy Lexis V3 word-book DB."""
from __future__ import annotations
import argparse
import csv
import importlib.util
import json
import re
import sqlite3
from pathlib import Path
from typing import Iterable
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_CATALOG = ROOT / "runtime-dump" / "book-catalog.json"
DEFAULT_DB_DIR = ROOT / "runtime-dump" / "databases"
DEFAULT_DECRYPTED_DIR = ROOT / "runtime-dump" / "decrypted-databases"
DEFAULT_PER_BOOK_DIR = ROOT / "runtime-dump" / "wordbook-words"
DEFAULT_COMBINED_CSV = ROOT / "runtime-dump" / "all-wordbook-words.csv"
DEFAULT_SUMMARY_CSV = ROOT / "runtime-dump" / "wordbook-word-summary.csv"
DECRYPTOR_PATH = ROOT / "tools" / "decrypt-sqlcipher2-db.py"
WORD_FIELDS = (
"book_id",
"book_code",
"book_name",
"book_count",
"book_exam",
"book_version",
"id",
"word",
"main_word",
"seq_book",
"seq_az",
"seq_freq",
"seq_rand",
"zpk",
)
SUMMARY_FIELDS = (
"book_id",
"book_code",
"book_name",
"book_count",
"exported_count",
"status",
"table",
"local_db",
"decrypted_db",
"export_csv",
"error",
)
def load_decrypt_db():
spec = importlib.util.spec_from_file_location("decrypt_sqlcipher2_db", DECRYPTOR_PATH)
if spec is None or spec.loader is None:
raise RuntimeError(f"Cannot load decryptor: {DECRYPTOR_PATH}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module.decrypt_db
def load_books(path: Path) -> list[dict]:
with path.open("r", encoding="utf-8") as fh:
data = json.load(fh)
if isinstance(data, dict) and isinstance(data.get("books"), list):
return data["books"]
if isinstance(data, list):
return data
raise ValueError(f"Unsupported catalog shape: {path}")
def safe_code_filename(code: object) -> str:
text = str(code or "UNKNOWN")
return re.sub(r"[^A-Za-z0-9_.-]+", "_", text).strip("._") or "UNKNOWN"
def quote_identifier(name: str) -> str:
return '"' + name.replace('"', '""') + '"'
def list_tables(conn: sqlite3.Connection) -> list[str]:
return [
row[0]
for row in conn.execute(
"select name from sqlite_master where type='table' and name not like 'sqlite_%' order by name"
)
]
def table_columns(conn: sqlite3.Connection, table: str) -> set[str]:
return {row[1] for row in conn.execute(f"pragma table_info({quote_identifier(table)})")}
def choose_word_table(conn: sqlite3.Connection, code: str) -> str:
tables = list_tables(conn)
preferred = f"LangEasyLexisV3_{code}"
if preferred in tables:
return preferred
for table in tables:
cols = table_columns(conn, table)
if {"id", "word"}.issubset(cols):
return table
raise RuntimeError(f"no word table found; tables={tables}")
def ensure_decrypted(src: Path, dst: Path, decrypt_db, force: bool) -> None:
if (
not force
and dst.exists()
and dst.stat().st_size > 0
and dst.stat().st_mtime >= src.stat().st_mtime
):
return
decrypt_db(src, dst)
def iter_words(conn: sqlite3.Connection, table: str) -> Iterable[sqlite3.Row]:
conn.row_factory = sqlite3.Row
cols = table_columns(conn, table)
required = {"id", "word", "main_word", "seq_book", "seq_az", "seq_freq", "seq_rand", "zpk"}
missing = sorted(required - cols)
if missing:
raise RuntimeError(f"table {table} missing columns: {missing}")
query = (
f"select id, word, main_word, seq_book, seq_az, seq_freq, seq_rand, zpk "
f"from {quote_identifier(table)} order by seq_book, id"
)
return conn.execute(query)
def export_one(
row: dict,
db_dir: Path,
decrypted_dir: Path,
per_book_dir: Path,
decrypt_db,
force_decrypt: bool,
) -> dict:
code = str(row.get("code") or "")
src = db_dir / f"LangEasyLexisV3_{code}.db"
if not src.exists():
return summary(row, "missing_db", "", src, Path(), Path(), 0, f"not found: {src}")
db_name = src.with_suffix(".sqlite").name
decrypted = decrypted_dir / db_name
export_csv = per_book_dir / f"{safe_code_filename(code)}.csv"
ensure_decrypted(src, decrypted, decrypt_db, force_decrypt)
per_book_dir.mkdir(parents=True, exist_ok=True)
with sqlite3.connect(decrypted) as conn:
table = choose_word_table(conn, code)
count = 0
with export_csv.open("w", encoding="utf-8-sig", newline="") as fh:
writer = csv.DictWriter(fh, fieldnames=WORD_FIELDS)
writer.writeheader()
for word_row in iter_words(conn, table):
writer.writerow(
{
"book_id": row.get("id", ""),
"book_code": code,
"book_name": row.get("name", ""),
"book_count": row.get("count", ""),
"book_exam": row.get("exam", ""),
"book_version": row.get("version", ""),
"id": word_row["id"],
"word": word_row["word"],
"main_word": word_row["main_word"],
"seq_book": word_row["seq_book"],
"seq_az": word_row["seq_az"],
"seq_freq": word_row["seq_freq"],
"seq_rand": word_row["seq_rand"],
"zpk": word_row["zpk"],
}
)
count += 1
return summary(row, "ok", table, src, decrypted, export_csv, count, "")
def summary(
row: dict,
status: str,
table: str,
src: Path,
decrypted: Path,
export_csv: Path,
exported_count: int,
error: str,
) -> dict:
return {
"book_id": row.get("id", ""),
"book_code": row.get("code", ""),
"book_name": row.get("name", ""),
"book_count": row.get("count", ""),
"exported_count": exported_count,
"status": status,
"table": table,
"local_db": str(src) if src else "",
"decrypted_db": str(decrypted) if decrypted else "",
"export_csv": str(export_csv) if export_csv else "",
"error": error,
}
def write_summary(rows: list[dict], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8-sig", newline="") as fh:
writer = csv.DictWriter(fh, fieldnames=SUMMARY_FIELDS)
writer.writeheader()
writer.writerows(rows)
def write_combined(summary_rows: list[dict], path: Path) -> int:
total = 0
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8-sig", newline="") as out_fh:
writer = csv.DictWriter(out_fh, fieldnames=WORD_FIELDS)
writer.writeheader()
for item in summary_rows:
if item["status"] != "ok":
continue
with Path(item["export_csv"]).open("r", encoding="utf-8-sig", newline="") as in_fh:
reader = csv.DictReader(in_fh)
for row in reader:
writer.writerow(row)
total += 1
return total
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
parser.add_argument("--db-dir", type=Path, default=DEFAULT_DB_DIR)
parser.add_argument("--decrypted-dir", type=Path, default=DEFAULT_DECRYPTED_DIR)
parser.add_argument("--per-book-dir", type=Path, default=DEFAULT_PER_BOOK_DIR)
parser.add_argument("--combined-csv", type=Path, default=DEFAULT_COMBINED_CSV)
parser.add_argument("--summary-csv", type=Path, default=DEFAULT_SUMMARY_CSV)
parser.add_argument("--limit", type=int, default=0)
parser.add_argument("--force-decrypt", action="store_true")
parser.add_argument("--skip-combined", action="store_true")
parser.add_argument("--progress-every", type=int, default=25)
args = parser.parse_args()
decrypt_db = load_decrypt_db()
books = [row for row in load_books(args.catalog) if row.get("url") != "*"]
if args.limit:
books = books[: args.limit]
summaries = []
for index, row in enumerate(books, start=1):
try:
item = export_one(row, args.db_dir, args.decrypted_dir, args.per_book_dir, decrypt_db, args.force_decrypt)
except Exception as exc: # Keep the batch moving and surface failures in the summary.
item = summary(row, "error", "", args.db_dir / f"LangEasyLexisV3_{row.get('code')}.db", Path(), Path(), 0, repr(exc))
summaries.append(item)
if args.progress_every and (index % args.progress_every == 0 or index == len(books)):
ok = sum(1 for entry in summaries if entry["status"] == "ok")
print(f"progress={index}/{len(books)} ok={ok}")
write_summary(summaries, args.summary_csv)
combined_total = 0 if args.skip_combined else write_combined(summaries, args.combined_csv)
ok_count = sum(1 for item in summaries if item["status"] == "ok")
failed_count = len(summaries) - ok_count
exported_total = sum(int(item["exported_count"] or 0) for item in summaries)
print(f"books={len(summaries)} ok={ok_count} failed={failed_count} words={exported_total}")
print(f"summary={args.summary_csv}")
print(f"per_book_dir={args.per_book_dir}")
if not args.skip_combined:
print(f"combined={args.combined_csv} combined_words={combined_total}")
return 0 if failed_count == 0 else 1
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,69 @@
import html
import json
import os
import sys
import xml.etree.ElementTree as ET
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
PREF = sys.argv[1] if len(sys.argv) > 1 else "runtime-dump/shared_prefs/User_Preference_344124367.xml"
OUT = sys.argv[2] if len(sys.argv) > 2 else "runtime-dump/book-catalog.json"
FIELDS = ("id", "code", "name", "count", "exam", "url", "version", "bookVersion", "purchased", "forSale")
def get_pref_map(path):
result = {}
root = ET.parse(path).getroot()
for el in root:
name = el.attrib.get("name")
if not name:
continue
result[name] = html.unescape(el.attrib.get("value", el.text or "")).strip()
return result
def compact_book(book):
return {key: book.get(key) for key in FIELDS if key in book}
def walk_books(value):
if isinstance(value, dict):
if "code" in value and "url" in value:
yield value
for child in value.values():
yield from walk_books(child)
elif isinstance(value, list):
for child in value:
yield from walk_books(child)
def main():
prefs = get_pref_map(PREF)
found = {}
for key in ("key_group_book_List_V3", "lexis_matched_books", "library_learning"):
raw = prefs.get(key)
if not raw:
continue
try:
value = json.loads(raw)
except json.JSONDecodeError:
continue
for book in walk_books(value):
code = book.get("code")
if code and code not in found:
found[code] = compact_book(book)
catalog = sorted(found.values(), key=lambda item: (str(item.get("exam", "")), str(item.get("code", ""))))
os.makedirs(os.path.dirname(OUT), exist_ok=True)
with open(OUT, "w", encoding="utf-8") as f:
json.dump(catalog, f, ensure_ascii=False, indent=2)
f.write("\n")
print(f"wrote {len(catalog)} books -> {OUT}")
for item in catalog[:30]:
print(json.dumps(item, ensure_ascii=False))
if __name__ == "__main__":
main()