chore(importer): 归档旧版分步工具脚本
保留早期的全量下载、词表导出、目录提取与 DB key 派生脚本, 供复核历史流程与 libcommon.so 常量时参考,日常流程不再使用。 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
f030334243
commit
f03ef43c21
7 files changed
+1130
No files matched your search
@@ -0,0 +1,103 @@
|
||||
"""Build a sanitized remote word-book manifest from the runtime catalog."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_CATALOG = ROOT / "runtime-dump" / "book-catalog.json"
|
||||
DEFAULT_DB_DIR = ROOT / "runtime-dump" / "databases"
|
||||
DEFAULT_JSONL = ROOT / "runtime-dump" / "book-download-manifest.jsonl"
|
||||
DEFAULT_CSV = ROOT / "runtime-dump" / "book-download-manifest.csv"
|
||||
DEFAULT_HOST = "https://api.beingfine.cn"
|
||||
|
||||
FIELDS = (
|
||||
"id",
|
||||
"code",
|
||||
"name",
|
||||
"count",
|
||||
"exam",
|
||||
"url",
|
||||
"remote_url",
|
||||
"version",
|
||||
"bookVersion",
|
||||
"purchased",
|
||||
"forSale",
|
||||
"is_remote",
|
||||
"local_db",
|
||||
"local_size",
|
||||
)
|
||||
|
||||
|
||||
def load_books(path: Path) -> list[dict]:
|
||||
with path.open("r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
if isinstance(data, dict) and isinstance(data.get("books"), list):
|
||||
return data["books"]
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
raise ValueError(f"Unsupported catalog shape: {path}")
|
||||
|
||||
|
||||
def build_rows(books: list[dict], db_dir: Path, host: str) -> list[dict]:
|
||||
rows = []
|
||||
for book in books:
|
||||
url = str(book.get("url") or "")
|
||||
filename = Path(url).name
|
||||
local_path = db_dir / filename if filename else None
|
||||
exists = bool(local_path and local_path.exists())
|
||||
row = {field: "" for field in FIELDS}
|
||||
for field in ("id", "code", "name", "count", "exam", "url", "version", "bookVersion", "purchased", "forSale"):
|
||||
row[field] = book.get(field, "")
|
||||
row["remote_url"] = host.rstrip("/") + url if url.startswith("/") else url
|
||||
row["is_remote"] = bool(url.startswith("/"))
|
||||
row["local_db"] = str(local_path) if exists else ""
|
||||
row["local_size"] = local_path.stat().st_size if exists and local_path else ""
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def write_jsonl(rows: list[dict], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as fh:
|
||||
for row in rows:
|
||||
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def write_csv(rows: list[dict], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=FIELDS)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
|
||||
parser.add_argument("--db-dir", type=Path, default=DEFAULT_DB_DIR)
|
||||
parser.add_argument("--host", default=DEFAULT_HOST)
|
||||
parser.add_argument("--jsonl", type=Path, default=DEFAULT_JSONL)
|
||||
parser.add_argument("--csv", type=Path, default=DEFAULT_CSV)
|
||||
args = parser.parse_args()
|
||||
|
||||
books = load_books(args.catalog)
|
||||
rows = build_rows(books, args.db_dir, args.host)
|
||||
write_jsonl(rows, args.jsonl)
|
||||
write_csv(rows, args.csv)
|
||||
|
||||
local_count = sum(1 for row in rows if row["local_db"])
|
||||
remote_count = sum(1 for row in rows if row["is_remote"])
|
||||
missing_count = sum(1 for row in rows if row["is_remote"] and not row["local_db"])
|
||||
print(f"books={len(rows)} remote={remote_count} local={local_count} missing_remote={missing_count}")
|
||||
print(f"jsonl={args.jsonl}")
|
||||
print(f"csv={args.csv}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import hashlib
|
||||
|
||||
|
||||
LIBCOMMON_X86 = Path("jadx-out/resources/lib/x86/libcommon.so")
|
||||
PACKAGE = b"cn.com.langeasy.LangEasyLexis"
|
||||
PACKAGE_SHA256 = hashlib.sha256(PACKAGE).hexdigest().encode("ascii")
|
||||
|
||||
|
||||
# x86 getDBKey() computes ebx as 0x2ff0 and dec(type) before jump dispatch.
|
||||
# type=0 falls through to the default branch.
|
||||
TABLES = {
|
||||
0: (0x2FF0 - 0x260B, 0x0E),
|
||||
1: (0x2FF0 - 0x26E3, 0x24),
|
||||
2: (0x2FF0 - 0x2652, 0x25),
|
||||
3: (0x2FF0 - 0x2699, 0x25),
|
||||
4: (0x2FF0 - 0x2674, 0x22),
|
||||
5: (0x2FF0 - 0x26BF, 0x26),
|
||||
6: (0x2FF0 - 0x262D, 0x22),
|
||||
}
|
||||
|
||||
|
||||
def decode(blob: bytes, offset: int, size: int) -> str:
|
||||
out = bytearray()
|
||||
for i, b in enumerate(blob[offset : offset + size]):
|
||||
out.append(b ^ PACKAGE_SHA256[i % len(PACKAGE_SHA256)])
|
||||
return out.decode("utf-8")
|
||||
|
||||
|
||||
def jump_targets(blob: bytes) -> dict[int, int]:
|
||||
base = 0x2FF0
|
||||
table = base - 0x2820
|
||||
targets = {}
|
||||
for selector in range(1, 7):
|
||||
rel = struct.unpack_from("<i", blob, table + (selector - 1) * 4)[0]
|
||||
targets[selector] = base + rel
|
||||
return targets
|
||||
|
||||
|
||||
def main() -> int:
|
||||
blob = LIBCOMMON_X86.read_bytes()
|
||||
print("jump_targets:")
|
||||
for selector, target in jump_targets(blob).items():
|
||||
print(f" type={selector}: 0x{target:x}")
|
||||
print("keys:")
|
||||
for key_type, (offset, size) in TABLES.items():
|
||||
print(f" DBKEY[{key_type}]={decode(blob, offset, size)}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import base64
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import zipfile
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CATALOG = ROOT / "runtime-dump" / "book-catalog.json"
|
||||
ZIP_DIR = ROOT / "runtime-dump" / "remote-zips"
|
||||
DB_DIR = ROOT / "runtime-dump" / "databases"
|
||||
REPORT_CSV = ROOT / "runtime-dump" / "wordbook-download-report.csv"
|
||||
|
||||
BASE_STATIC_URLS = [
|
||||
"http://static.beingfine.cn",
|
||||
"https://hwstatic.beingfine.cn",
|
||||
"https://ccastatic.beingfine.cn",
|
||||
]
|
||||
REFERER = "https://app.bbdc.cn"
|
||||
USER_AGENT = "LangEasyLexis/5.11.1"
|
||||
|
||||
|
||||
def load_books(path: Path) -> list[dict]:
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, dict) and isinstance(data.get("books"), list):
|
||||
return data["books"]
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
raise ValueError(f"Unsupported catalog shape: {path}")
|
||||
|
||||
|
||||
def book_url(book: dict, base_url: str) -> str:
|
||||
code = book["code"]
|
||||
version = str(book.get("version") or 0)
|
||||
db_name = f"LangEasyLexisV3_{code}.db"
|
||||
return f"{base_url}/dicts/v3/{code}/{version}/{db_name}.zip"
|
||||
|
||||
|
||||
def request_url(url: str):
|
||||
req = Request(
|
||||
url,
|
||||
headers={
|
||||
"Referer": REFERER,
|
||||
"User-Agent": USER_AGENT,
|
||||
},
|
||||
)
|
||||
return urlopen(req, timeout=45)
|
||||
|
||||
|
||||
def md5_b64(path: Path) -> str:
|
||||
h = hashlib.md5()
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return base64.b64encode(h.digest()).decode("ascii")
|
||||
|
||||
|
||||
def download_one(book: dict, force: bool = False) -> dict:
|
||||
code = book["code"]
|
||||
if book.get("url") == "*":
|
||||
return {"code": code, "status": "skip-virtual-book", "url": "", "bytes": 0}
|
||||
|
||||
zip_path = ZIP_DIR / f"LangEasyLexisV3_{code}.db.zip"
|
||||
db_path = DB_DIR / f"LangEasyLexisV3_{code}.db"
|
||||
|
||||
if db_path.exists() and not force:
|
||||
return {"code": code, "status": "skip-db-exists", "url": "", "bytes": db_path.stat().st_size}
|
||||
|
||||
last_error = ""
|
||||
for base in BASE_STATIC_URLS:
|
||||
url = book_url(book, base)
|
||||
try:
|
||||
with request_url(url) as resp:
|
||||
status = getattr(resp, "status", 200)
|
||||
if status != 200:
|
||||
last_error = f"http-{status}"
|
||||
continue
|
||||
|
||||
ZIP_DIR.mkdir(parents=True, exist_ok=True)
|
||||
tmp_path = zip_path.with_suffix(zip_path.suffix + ".tmp")
|
||||
with tmp_path.open("wb") as out:
|
||||
while True:
|
||||
chunk = resp.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
out.write(chunk)
|
||||
|
||||
content_md5 = resp.headers.get("Content-MD5")
|
||||
if content_md5 and md5_b64(tmp_path) != content_md5:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
last_error = "md5-mismatch"
|
||||
continue
|
||||
|
||||
tmp_path.replace(zip_path)
|
||||
DB_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
names = zf.namelist()
|
||||
wanted = f"LangEasyLexisV3_{code}.db"
|
||||
member = wanted if wanted in names else names[0]
|
||||
zf.extract(member, DB_DIR)
|
||||
extracted = DB_DIR / member
|
||||
if extracted != db_path:
|
||||
extracted.replace(db_path)
|
||||
|
||||
return {"code": code, "status": "downloaded", "url": url, "bytes": db_path.stat().st_size}
|
||||
except HTTPError as e:
|
||||
last_error = f"http-{e.code}"
|
||||
except (URLError, TimeoutError, zipfile.BadZipFile, OSError) as e:
|
||||
last_error = type(e).__name__
|
||||
|
||||
return {"code": code, "status": f"failed:{last_error}", "url": "", "bytes": 0}
|
||||
|
||||
|
||||
def write_report(rows: list[dict]) -> None:
|
||||
REPORT_CSV.parent.mkdir(parents=True, exist_ok=True)
|
||||
with REPORT_CSV.open("w", newline="", encoding="utf-8-sig") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=["code", "status", "bytes", "url"])
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Download all Lexis V3 wordbook DB zips from the app's CDN interface.")
|
||||
parser.add_argument("--catalog", type=Path, default=CATALOG)
|
||||
parser.add_argument("--force", action="store_true", help="re-download books even if the extracted DB already exists")
|
||||
parser.add_argument("--limit", type=int, default=0, help="download at most N books, useful for smoke tests")
|
||||
parser.add_argument("--sleep", type=float, default=0.15, help="seconds to sleep between downloads in single-worker mode")
|
||||
parser.add_argument("--workers", type=int, default=1, help="parallel downloads")
|
||||
parser.add_argument("--include-existing", action="store_true", help="include already cached local DBs in the run")
|
||||
args = parser.parse_args()
|
||||
|
||||
books = load_books(args.catalog)
|
||||
if not args.include_existing and not args.force:
|
||||
books = [b for b in books if not (DB_DIR / f"LangEasyLexisV3_{b['code']}.db").exists()]
|
||||
if args.limit:
|
||||
books = books[: args.limit]
|
||||
|
||||
total = len(books)
|
||||
results = []
|
||||
done = 0
|
||||
workers = max(1, args.workers)
|
||||
if workers == 1:
|
||||
for book in books:
|
||||
result = download_one(book, force=args.force)
|
||||
results.append(result)
|
||||
done += 1
|
||||
print(f"[{done}/{total}] {result['code']} {result['status']} {result['bytes']}", flush=True)
|
||||
if args.sleep:
|
||||
time.sleep(args.sleep)
|
||||
else:
|
||||
with ThreadPoolExecutor(max_workers=workers) as executor:
|
||||
futures = [executor.submit(download_one, book, args.force) for book in books]
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
results.append(result)
|
||||
done += 1
|
||||
print(f"[{done}/{total}] {result['code']} {result['status']} {result['bytes']}", flush=True)
|
||||
|
||||
write_report(results)
|
||||
ok = sum(1 for r in results if r["status"] in {"downloaded", "skip-db-exists", "skip-virtual-book"})
|
||||
failed = len(results) - ok
|
||||
print(f"done ok={ok} failed={failed} report={REPORT_CSV}", flush=True)
|
||||
return 0 if failed == 0 else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,324 @@
|
||||
#!/usr/bin/env python3
|
||||
"""批量下载所有 zpk 文件。
|
||||
|
||||
来源:unique-zpk-index.csv 中的唯一 zpk 名(115104 个)。
|
||||
CDN:{base}/r/{zpkName}.zpk,Referer 同词书下载。
|
||||
落地:runtime-dump/zpk-cache/0/{zpkName}.zpk(与设备目录结构一致)。
|
||||
已存在的文件自动跳过,可断点续传。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import csv
|
||||
import hashlib
|
||||
import http.client
|
||||
import queue
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
INDEX_CSV = ROOT / "runtime-dump" / "unique-zpk-index.csv"
|
||||
ZPK_DIR = ROOT / "runtime-dump" / "zpk-cache" / "0"
|
||||
REPORT_CSV = ROOT / "runtime-dump" / "zpk-download-report.csv"
|
||||
|
||||
BASE_STATIC_URLS = [
|
||||
"http://static.beingfine.cn",
|
||||
"https://hwstatic.beingfine.cn",
|
||||
"https://ccastatic.beingfine.cn",
|
||||
]
|
||||
REFERER = "https://app.bbdc.cn"
|
||||
USER_AGENT = "LangEasyLexis/5.11.1"
|
||||
REQUEST_TIMEOUT = 8.0
|
||||
STOP_EVENT = threading.Event()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CdnEndpoint:
|
||||
base_url: str
|
||||
scheme: str
|
||||
host: str
|
||||
port: int | None
|
||||
|
||||
@classmethod
|
||||
def parse(cls, base_url: str) -> "CdnEndpoint":
|
||||
parsed = urlparse(base_url)
|
||||
return cls(base_url=base_url, scheme=parsed.scheme, host=parsed.hostname or "", port=parsed.port)
|
||||
|
||||
|
||||
class CdnClient:
|
||||
def __init__(self, endpoints: list[CdnEndpoint], timeout: float) -> None:
|
||||
self.endpoints = endpoints
|
||||
self.timeout = timeout
|
||||
self.connections: dict[str, http.client.HTTPConnection] = {}
|
||||
|
||||
def close(self) -> None:
|
||||
for conn in self.connections.values():
|
||||
conn.close()
|
||||
self.connections.clear()
|
||||
|
||||
def _conn(self, endpoint: CdnEndpoint) -> http.client.HTTPConnection:
|
||||
key = endpoint.base_url
|
||||
conn = self.connections.get(key)
|
||||
if conn is not None:
|
||||
return conn
|
||||
conn_cls = http.client.HTTPSConnection if endpoint.scheme == "https" else http.client.HTTPConnection
|
||||
conn = conn_cls(endpoint.host, endpoint.port, timeout=self.timeout)
|
||||
self.connections[key] = conn
|
||||
return conn
|
||||
|
||||
def get(self, endpoint: CdnEndpoint, name: str) -> tuple[str, http.client.HTTPResponse]:
|
||||
url_path = f"/r/{name}.zpk"
|
||||
headers = {
|
||||
"Connection": "keep-alive",
|
||||
"Referer": REFERER,
|
||||
"User-Agent": USER_AGENT,
|
||||
}
|
||||
conn = self._conn(endpoint)
|
||||
try:
|
||||
conn.request("GET", url_path, headers=headers)
|
||||
return f"{endpoint.base_url}{url_path}", conn.getresponse()
|
||||
except (http.client.HTTPException, OSError):
|
||||
conn.close()
|
||||
self.connections.pop(endpoint.base_url, None)
|
||||
raise
|
||||
|
||||
def drop(self, endpoint: CdnEndpoint) -> None:
|
||||
conn = self.connections.pop(endpoint.base_url, None)
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
|
||||
|
||||
def build_endpoints(cdn_mode: str, worker_index: int = 0) -> list[CdnEndpoint]:
|
||||
base_urls = BASE_STATIC_URLS
|
||||
if cdn_mode == "spread" and base_urls:
|
||||
offset = worker_index % len(base_urls)
|
||||
base_urls = base_urls[offset:] + base_urls[:offset]
|
||||
return [CdnEndpoint.parse(base) for base in base_urls]
|
||||
|
||||
|
||||
def load_names(path: Path) -> list[str]:
|
||||
names = []
|
||||
seen = set()
|
||||
with path.open("r", encoding="utf-8-sig", newline="") as f:
|
||||
for row in csv.DictReader(f):
|
||||
name = row.get("zpk")
|
||||
if name and name not in seen:
|
||||
seen.add(name)
|
||||
names.append(name)
|
||||
return names
|
||||
|
||||
|
||||
def read_and_save_response(resp: http.client.HTTPResponse, dest: Path) -> int:
|
||||
tmp = dest.with_name(f"{dest.name}.{threading.get_ident()}.tmp")
|
||||
h = hashlib.md5()
|
||||
size = 0
|
||||
content_md5 = resp.getheader("Content-MD5")
|
||||
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tmp.open("wb") as out:
|
||||
while not STOP_EVENT.is_set():
|
||||
chunk = resp.read(256 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
out.write(chunk)
|
||||
h.update(chunk)
|
||||
size += len(chunk)
|
||||
|
||||
if STOP_EVENT.is_set():
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise InterruptedError
|
||||
|
||||
if content_md5 and base64.b64encode(h.digest()).decode("ascii") != content_md5:
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise ValueError("md5-mismatch")
|
||||
|
||||
tmp.replace(dest)
|
||||
return size
|
||||
|
||||
|
||||
def download_one(name: str, zpk_dir: Path, client: CdnClient) -> dict:
|
||||
dest = zpk_dir / f"{name}.zpk"
|
||||
if dest.exists() and dest.stat().st_size > 0:
|
||||
return {"name": name, "status": "skip-exists", "bytes": dest.stat().st_size, "url": ""}
|
||||
|
||||
last_error = ""
|
||||
for endpoint in client.endpoints:
|
||||
if STOP_EVENT.is_set():
|
||||
return {"name": name, "status": "cancelled", "bytes": 0, "url": ""}
|
||||
|
||||
try:
|
||||
url, resp = client.get(endpoint, name)
|
||||
try:
|
||||
if resp.status != 200:
|
||||
last_error = f"http-{resp.status}"
|
||||
resp.read()
|
||||
if resp.status >= 500:
|
||||
client.drop(endpoint)
|
||||
continue
|
||||
size = read_and_save_response(resp, dest)
|
||||
return {"name": name, "status": "downloaded", "bytes": size, "url": url}
|
||||
finally:
|
||||
resp.close()
|
||||
except InterruptedError:
|
||||
return {"name": name, "status": "cancelled", "bytes": 0, "url": ""}
|
||||
except ValueError as e:
|
||||
last_error = str(e)
|
||||
client.drop(endpoint)
|
||||
except (http.client.HTTPException, OSError, TimeoutError) as e:
|
||||
last_error = type(e).__name__
|
||||
client.drop(endpoint)
|
||||
|
||||
return {"name": name, "status": f"failed:{last_error}", "bytes": 0, "url": ""}
|
||||
|
||||
|
||||
def progress_line(done: int, total: int, result: dict, started_at: float, bytes_done: int) -> str:
|
||||
elapsed = max(0.001, time.monotonic() - started_at)
|
||||
files_per_sec = done / elapsed
|
||||
mib_per_sec = bytes_done / 1024 / 1024 / elapsed
|
||||
return (
|
||||
f"[{done}/{total}] {result['name']} {result['status']} {result['bytes']} "
|
||||
f"rate={files_per_sec:.1f}/s {mib_per_sec:.2f}MiB/s"
|
||||
)
|
||||
|
||||
|
||||
def run_parallel(names: list[str], zpk_dir: Path, workers: int, progress_every: int, cdn_mode: str) -> list[dict]:
|
||||
work: queue.Queue[str | None] = queue.Queue(maxsize=max(1, workers * 4))
|
||||
results: list[dict] = []
|
||||
results_lock = threading.Lock()
|
||||
started_at = time.monotonic()
|
||||
done = 0
|
||||
bytes_done = 0
|
||||
total = len(names)
|
||||
|
||||
def record(result: dict) -> None:
|
||||
nonlocal done, bytes_done
|
||||
with results_lock:
|
||||
results.append(result)
|
||||
done += 1
|
||||
if result["status"] in {"downloaded", "skip-exists"}:
|
||||
bytes_done += int(result["bytes"])
|
||||
if done % progress_every == 0 or done == total or STOP_EVENT.is_set():
|
||||
print(progress_line(done, total, result, started_at, bytes_done), flush=True)
|
||||
|
||||
def worker(worker_index: int) -> None:
|
||||
client = CdnClient(build_endpoints(cdn_mode, worker_index), REQUEST_TIMEOUT)
|
||||
try:
|
||||
while not STOP_EVENT.is_set():
|
||||
try:
|
||||
name = work.get(timeout=0.25)
|
||||
except queue.Empty:
|
||||
continue
|
||||
try:
|
||||
if name is None:
|
||||
return
|
||||
record(download_one(name, zpk_dir, client))
|
||||
finally:
|
||||
work.task_done()
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(workers)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
|
||||
try:
|
||||
for name in names:
|
||||
while not STOP_EVENT.is_set():
|
||||
try:
|
||||
work.put(name, timeout=0.25)
|
||||
break
|
||||
except queue.Full:
|
||||
continue
|
||||
if STOP_EVENT.is_set():
|
||||
break
|
||||
for _ in threads:
|
||||
work.put(None)
|
||||
work.join()
|
||||
except KeyboardInterrupt:
|
||||
STOP_EVENT.set()
|
||||
print(f"\ninterrupted done={done}/{total}; writing partial report...", flush=True)
|
||||
finally:
|
||||
for thread in threads:
|
||||
thread.join(timeout=REQUEST_TIMEOUT + 1)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def write_report(rows: list[dict], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", newline="", encoding="utf-8-sig") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=["name", "status", "bytes", "url"])
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
global REQUEST_TIMEOUT
|
||||
parser = argparse.ArgumentParser(description="批量下载所有 zpk 文件。")
|
||||
parser.add_argument("--index", type=Path, default=INDEX_CSV)
|
||||
parser.add_argument("--out-dir", type=Path, default=ZPK_DIR)
|
||||
parser.add_argument("--report", type=Path, default=REPORT_CSV)
|
||||
parser.add_argument("--workers", type=int, default=64)
|
||||
parser.add_argument("--timeout", type=float, default=8, help="单个 CDN 请求超时秒数;Ctrl+C 最多等待活跃请求到此超时")
|
||||
parser.add_argument("--progress-every", type=int, default=500)
|
||||
parser.add_argument(
|
||||
"--cdn-mode",
|
||||
choices=("spread", "primary"),
|
||||
default="primary",
|
||||
help="primary 会始终优先第一个 CDN;spread 会把 worker 分散到 3 个 CDN",
|
||||
)
|
||||
parser.add_argument("--limit", type=int, default=0, help="最多下载 N 个(冒烟测试)")
|
||||
parser.add_argument("--force", action="store_true", help="已存在也重新下载")
|
||||
args = parser.parse_args()
|
||||
|
||||
REQUEST_TIMEOUT = max(1.0, args.timeout)
|
||||
zpk_dir = args.out_dir
|
||||
|
||||
names = load_names(args.index)
|
||||
if not args.force:
|
||||
names = [n for n in names if not (zpk_dir / f"{n}.zpk").exists()]
|
||||
if args.limit:
|
||||
names = names[: args.limit]
|
||||
|
||||
total = len(names)
|
||||
print(f"total_to_download={total} out_dir={zpk_dir}", flush=True)
|
||||
|
||||
workers = max(1, args.workers)
|
||||
if workers == 1:
|
||||
endpoint_client = CdnClient(build_endpoints(args.cdn_mode), REQUEST_TIMEOUT)
|
||||
results = []
|
||||
started_at = time.monotonic()
|
||||
bytes_done = 0
|
||||
try:
|
||||
for done, name in enumerate(names, 1):
|
||||
r = download_one(name, zpk_dir, endpoint_client)
|
||||
results.append(r)
|
||||
if r["status"] in {"downloaded", "skip-exists"}:
|
||||
bytes_done += int(r["bytes"])
|
||||
print(progress_line(done, total, r, started_at, bytes_done), flush=True)
|
||||
except KeyboardInterrupt:
|
||||
STOP_EVENT.set()
|
||||
print(f"\ninterrupted done={len(results)}/{total}; writing partial report...", flush=True)
|
||||
finally:
|
||||
endpoint_client.close()
|
||||
else:
|
||||
results = run_parallel(names, zpk_dir, workers, max(1, args.progress_every), args.cdn_mode)
|
||||
|
||||
write_report(results, args.report)
|
||||
ok = sum(1 for r in results if r["status"] in {"downloaded", "skip-exists"})
|
||||
failed = len(results) - ok
|
||||
cancelled = sum(1 for r in results if r["status"] == "cancelled")
|
||||
print(f"done ok={ok} failed={failed} cancelled={cancelled} report={args.report}", flush=True)
|
||||
if STOP_EVENT.is_set():
|
||||
return 130
|
||||
return 0 if failed == 0 else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Expand each exported word zpk name into structured index fields."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_WORDS = ROOT / "runtime-dump" / "all-wordbook-words.csv"
|
||||
DEFAULT_OUT = ROOT / "runtime-dump" / "all-wordbook-zpk-index.csv"
|
||||
DEFAULT_UNIQUE_OUT = ROOT / "runtime-dump" / "unique-zpk-index.csv"
|
||||
|
||||
OUT_FIELDS = (
|
||||
"book_id",
|
||||
"book_code",
|
||||
"book_name",
|
||||
"word",
|
||||
"word_id",
|
||||
"zpk",
|
||||
"zpk_bid",
|
||||
"zpk_word_id",
|
||||
"zpk_version",
|
||||
"zpk_updated_unix",
|
||||
"zpk_updated_utc",
|
||||
)
|
||||
|
||||
UNIQUE_FIELDS = (
|
||||
"zpk",
|
||||
"zpk_bid",
|
||||
"zpk_word_id",
|
||||
"zpk_version",
|
||||
"zpk_updated_unix",
|
||||
"zpk_updated_utc",
|
||||
"first_word",
|
||||
"book_occurrences",
|
||||
)
|
||||
|
||||
|
||||
def parse_zpk(zpk: str) -> dict[str, str]:
|
||||
parts = (zpk or "").split("_")
|
||||
if len(parts) != 4 or not all(part.isdigit() for part in parts):
|
||||
return {
|
||||
"zpk_bid": "",
|
||||
"zpk_word_id": "",
|
||||
"zpk_version": "",
|
||||
"zpk_updated_unix": "",
|
||||
"zpk_updated_utc": "",
|
||||
}
|
||||
|
||||
updated = int(parts[3])
|
||||
try:
|
||||
updated_utc = datetime.fromtimestamp(updated, tz=timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
except (OverflowError, OSError, ValueError):
|
||||
updated_utc = ""
|
||||
|
||||
return {
|
||||
"zpk_bid": parts[0],
|
||||
"zpk_word_id": parts[1],
|
||||
"zpk_version": parts[2],
|
||||
"zpk_updated_unix": parts[3],
|
||||
"zpk_updated_utc": updated_utc,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--words", type=Path, default=DEFAULT_WORDS)
|
||||
parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
|
||||
parser.add_argument("--unique-out", type=Path, default=DEFAULT_UNIQUE_OUT)
|
||||
args = parser.parse_args()
|
||||
|
||||
unique: dict[str, dict[str, str | int]] = {}
|
||||
total = 0
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
with args.words.open("r", encoding="utf-8-sig", newline="") as in_fh, args.out.open(
|
||||
"w", encoding="utf-8-sig", newline=""
|
||||
) as out_fh:
|
||||
reader = csv.DictReader(in_fh)
|
||||
writer = csv.DictWriter(out_fh, fieldnames=OUT_FIELDS)
|
||||
writer.writeheader()
|
||||
for row in reader:
|
||||
zpk = row.get("zpk", "")
|
||||
parsed = parse_zpk(zpk)
|
||||
out_row = {
|
||||
"book_id": row.get("book_id", ""),
|
||||
"book_code": row.get("book_code", ""),
|
||||
"book_name": row.get("book_name", ""),
|
||||
"word": row.get("word", ""),
|
||||
"word_id": row.get("id", ""),
|
||||
"zpk": zpk,
|
||||
**parsed,
|
||||
}
|
||||
writer.writerow(out_row)
|
||||
total += 1
|
||||
|
||||
if zpk not in unique:
|
||||
unique[zpk] = {
|
||||
"zpk": zpk,
|
||||
**parsed,
|
||||
"first_word": row.get("word", ""),
|
||||
"book_occurrences": 0,
|
||||
}
|
||||
unique[zpk]["book_occurrences"] = int(unique[zpk]["book_occurrences"]) + 1
|
||||
|
||||
with args.unique_out.open("w", encoding="utf-8-sig", newline="") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=UNIQUE_FIELDS)
|
||||
writer.writeheader()
|
||||
writer.writerows(unique.values())
|
||||
|
||||
mismatches = sum(1 for zpk, item in unique.items() if item["zpk_word_id"] and item["zpk_word_id"] != "")
|
||||
print(f"rows={total} unique_zpk={len(unique)} parsed_unique={mismatches}")
|
||||
print(f"out={args.out}")
|
||||
print(f"unique_out={args.unique_out}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,282 @@
|
||||
"""Export words from every downloaded LangEasy Lexis V3 word-book DB."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_CATALOG = ROOT / "runtime-dump" / "book-catalog.json"
|
||||
DEFAULT_DB_DIR = ROOT / "runtime-dump" / "databases"
|
||||
DEFAULT_DECRYPTED_DIR = ROOT / "runtime-dump" / "decrypted-databases"
|
||||
DEFAULT_PER_BOOK_DIR = ROOT / "runtime-dump" / "wordbook-words"
|
||||
DEFAULT_COMBINED_CSV = ROOT / "runtime-dump" / "all-wordbook-words.csv"
|
||||
DEFAULT_SUMMARY_CSV = ROOT / "runtime-dump" / "wordbook-word-summary.csv"
|
||||
DECRYPTOR_PATH = ROOT / "tools" / "decrypt-sqlcipher2-db.py"
|
||||
|
||||
WORD_FIELDS = (
|
||||
"book_id",
|
||||
"book_code",
|
||||
"book_name",
|
||||
"book_count",
|
||||
"book_exam",
|
||||
"book_version",
|
||||
"id",
|
||||
"word",
|
||||
"main_word",
|
||||
"seq_book",
|
||||
"seq_az",
|
||||
"seq_freq",
|
||||
"seq_rand",
|
||||
"zpk",
|
||||
)
|
||||
|
||||
SUMMARY_FIELDS = (
|
||||
"book_id",
|
||||
"book_code",
|
||||
"book_name",
|
||||
"book_count",
|
||||
"exported_count",
|
||||
"status",
|
||||
"table",
|
||||
"local_db",
|
||||
"decrypted_db",
|
||||
"export_csv",
|
||||
"error",
|
||||
)
|
||||
|
||||
|
||||
def load_decrypt_db():
|
||||
spec = importlib.util.spec_from_file_location("decrypt_sqlcipher2_db", DECRYPTOR_PATH)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError(f"Cannot load decryptor: {DECRYPTOR_PATH}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module.decrypt_db
|
||||
|
||||
|
||||
def load_books(path: Path) -> list[dict]:
|
||||
with path.open("r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
if isinstance(data, dict) and isinstance(data.get("books"), list):
|
||||
return data["books"]
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
raise ValueError(f"Unsupported catalog shape: {path}")
|
||||
|
||||
|
||||
def safe_code_filename(code: object) -> str:
|
||||
text = str(code or "UNKNOWN")
|
||||
return re.sub(r"[^A-Za-z0-9_.-]+", "_", text).strip("._") or "UNKNOWN"
|
||||
|
||||
|
||||
def quote_identifier(name: str) -> str:
|
||||
return '"' + name.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def list_tables(conn: sqlite3.Connection) -> list[str]:
|
||||
return [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"select name from sqlite_master where type='table' and name not like 'sqlite_%' order by name"
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def table_columns(conn: sqlite3.Connection, table: str) -> set[str]:
|
||||
return {row[1] for row in conn.execute(f"pragma table_info({quote_identifier(table)})")}
|
||||
|
||||
|
||||
def choose_word_table(conn: sqlite3.Connection, code: str) -> str:
|
||||
tables = list_tables(conn)
|
||||
preferred = f"LangEasyLexisV3_{code}"
|
||||
if preferred in tables:
|
||||
return preferred
|
||||
|
||||
for table in tables:
|
||||
cols = table_columns(conn, table)
|
||||
if {"id", "word"}.issubset(cols):
|
||||
return table
|
||||
|
||||
raise RuntimeError(f"no word table found; tables={tables}")
|
||||
|
||||
|
||||
def ensure_decrypted(src: Path, dst: Path, decrypt_db, force: bool) -> None:
|
||||
if (
|
||||
not force
|
||||
and dst.exists()
|
||||
and dst.stat().st_size > 0
|
||||
and dst.stat().st_mtime >= src.stat().st_mtime
|
||||
):
|
||||
return
|
||||
decrypt_db(src, dst)
|
||||
|
||||
|
||||
def iter_words(conn: sqlite3.Connection, table: str) -> Iterable[sqlite3.Row]:
|
||||
conn.row_factory = sqlite3.Row
|
||||
cols = table_columns(conn, table)
|
||||
required = {"id", "word", "main_word", "seq_book", "seq_az", "seq_freq", "seq_rand", "zpk"}
|
||||
missing = sorted(required - cols)
|
||||
if missing:
|
||||
raise RuntimeError(f"table {table} missing columns: {missing}")
|
||||
|
||||
query = (
|
||||
f"select id, word, main_word, seq_book, seq_az, seq_freq, seq_rand, zpk "
|
||||
f"from {quote_identifier(table)} order by seq_book, id"
|
||||
)
|
||||
return conn.execute(query)
|
||||
|
||||
|
||||
def export_one(
|
||||
row: dict,
|
||||
db_dir: Path,
|
||||
decrypted_dir: Path,
|
||||
per_book_dir: Path,
|
||||
decrypt_db,
|
||||
force_decrypt: bool,
|
||||
) -> dict:
|
||||
code = str(row.get("code") or "")
|
||||
src = db_dir / f"LangEasyLexisV3_{code}.db"
|
||||
if not src.exists():
|
||||
return summary(row, "missing_db", "", src, Path(), Path(), 0, f"not found: {src}")
|
||||
|
||||
db_name = src.with_suffix(".sqlite").name
|
||||
decrypted = decrypted_dir / db_name
|
||||
export_csv = per_book_dir / f"{safe_code_filename(code)}.csv"
|
||||
|
||||
ensure_decrypted(src, decrypted, decrypt_db, force_decrypt)
|
||||
per_book_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with sqlite3.connect(decrypted) as conn:
|
||||
table = choose_word_table(conn, code)
|
||||
count = 0
|
||||
with export_csv.open("w", encoding="utf-8-sig", newline="") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=WORD_FIELDS)
|
||||
writer.writeheader()
|
||||
for word_row in iter_words(conn, table):
|
||||
writer.writerow(
|
||||
{
|
||||
"book_id": row.get("id", ""),
|
||||
"book_code": code,
|
||||
"book_name": row.get("name", ""),
|
||||
"book_count": row.get("count", ""),
|
||||
"book_exam": row.get("exam", ""),
|
||||
"book_version": row.get("version", ""),
|
||||
"id": word_row["id"],
|
||||
"word": word_row["word"],
|
||||
"main_word": word_row["main_word"],
|
||||
"seq_book": word_row["seq_book"],
|
||||
"seq_az": word_row["seq_az"],
|
||||
"seq_freq": word_row["seq_freq"],
|
||||
"seq_rand": word_row["seq_rand"],
|
||||
"zpk": word_row["zpk"],
|
||||
}
|
||||
)
|
||||
count += 1
|
||||
|
||||
return summary(row, "ok", table, src, decrypted, export_csv, count, "")
|
||||
|
||||
|
||||
def summary(
|
||||
row: dict,
|
||||
status: str,
|
||||
table: str,
|
||||
src: Path,
|
||||
decrypted: Path,
|
||||
export_csv: Path,
|
||||
exported_count: int,
|
||||
error: str,
|
||||
) -> dict:
|
||||
return {
|
||||
"book_id": row.get("id", ""),
|
||||
"book_code": row.get("code", ""),
|
||||
"book_name": row.get("name", ""),
|
||||
"book_count": row.get("count", ""),
|
||||
"exported_count": exported_count,
|
||||
"status": status,
|
||||
"table": table,
|
||||
"local_db": str(src) if src else "",
|
||||
"decrypted_db": str(decrypted) if decrypted else "",
|
||||
"export_csv": str(export_csv) if export_csv else "",
|
||||
"error": error,
|
||||
}
|
||||
|
||||
|
||||
def write_summary(rows: list[dict], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=SUMMARY_FIELDS)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def write_combined(summary_rows: list[dict], path: Path) -> int:
|
||||
total = 0
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as out_fh:
|
||||
writer = csv.DictWriter(out_fh, fieldnames=WORD_FIELDS)
|
||||
writer.writeheader()
|
||||
for item in summary_rows:
|
||||
if item["status"] != "ok":
|
||||
continue
|
||||
with Path(item["export_csv"]).open("r", encoding="utf-8-sig", newline="") as in_fh:
|
||||
reader = csv.DictReader(in_fh)
|
||||
for row in reader:
|
||||
writer.writerow(row)
|
||||
total += 1
|
||||
return total
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
|
||||
parser.add_argument("--db-dir", type=Path, default=DEFAULT_DB_DIR)
|
||||
parser.add_argument("--decrypted-dir", type=Path, default=DEFAULT_DECRYPTED_DIR)
|
||||
parser.add_argument("--per-book-dir", type=Path, default=DEFAULT_PER_BOOK_DIR)
|
||||
parser.add_argument("--combined-csv", type=Path, default=DEFAULT_COMBINED_CSV)
|
||||
parser.add_argument("--summary-csv", type=Path, default=DEFAULT_SUMMARY_CSV)
|
||||
parser.add_argument("--limit", type=int, default=0)
|
||||
parser.add_argument("--force-decrypt", action="store_true")
|
||||
parser.add_argument("--skip-combined", action="store_true")
|
||||
parser.add_argument("--progress-every", type=int, default=25)
|
||||
args = parser.parse_args()
|
||||
|
||||
decrypt_db = load_decrypt_db()
|
||||
books = [row for row in load_books(args.catalog) if row.get("url") != "*"]
|
||||
if args.limit:
|
||||
books = books[: args.limit]
|
||||
|
||||
summaries = []
|
||||
for index, row in enumerate(books, start=1):
|
||||
try:
|
||||
item = export_one(row, args.db_dir, args.decrypted_dir, args.per_book_dir, decrypt_db, args.force_decrypt)
|
||||
except Exception as exc: # Keep the batch moving and surface failures in the summary.
|
||||
item = summary(row, "error", "", args.db_dir / f"LangEasyLexisV3_{row.get('code')}.db", Path(), Path(), 0, repr(exc))
|
||||
summaries.append(item)
|
||||
if args.progress_every and (index % args.progress_every == 0 or index == len(books)):
|
||||
ok = sum(1 for entry in summaries if entry["status"] == "ok")
|
||||
print(f"progress={index}/{len(books)} ok={ok}")
|
||||
|
||||
write_summary(summaries, args.summary_csv)
|
||||
combined_total = 0 if args.skip_combined else write_combined(summaries, args.combined_csv)
|
||||
ok_count = sum(1 for item in summaries if item["status"] == "ok")
|
||||
failed_count = len(summaries) - ok_count
|
||||
exported_total = sum(int(item["exported_count"] or 0) for item in summaries)
|
||||
|
||||
print(f"books={len(summaries)} ok={ok_count} failed={failed_count} words={exported_total}")
|
||||
print(f"summary={args.summary_csv}")
|
||||
print(f"per_book_dir={args.per_book_dir}")
|
||||
if not args.skip_combined:
|
||||
print(f"combined={args.combined_csv} combined_words={combined_total}")
|
||||
return 0 if failed_count == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,69 @@
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
|
||||
PREF = sys.argv[1] if len(sys.argv) > 1 else "runtime-dump/shared_prefs/User_Preference_344124367.xml"
|
||||
OUT = sys.argv[2] if len(sys.argv) > 2 else "runtime-dump/book-catalog.json"
|
||||
FIELDS = ("id", "code", "name", "count", "exam", "url", "version", "bookVersion", "purchased", "forSale")
|
||||
|
||||
|
||||
def get_pref_map(path):
|
||||
result = {}
|
||||
root = ET.parse(path).getroot()
|
||||
for el in root:
|
||||
name = el.attrib.get("name")
|
||||
if not name:
|
||||
continue
|
||||
result[name] = html.unescape(el.attrib.get("value", el.text or "")).strip()
|
||||
return result
|
||||
|
||||
|
||||
def compact_book(book):
|
||||
return {key: book.get(key) for key in FIELDS if key in book}
|
||||
|
||||
|
||||
def walk_books(value):
|
||||
if isinstance(value, dict):
|
||||
if "code" in value and "url" in value:
|
||||
yield value
|
||||
for child in value.values():
|
||||
yield from walk_books(child)
|
||||
elif isinstance(value, list):
|
||||
for child in value:
|
||||
yield from walk_books(child)
|
||||
|
||||
|
||||
def main():
|
||||
prefs = get_pref_map(PREF)
|
||||
found = {}
|
||||
for key in ("key_group_book_List_V3", "lexis_matched_books", "library_learning"):
|
||||
raw = prefs.get(key)
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
for book in walk_books(value):
|
||||
code = book.get("code")
|
||||
if code and code not in found:
|
||||
found[code] = compact_book(book)
|
||||
|
||||
catalog = sorted(found.values(), key=lambda item: (str(item.get("exam", "")), str(item.get("code", ""))))
|
||||
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
||||
with open(OUT, "w", encoding="utf-8") as f:
|
||||
json.dump(catalog, f, ensure_ascii=False, indent=2)
|
||||
f.write("\n")
|
||||
print(f"wrote {len(catalog)} books -> {OUT}")
|
||||
for item in catalog[:30]:
|
||||
print(json.dumps(item, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user