diff --git a/importer/import_lexis.py b/importer/import_lexis.py new file mode 100644 index 0000000..cef4a28 --- /dev/null +++ b/importer/import_lexis.py @@ -0,0 +1,1955 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Lexis 词库一键建表 + 导入脚本 + +用法(默认 SQLite,无需任何服务): + python import_lexis.py --recreate + python import_lexis.py --sqlite-path D:/data/lexis.db --recreate + +PostgreSQL: + pip install psycopg2-binary + python import_lexis.py --db postgres --dsn "postgresql://user:pass@host:5432/lexis" --recreate + +MySQL: + pip install pymysql + python import_lexis.py --db mysql --dsn "mysql://user:pass@host:3306/lexis" --recreate + +只导入解密数据,自动跳过 wordbooks/original 与 zpk/original。 +""" + +from __future__ import annotations + +import argparse +import csv +import glob +import json +import os +import re +import sqlite3 +import sys +import time +from collections import defaultdict +from urllib.parse import urlparse, unquote + +# --------------------------------------------------------------------------- +# 表定义 +# --------------------------------------------------------------------------- + +TABLES = [ + "exam", "book", "book_phrase_book", "word_entry", "book_entry", + "word_spelling_variant", "word_sense", "sense_object", "sense_exam", + "sense_relation", "entry_exam_meaning_order", "phrase", "phrase_exam", + "phrase_exam_order", "example_group", "example_group_exam", + "example_usage", "example_sentence", "entry_exam_sentence", + "confusable_word", "morpheme", "card", "card_exam", "word_family", + "word_family_exam", "word_family_exam_order", "word_family_variant", + "word_family_variant_member", "word_family_mini_group", + "word_family_mini_group_member", "card_family_link", "synonym_word", + "synonym_sense", "synonym_group", "synonym_group_member", "synonym_antonym", +] + +COLUMNS = { + "exam": ("exam_id", "exam_code", "exam_name_zh", "is_grade_usable", "is_pseudo"), + "book": ("book_id", "book_code", "book_name", "description", "exam_id", + "declared_entry_count", "cover_url", "package_url", "content_version", + "schema_version", "sort_by", "sort_at", "is_for_sale", "is_purchased", + "is_opt", "source_code", "catalog_type", "labels", "publish_log", + "pkg_page", "external_link", "has_wordbook_file"), + "book_phrase_book": ("book_id", "phrase_book_id", "position"), + "word_entry": ("entry_id", "headword", "index_word", "definition_text", + "us_pronunciation", "uk_pronunciation", "data_version", + "book_exam_id", "occurrence_count", "has_listening_examples", + "source_zpk_key", "source_variant_code", "source_timestamp"), + "book_entry": ("book_entry_id", "book_id", "entry_id", "display_word", + "headword_text", "seq_book", "seq_az", "seq_freq", "seq_rand", + "source_zpk_key"), + "word_spelling_variant": ("entry_id", "position", "variant"), + "word_sense": ("sense_id", "entry_id", "sense_no", "part_of_speech", + "gloss_zh", "secondary_exam_codes", "flag_h"), + "sense_object": ("meaning_id", "entry_id", "source_meaning_id", "sense_id"), + "sense_exam": ("sense_id", "exam_id"), + "sense_relation": ("sense_id", "relation_type", "position", "target_meaning_id"), + "entry_exam_meaning_order": ("entry_id", "exam_id", "scope", "position", "meaning_id"), + "phrase": ("phrase_id", "entry_id", "source_phrase_id", "meaning_id", + "group_type", "text_en", "text_zh", "audio_ref", "extra_code"), + "phrase_exam": ("phrase_id", "exam_id"), + "phrase_exam_order": ("entry_id", "exam_id", "group_type", "position", "phrase_id"), + "example_group": ("example_group_id", "entry_id", "meaning_id", "group_kind", + "part_of_speech", "entry_kind_code", "head_form", + "definition_en", "definition_zh", "youdao_code_1", + "youdao_code_2", "source_exam_codes"), + "example_group_exam": ("example_group_id", "exam_id"), + "example_usage": ("usage_id", "example_group_id", "pattern_id", + "pattern_text", "position"), + "example_sentence": ("sentence_id", "source_sentence_key", + "source_sentence_ext_key", "example_group_id", "usage_id", + "position", "text_en", "text_zh", "audio_ref", "audio_url", + "image_path", "source_title", "is_listening_corpus"), + "entry_exam_sentence": ("entry_id", "exam_id", "position", "text_en", + "text_zh", "source_desc"), + "confusable_word": ("confusable_id", "entry_id", "position", "word_text", + "gloss_zh", "pos_code", "spelling_flag"), + "morpheme": ("morpheme_id", "morpheme_type", "source_morpheme_id", "form", + "display_text", "meaning_en", "meaning_zh", "note"), + "card": ("card_id", "word", "highlighted_word", "memory_note", "example_text", + "prefix_morpheme_id", "root_morpheme_id", "suffix_morpheme_id", + "default_family_ref", "is_hidden"), + "card_exam": ("card_id", "exam_id"), + "word_family": ("family_entry_id", "top_family_id", "word", "highlighted_word", + "interpretation", "prefix_morpheme_id", "suffix_morpheme_id", + "group_level_1", "group_level_2", "group_level_3"), + "word_family_exam": ("family_entry_id", "exam_id"), + "word_family_exam_order": ("family_entry_id", "exam_id", "position"), + "word_family_variant": ("variant_id", "variant_word", "family_entry_id", + "top_family_id", "highlighted_word", "interpretation", + "prefix_morpheme_id", "suffix_morpheme_id"), + "word_family_variant_member": ("variant_id", "exam_id", "position", + "member_word", "is_placeholder"), + "word_family_mini_group": ("mini_group_id", "size"), + "word_family_mini_group_member": ("mini_group_id", "family_entry_id", "position"), + "card_family_link": ("link_id", "card_id", "group_no", "family_ref", "is_hidden"), + "synonym_word": ("word_id", "word"), + "synonym_sense": ("sense_id", "word_id", "part_of_speech", "gloss_zh"), + "synonym_group": ("sense_id", "exam_id", "canonical_word_id", "member_count"), + "synonym_group_member": ("sense_id", "exam_id", "position", "member_word_id"), + "synonym_antonym": ("sense_id", "exam_id", "position", "antonym_word_id"), +} + +# 字段注释:DDL 生成时强制校验,缺一不可 +COLUMN_COMMENTS = { + # exam + ("exam", "exam_id"): "主键;0=def 默认视图,1-13 真实考试,14=XX 未分类,15=OTHER 其他", + ("exam", "exam_code"): "考试业务编码,仅作唯一键,不参与外键", + ("exam", "exam_name_zh"): "考试中文名称", + ("exam", "is_grade_usable"): "是否可用于年级/考试编码(仅 1-10 为真)", + ("exam", "is_pseudo"): "是否为伪考试视图(def/XX/OTHER)", + # book + ("book", "book_id"): "词书主键,对应目录 id", + ("book", "book_code"): "词书业务唯一编码,取自文件名后缀", + ("book", "book_name"): "词书名称", + ("book", "description"): "词书描述", + ("book", "exam_id"): "关联考试 id,空表示非考试类", + ("book", "declared_entry_count"): "源端声明的词条数", + ("book", "cover_url"): "封面图片地址", + ("book", "package_url"): "词书数据包地址", + ("book", "content_version"): "内容版本", + ("book", "schema_version"): "目录结构版本", + ("book", "sort_by"): "排序方式", + ("book", "sort_at"): "排序时间,Unix 秒", + ("book", "is_for_sale"): "是否售卖", + ("book", "is_purchased"): "是否已购买", + ("book", "is_opt"): "是否可选项", + ("book", "source_code"): "来源编码", + ("book", "catalog_type"): "目录类型", + ("book", "labels"): "标签文本", + ("book", "publish_log"): "发布日志", + ("book", "pkg_page"): "包页面配置,JSON", + ("book", "external_link"): "外部链接配置,JSON", + ("book", "has_wordbook_file"): "是否存在词书清单文件", + # book_phrase_book + ("book_phrase_book", "book_id"): "词书 id", + ("book_phrase_book", "phrase_book_id"): "内嵌短语书的词书 id", + ("book_phrase_book", "position"): "内嵌顺序", + # word_entry + ("word_entry", "entry_id"): "词条主键,取自 zpk 文件名第 2 段", + ("word_entry", "headword"): "词条原形", + ("word_entry", "index_word"): "归一化索引词", + ("word_entry", "definition_text"): "原始释义文本(词性 + 中文)", + ("word_entry", "us_pronunciation"): "美式音标", + ("word_entry", "uk_pronunciation"): "英式音标", + ("word_entry", "data_version"): "源数据版本", + ("word_entry", "book_exam_id"): "所属考试 id", + ("word_entry", "occurrence_count"): "源端声明的收录词书数", + ("word_entry", "has_listening_examples"): "是否包含听力语料例句", + ("word_entry", "source_zpk_key"): "源 zpk 文件键,仅溯源", + ("word_entry", "source_variant_code"): "源文件名中的变体编码", + ("word_entry", "source_timestamp"): "源文件名中的时间戳,Unix 秒", + # book_entry + ("book_entry", "book_entry_id"): "词书词条关联主键", + ("book_entry", "book_id"): "词书 id", + ("book_entry", "entry_id"): "词条 id", + ("book_entry", "display_word"): "词书中的展示词形", + ("book_entry", "headword_text"): "主词/词根提示,弱文本引用", + ("book_entry", "seq_book"): "词书内顺序", + ("book_entry", "seq_az"): "字母序", + ("book_entry", "seq_freq"): "词频排名,0 表示未知", + ("book_entry", "seq_rand"): "随机顺序", + ("book_entry", "source_zpk_key"): "源 zpk 文件键,仅溯源", + # word_spelling_variant + ("word_spelling_variant", "entry_id"): "词条 id", + ("word_spelling_variant", "position"): "变体顺序", + ("word_spelling_variant", "variant"): "拼写变体文本", + # word_sense + ("word_sense", "sense_id"): "义项主键(代理)", + ("word_sense", "entry_id"): "词条 id", + ("word_sense", "sense_no"): "词条内义项序号,源值不连续", + ("word_sense", "part_of_speech"): "词性", + ("word_sense", "gloss_zh"): "中文释义", + ("word_sense", "secondary_exam_codes"): "源字段 s,次要考试编码列表,逗号分隔", + ("word_sense", "flag_h"): "源字段 h,数值标记", + # sense_object + ("sense_object", "meaning_id"): "释义对象主键(代理)", + ("sense_object", "entry_id"): "词条 id", + ("sense_object", "source_meaning_id"): "源释义对象 id(oid),仅词条内唯一", + ("sense_object", "sense_id"): "所属义项 id", + # sense_exam + ("sense_exam", "sense_id"): "义项 id", + ("sense_exam", "exam_id"): "考试 id", + # sense_relation + ("sense_relation", "sense_id"): "义项 id", + ("sense_relation", "relation_type"): "关系类型:synonym/antonym/reference", + ("sense_relation", "position"): "关系内顺序", + ("sense_relation", "target_meaning_id"): "目标释义对象 id", + # entry_exam_meaning_order + ("entry_exam_meaning_order", "entry_id"): "词条 id", + ("entry_exam_meaning_order", "exam_id"): "考试 id", + ("entry_exam_meaning_order", "scope"): "释义范围:core 核心 / ext 扩展", + ("entry_exam_meaning_order", "position"): "范围内顺序", + ("entry_exam_meaning_order", "meaning_id"): "释义对象 id", + # phrase + ("phrase", "phrase_id"): "短语主键(代理)", + ("phrase", "entry_id"): "词条 id", + ("phrase", "source_phrase_id"): "源短语 id(pid),仅词条内唯一", + ("phrase", "meaning_id"): "所属释义对象 id", + ("phrase", "group_type"): "短语组类型,源值 1/2", + ("phrase", "text_en"): "英文短语", + ("phrase", "text_zh"): "中文释义", + ("phrase", "audio_ref"): "发音资源标识,非外键", + ("phrase", "extra_code"): "源附加编码 bd", + # phrase_exam + ("phrase_exam", "phrase_id"): "短语 id", + ("phrase_exam", "exam_id"): "考试 id", + # phrase_exam_order + ("phrase_exam_order", "entry_id"): "词条 id", + ("phrase_exam_order", "exam_id"): "考试 id", + ("phrase_exam_order", "group_type"): "短语组类型", + ("phrase_exam_order", "position"): "分考试顺序", + ("phrase_exam_order", "phrase_id"): "短语 id", + # example_group + ("example_group", "example_group_id"): "例句块主键(代理)", + ("example_group", "entry_id"): "词条 id", + ("example_group", "meaning_id"): "所属释义对象 id,type 99 为空", + ("example_group", "group_kind"): "例句块类型:1 核心,2 扩展,3 短语动词,4 习语,99 听力语料", + ("example_group", "part_of_speech"): "词性", + ("example_group", "entry_kind_code"): "词条类型码:NORMAL/PHRV/IDM/PHR", + ("example_group", "head_form"): "词头形式", + ("example_group", "definition_en"): "英文释义", + ("example_group", "definition_zh"): "中文释义", + ("example_group", "youdao_code_1"): "源附加编码 yu", + ("example_group", "youdao_code_2"): "源附加编码 yo", + ("example_group", "source_exam_codes"): "源考试编码快照,规范值见 example_group_exam", + # example_group_exam + ("example_group_exam", "example_group_id"): "例句块 id", + ("example_group_exam", "exam_id"): "考试 id", + # example_usage + ("example_usage", "usage_id"): "用法模式主键(代理)", + ("example_usage", "example_group_id"): "例句块 id", + ("example_usage", "pattern_id"): "关联短语 id(源 uid 解析)", + ("example_usage", "pattern_text"): "用法模式文本", + ("example_usage", "position"): "组内顺序", + # example_sentence + ("example_sentence", "sentence_id"): "句子主键(代理)", + ("example_sentence", "source_sentence_key"): "源句子标识 sid,不唯一", + ("example_sentence", "source_sentence_ext_key"): "源句子扩展标识 eid,不唯一", + ("example_sentence", "example_group_id"): "例句块 id", + ("example_sentence", "usage_id"): "用法模式 id,type 99 为空", + ("example_sentence", "position"): "组内顺序", + ("example_sentence", "text_en"): "英文句子,含高亮标签", + ("example_sentence", "text_zh"): "中文翻译", + ("example_sentence", "audio_ref"): "发音资源标识,非外键", + ("example_sentence", "audio_url"): "音频地址", + ("example_sentence", "image_path"): "图片路径", + ("example_sentence", "source_title"): "来源书名或剧名", + ("example_sentence", "is_listening_corpus"): "是否听力语料例句", + # entry_exam_sentence + ("entry_exam_sentence", "entry_id"): "词条 id", + ("entry_exam_sentence", "exam_id"): "考试 id", + ("entry_exam_sentence", "position"): "顺序", + ("entry_exam_sentence", "text_en"): "英文例句", + ("entry_exam_sentence", "text_zh"): "中文翻译", + ("entry_exam_sentence", "source_desc"): "来源说明", + # confusable_word + ("confusable_word", "confusable_id"): "易混词主键(代理)", + ("confusable_word", "entry_id"): "词条 id", + ("confusable_word", "position"): "顺序", + ("confusable_word", "word_text"): "易混词文本", + ("confusable_word", "gloss_zh"): "中文释义", + ("confusable_word", "pos_code"): "源词性编码", + ("confusable_word", "spelling_flag"): "拼写标记", + # morpheme + ("morpheme", "morpheme_id"): "词素主键(统一代理)", + ("morpheme", "morpheme_type"): "词素类型:prefix/root/suffix", + ("morpheme", "source_morpheme_id"): "源表内 id", + ("morpheme", "form"): "词素形式", + ("morpheme", "display_text"): "原始展示文本", + ("morpheme", "meaning_en"): "英文释义", + ("morpheme", "meaning_zh"): "中文释义", + ("morpheme", "note"): "备注", + # card + ("card", "card_id"): "卡片主键", + ("card", "word"): "单词", + ("card", "highlighted_word"): "高亮 HTML 快照", + ("card", "memory_note"): "记忆提示", + ("card", "example_text"): "例句文本", + ("card", "prefix_morpheme_id"): "前缀词素 id", + ("card", "root_morpheme_id"): "词根词素 id", + ("card", "suffix_morpheme_id"): "后缀词素 id", + ("card", "default_family_ref"): "旧版词族引用,不可解析,无外键", + ("card", "is_hidden"): "是否隐藏", + # card_exam + ("card_exam", "card_id"): "卡片 id", + ("card_exam", "exam_id"): "考试 id", + # word_family + ("word_family", "family_entry_id"): "词族词条主键", + ("word_family", "top_family_id"): "顶层词族 id,自引用", + ("word_family", "word"): "单词", + ("word_family", "highlighted_word"): "高亮 HTML 快照", + ("word_family", "interpretation"): "词性 + 中文释义", + ("word_family", "prefix_morpheme_id"): "前缀词素 id", + ("word_family", "suffix_morpheme_id"): "后缀词素 id", + ("word_family", "group_level_1"): "分组层级 1", + ("word_family", "group_level_2"): "分组层级 2", + ("word_family", "group_level_3"): "分组层级 3", + # word_family_exam + ("word_family_exam", "family_entry_id"): "词族词条 id", + ("word_family_exam", "exam_id"): "考试 id", + # word_family_exam_order + ("word_family_exam_order", "family_entry_id"): "词族词条 id", + ("word_family_exam_order", "exam_id"): "考试 id", + ("word_family_exam_order", "position"): "分考试顺序", + # word_family_variant + ("word_family_variant", "variant_id"): "变体主键", + ("word_family_variant", "variant_word"): "变体词", + ("word_family_variant", "family_entry_id"): "所属词族词条 id", + ("word_family_variant", "top_family_id"): "顶层词族 id", + ("word_family_variant", "highlighted_word"): "高亮 HTML 快照", + ("word_family_variant", "interpretation"): "词性 + 中文释义", + ("word_family_variant", "prefix_morpheme_id"): "前缀词素 id", + ("word_family_variant", "suffix_morpheme_id"): "后缀词素 id", + # word_family_variant_member + ("word_family_variant_member", "variant_id"): "变体 id", + ("word_family_variant_member", "exam_id"): "考试 id", + ("word_family_variant_member", "position"): "顺序", + ("word_family_variant_member", "member_word"): "成员词", + ("word_family_variant_member", "is_placeholder"): "是否为 * 占位符", + # word_family_mini_group + ("word_family_mini_group", "mini_group_id"): "小组主键", + ("word_family_mini_group", "size"): "小组成员数", + # word_family_mini_group_member + ("word_family_mini_group_member", "mini_group_id"): "小组 id", + ("word_family_mini_group_member", "family_entry_id"): "词族词条 id", + ("word_family_mini_group_member", "position"): "组内顺序", + # card_family_link + ("card_family_link", "link_id"): "关联主键", + ("card_family_link", "card_id"): "卡片 id", + ("card_family_link", "group_no"): "分组编号(1-50)", + ("card_family_link", "family_ref"): "旧版词族引用,不可解析,无外键", + ("card_family_link", "is_hidden"): "是否隐藏", + # synonym_word + ("synonym_word", "word_id"): "近义词词头主键", + ("synonym_word", "word"): "单词", + # synonym_sense + ("synonym_sense", "sense_id"): "近义词义项主键", + ("synonym_sense", "word_id"): "词头 id", + ("synonym_sense", "part_of_speech"): "词性", + ("synonym_sense", "gloss_zh"): "中文释义", + # synonym_group + ("synonym_group", "sense_id"): "近义词义项 id", + ("synonym_group", "exam_id"): "考试 id", + ("synonym_group", "canonical_word_id"): "规范词 id(源 free),可为空", + ("synonym_group", "member_count"): "近义词成员数", + # synonym_group_member + ("synonym_group_member", "sense_id"): "近义词义项 id", + ("synonym_group_member", "exam_id"): "考试 id", + ("synonym_group_member", "position"): "组内顺序", + ("synonym_group_member", "member_word_id"): "成员词 id", + # synonym_antonym + ("synonym_antonym", "sense_id"): "近义词义项 id", + ("synonym_antonym", "exam_id"): "考试 id", + ("synonym_antonym", "position"): "组内顺序", + ("synonym_antonym", "antonym_word_id"): "反义词 id", +} + +# PostgreSQL DDL;SQLite 通过类型替换复用 +SCHEMA_PG = """ +CREATE TABLE exam ( + exam_id bigint PRIMARY KEY, + exam_code varchar(16) NOT NULL UNIQUE, + exam_name_zh varchar(32) NOT NULL, + is_grade_usable boolean NOT NULL DEFAULT FALSE, + is_pseudo boolean NOT NULL DEFAULT FALSE +); + +CREATE TABLE book ( + book_id bigint PRIMARY KEY, + book_code varchar(32) NOT NULL UNIQUE, + book_name varchar(128) NOT NULL, + description text, + exam_id bigint REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED, + declared_entry_count integer, + cover_url text, + package_url text, + content_version integer, + schema_version integer, + sort_by varchar(16), + sort_at bigint, + is_for_sale smallint, + is_purchased smallint, + is_opt smallint, + source_code smallint, + catalog_type smallint, + labels text, + publish_log text, + pkg_page jsonb, + external_link jsonb, + has_wordbook_file boolean NOT NULL DEFAULT TRUE +); + +CREATE TABLE book_phrase_book ( + book_id bigint NOT NULL REFERENCES book(book_id) DEFERRABLE INITIALLY DEFERRED, + phrase_book_id bigint NOT NULL REFERENCES book(book_id) DEFERRABLE INITIALLY DEFERRED, + position smallint NOT NULL, + PRIMARY KEY (book_id, phrase_book_id) +); + +CREATE TABLE word_entry ( + entry_id bigint PRIMARY KEY, + headword varchar(256) NOT NULL, + index_word varchar(256) NOT NULL, + definition_text text, + us_pronunciation varchar(128), + uk_pronunciation varchar(128), + data_version varchar(8), + book_exam_id bigint REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED, + occurrence_count integer, + has_listening_examples boolean NOT NULL DEFAULT FALSE, + source_zpk_key varchar(64) UNIQUE, + source_variant_code integer, + source_timestamp bigint +); + +CREATE TABLE book_entry ( + book_entry_id bigint PRIMARY KEY, + book_id bigint NOT NULL REFERENCES book(book_id) DEFERRABLE INITIALLY DEFERRED, + entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED, + display_word varchar(256) NOT NULL, + headword_text varchar(256), + seq_book integer, + seq_az integer, + seq_freq integer, + seq_rand integer, + source_zpk_key varchar(64), + UNIQUE (book_id, entry_id) +); + +CREATE TABLE word_spelling_variant ( + entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED, + position smallint NOT NULL, + variant varchar(256) NOT NULL, + PRIMARY KEY (entry_id, position) +); + +CREATE TABLE word_sense ( + sense_id bigint PRIMARY KEY, + entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED, + sense_no integer NOT NULL, + part_of_speech varchar(32), + gloss_zh text, + secondary_exam_codes text, + flag_h smallint +); + +CREATE TABLE sense_object ( + meaning_id bigint PRIMARY KEY, + entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED, + source_meaning_id bigint NOT NULL, + sense_id bigint REFERENCES word_sense(sense_id) DEFERRABLE INITIALLY DEFERRED +); + +CREATE TABLE sense_exam ( + sense_id bigint NOT NULL REFERENCES word_sense(sense_id) DEFERRABLE INITIALLY DEFERRED, + exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED, + PRIMARY KEY (sense_id, exam_id) +); + +CREATE TABLE sense_relation ( + sense_id bigint NOT NULL REFERENCES word_sense(sense_id) DEFERRABLE INITIALLY DEFERRED, + relation_type varchar(16) NOT NULL, + position smallint NOT NULL, + target_meaning_id bigint NOT NULL REFERENCES sense_object(meaning_id) DEFERRABLE INITIALLY DEFERRED, + PRIMARY KEY (sense_id, relation_type, position) +); + +CREATE TABLE entry_exam_meaning_order ( + entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED, + exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED, + scope varchar(8) NOT NULL CHECK (scope IN ('core','ext')), + position smallint NOT NULL, + meaning_id bigint NOT NULL REFERENCES sense_object(meaning_id) DEFERRABLE INITIALLY DEFERRED, + PRIMARY KEY (entry_id, exam_id, scope, position) +); + +CREATE TABLE phrase ( + phrase_id bigint PRIMARY KEY, + entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED, + source_phrase_id bigint, + meaning_id bigint REFERENCES sense_object(meaning_id) DEFERRABLE INITIALLY DEFERRED, + group_type smallint NOT NULL, + text_en text NOT NULL, + text_zh text, + audio_ref varchar(64), + extra_code varchar(16), + UNIQUE (entry_id, source_phrase_id) +); + +CREATE TABLE phrase_exam ( + phrase_id bigint NOT NULL REFERENCES phrase(phrase_id) DEFERRABLE INITIALLY DEFERRED, + exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED, + PRIMARY KEY (phrase_id, exam_id) +); + +CREATE TABLE phrase_exam_order ( + entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED, + exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED, + group_type smallint NOT NULL, + position smallint NOT NULL, + phrase_id bigint NOT NULL REFERENCES phrase(phrase_id) DEFERRABLE INITIALLY DEFERRED, + PRIMARY KEY (entry_id, exam_id, group_type, position) +); + +CREATE TABLE example_group ( + example_group_id bigint PRIMARY KEY, + entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED, + meaning_id bigint REFERENCES sense_object(meaning_id) DEFERRABLE INITIALLY DEFERRED, + group_kind smallint NOT NULL, + part_of_speech varchar(32), + entry_kind_code varchar(16), + head_form varchar(256), + definition_en text, + definition_zh text, + youdao_code_1 varchar(64), + youdao_code_2 varchar(64), + source_exam_codes text +); + +CREATE TABLE example_group_exam ( + example_group_id bigint NOT NULL REFERENCES example_group(example_group_id) DEFERRABLE INITIALLY DEFERRED, + exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED, + PRIMARY KEY (example_group_id, exam_id) +); + +CREATE TABLE example_usage ( + usage_id bigint PRIMARY KEY, + example_group_id bigint NOT NULL REFERENCES example_group(example_group_id) DEFERRABLE INITIALLY DEFERRED, + pattern_id bigint REFERENCES phrase(phrase_id) DEFERRABLE INITIALLY DEFERRED, + pattern_text text, + position smallint NOT NULL +); + +CREATE TABLE example_sentence ( + sentence_id bigint PRIMARY KEY, + source_sentence_key varchar(64), + source_sentence_ext_key varchar(64), + example_group_id bigint NOT NULL REFERENCES example_group(example_group_id) DEFERRABLE INITIALLY DEFERRED, + usage_id bigint REFERENCES example_usage(usage_id) DEFERRABLE INITIALLY DEFERRED, + position smallint, + text_en text NOT NULL, + text_zh text, + audio_ref varchar(64), + audio_url text, + image_path text, + source_title varchar(256), + is_listening_corpus boolean NOT NULL DEFAULT FALSE +); + +CREATE TABLE entry_exam_sentence ( + entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED, + exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED, + position smallint NOT NULL, + text_en text NOT NULL, + text_zh text, + source_desc text, + PRIMARY KEY (entry_id, exam_id, position) +); + +CREATE TABLE confusable_word ( + confusable_id bigint PRIMARY KEY, + entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED, + position smallint NOT NULL, + word_text varchar(256) NOT NULL, + gloss_zh text, + pos_code smallint, + spelling_flag varchar(8), + UNIQUE (entry_id, word_text) +); + +CREATE TABLE morpheme ( + morpheme_id bigint PRIMARY KEY, + morpheme_type varchar(8) NOT NULL CHECK (morpheme_type IN ('prefix','root','suffix')), + source_morpheme_id bigint NOT NULL, + form varchar(64), + display_text varchar(256) NOT NULL, + meaning_en varchar(256), + meaning_zh varchar(256), + note varchar(256), + UNIQUE (morpheme_type, source_morpheme_id) +); + +CREATE TABLE card ( + card_id bigint PRIMARY KEY, + word varchar(256) NOT NULL, + highlighted_word text, + memory_note text, + example_text text, + prefix_morpheme_id bigint REFERENCES morpheme(morpheme_id) DEFERRABLE INITIALLY DEFERRED, + root_morpheme_id bigint REFERENCES morpheme(morpheme_id) DEFERRABLE INITIALLY DEFERRED, + suffix_morpheme_id bigint REFERENCES morpheme(morpheme_id) DEFERRABLE INITIALLY DEFERRED, + default_family_ref bigint, + is_hidden boolean NOT NULL DEFAULT FALSE +); + +CREATE TABLE card_exam ( + card_id bigint NOT NULL REFERENCES card(card_id) DEFERRABLE INITIALLY DEFERRED, + exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED, + PRIMARY KEY (card_id, exam_id) +); + +CREATE TABLE word_family ( + family_entry_id bigint PRIMARY KEY, + top_family_id bigint NOT NULL REFERENCES word_family(family_entry_id) DEFERRABLE INITIALLY DEFERRED, + word varchar(256) NOT NULL, + highlighted_word text, + interpretation text, + prefix_morpheme_id bigint REFERENCES morpheme(morpheme_id) DEFERRABLE INITIALLY DEFERRED, + suffix_morpheme_id bigint REFERENCES morpheme(morpheme_id) DEFERRABLE INITIALLY DEFERRED, + group_level_1 smallint, + group_level_2 smallint, + group_level_3 smallint +); + +CREATE TABLE word_family_exam ( + family_entry_id bigint NOT NULL REFERENCES word_family(family_entry_id) DEFERRABLE INITIALLY DEFERRED, + exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED, + PRIMARY KEY (family_entry_id, exam_id) +); + +CREATE TABLE word_family_exam_order ( + family_entry_id bigint NOT NULL REFERENCES word_family(family_entry_id) DEFERRABLE INITIALLY DEFERRED, + exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED, + position integer NOT NULL, + PRIMARY KEY (family_entry_id, exam_id) +); + +CREATE TABLE word_family_variant ( + variant_id bigint PRIMARY KEY, + variant_word varchar(256) NOT NULL, + family_entry_id bigint NOT NULL REFERENCES word_family(family_entry_id) DEFERRABLE INITIALLY DEFERRED, + top_family_id bigint NOT NULL REFERENCES word_family(family_entry_id) DEFERRABLE INITIALLY DEFERRED, + highlighted_word text, + interpretation text, + prefix_morpheme_id bigint REFERENCES morpheme(morpheme_id) DEFERRABLE INITIALLY DEFERRED, + suffix_morpheme_id bigint REFERENCES morpheme(morpheme_id) DEFERRABLE INITIALLY DEFERRED +); + +CREATE TABLE word_family_variant_member ( + variant_id bigint NOT NULL REFERENCES word_family_variant(variant_id) DEFERRABLE INITIALLY DEFERRED, + exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED, + position smallint NOT NULL, + member_word varchar(256) NOT NULL, + is_placeholder boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY (variant_id, exam_id, position) +); + +CREATE TABLE word_family_mini_group ( + mini_group_id bigint PRIMARY KEY, + size smallint NOT NULL +); + +CREATE TABLE word_family_mini_group_member ( + mini_group_id bigint NOT NULL REFERENCES word_family_mini_group(mini_group_id) DEFERRABLE INITIALLY DEFERRED, + family_entry_id bigint NOT NULL REFERENCES word_family(family_entry_id) DEFERRABLE INITIALLY DEFERRED, + position smallint NOT NULL, + PRIMARY KEY (mini_group_id, family_entry_id) +); + +CREATE TABLE card_family_link ( + link_id bigint PRIMARY KEY, + card_id bigint NOT NULL REFERENCES card(card_id) DEFERRABLE INITIALLY DEFERRED, + group_no smallint NOT NULL, + family_ref bigint, + is_hidden boolean NOT NULL DEFAULT FALSE +); + +CREATE TABLE synonym_word ( + word_id bigint PRIMARY KEY, + word varchar(256) NOT NULL +); + +CREATE TABLE synonym_sense ( + sense_id bigint PRIMARY KEY, + word_id bigint NOT NULL REFERENCES synonym_word(word_id) DEFERRABLE INITIALLY DEFERRED, + part_of_speech varchar(32), + gloss_zh text +); + +CREATE TABLE synonym_group ( + sense_id bigint NOT NULL REFERENCES synonym_sense(sense_id) DEFERRABLE INITIALLY DEFERRED, + exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED, + canonical_word_id bigint REFERENCES synonym_word(word_id) DEFERRABLE INITIALLY DEFERRED, + member_count smallint NOT NULL, + PRIMARY KEY (sense_id, exam_id) +); + +CREATE TABLE synonym_group_member ( + sense_id bigint NOT NULL, + exam_id bigint NOT NULL, + position smallint NOT NULL, + member_word_id bigint NOT NULL REFERENCES synonym_word(word_id) DEFERRABLE INITIALLY DEFERRED, + PRIMARY KEY (sense_id, exam_id, position), + FOREIGN KEY (sense_id, exam_id) REFERENCES synonym_group(sense_id, exam_id) DEFERRABLE INITIALLY DEFERRED +); + +CREATE TABLE synonym_antonym ( + sense_id bigint NOT NULL, + exam_id bigint NOT NULL, + position smallint NOT NULL, + antonym_word_id bigint NOT NULL REFERENCES synonym_word(word_id) DEFERRABLE INITIALLY DEFERRED, + PRIMARY KEY (sense_id, exam_id, position), + FOREIGN KEY (sense_id, exam_id) REFERENCES synonym_group(sense_id, exam_id) DEFERRABLE INITIALLY DEFERRED +); +""" + +INDEXES = """ +CREATE INDEX idx_book_entry_book_seq ON book_entry (book_id, seq_book); +CREATE INDEX idx_book_entry_entry ON book_entry (entry_id); +CREATE INDEX idx_word_entry_index_word ON word_entry (index_word); +CREATE INDEX idx_word_sense_entry ON word_sense (entry_id, sense_no); +CREATE INDEX idx_sense_object_entry ON sense_object (entry_id, source_meaning_id); +CREATE INDEX idx_phrase_entry ON phrase (entry_id); +CREATE INDEX idx_phrase_meaning ON phrase (meaning_id); +CREATE INDEX idx_example_group_entry ON example_group (entry_id, group_kind); +CREATE INDEX idx_example_group_meaning ON example_group (meaning_id); +CREATE INDEX idx_example_sentence_group ON example_sentence (example_group_id); +CREATE INDEX idx_example_sentence_usage ON example_sentence (usage_id); +CREATE INDEX idx_example_sentence_src ON example_sentence (source_sentence_key); +CREATE INDEX idx_entry_exam_sentence_entry ON entry_exam_sentence (entry_id); +CREATE INDEX idx_confusable_entry ON confusable_word (entry_id); +CREATE INDEX idx_morpheme_form ON morpheme (morpheme_type, form); +CREATE INDEX idx_word_family_word ON word_family (word); +CREATE INDEX idx_word_family_top ON word_family (top_family_id); +CREATE INDEX idx_mini_member_family ON word_family_mini_group_member (family_entry_id); +CREATE INDEX idx_card_word ON card (word); +CREATE INDEX idx_card_family_link_card ON card_family_link (card_id); +CREATE INDEX idx_synonym_word_text ON synonym_word (word); +CREATE INDEX idx_syn_group_member_word ON synonym_group_member (member_word_id); +CREATE INDEX idx_syn_antonym_word ON synonym_antonym (antonym_word_id); +""" + +EXAM_CODES = { + "def": (0, "默认视图", True), + "ZK": (1, "中考", False), + "GK": (2, "高考", False), + "CET4": (3, "四级", False), + "CET6": (4, "六级", False), + "KY": (5, "考研", False), + "IELTS": (6, "雅思", False), + "TOEFL": (7, "托福", False), + "PRO4": (8, "专四", False), + "PRO8": (9, "专八", False), + "ZSB": (10, "专升本", False), + "KB": (11, "考博", False), + "SAT": (12, "SAT", False), + "GRE": (13, "GRE", False), + "XX": (14, "未分类", True), + "OTHER": (15, "其他", True), +} +GRADE_USABLE = set(range(1, 11)) +SYNONYM_EXAM_COLUMNS = { + "zk": 1, "gk": 2, "cet4": 3, "cet6": 4, "ky": 5, + "ielts": 6, "toefl": 7, "zsb": 10, "other": 15, +} + + +# --------------------------------------------------------------------------- +# 工具函数 +# --------------------------------------------------------------------------- + +def _to_int(value, default=None): + if value is None: + return default + if isinstance(value, bool): + return int(value) + if isinstance(value, int): + return value + s = str(value).strip() + if not s: + return default + try: + return int(s) + except ValueError: + try: + return int(float(s)) + except ValueError: + return default + + +def _to_bool(value): + return 1 if str(value).strip().lower() in ("1", "true", "yes", "y") else 0 + + +def _text(value): + if value is None: + return None + s = str(value) + return s if s != "" else None + + +def read_csv(path): + with open(path, encoding="utf-8-sig", newline="") as f: + yield from csv.DictReader(f) + + +def exam_id_of(code): + if code is None: + return None + s = str(code).strip() + if not s: + return None + if s in EXAM_CODES: + return EXAM_CODES[s][0] + up = s.upper() + if up in EXAM_CODES: + return EXAM_CODES[up][0] + return None + + +def parse_affix(display): + """'duct = to lead 引导' -> (form, meaning_en, meaning_zh, note)""" + text = (display or "").strip() + if "=" in text: + left, right = text.split("=", 1) + form = left.strip() + right = right.strip() + else: + form, right = text, "" + # 以第一个 CJK 字符为界 + m = re.search(r"[\u3400-\u9fff]", right) + if m: + en = right[:m.start()].strip() or None + zh = right[m.start():].strip() or None + note = None + if ":" in zh: + zh, note = zh.split(":", 1) + zh, note = zh.strip(), note.strip() or None + elif ":" in zh: + zh, note = zh.split(":", 1) + zh, note = zh.strip(), note.strip() or None + return form, en, zh, note + return form, right or None, None, None + + +# --------------------------------------------------------------------------- +# 数据库适配 +# --------------------------------------------------------------------------- + +class SQLiteAdapter: + placeholder = "?" + + def __init__(self, path): + self.conn = sqlite3.connect(path) + self.conn.execute("PRAGMA journal_mode=OFF") + self.conn.execute("PRAGMA synchronous=OFF") + self.conn.execute("PRAGMA temp_store=MEMORY") + self.conn.execute("PRAGMA cache_size=-200000") + self.conn.execute("PRAGMA foreign_keys=OFF") + self.cur = self.conn.cursor() + + def begin(self): + self.conn.execute("BEGIN") + + def execute(self, sql, params=()): + self.cur.execute(sql, params) + + def execute_script(self, script): + self.cur.executescript(script) + + def insert_many(self, table, columns, rows): + sql = "INSERT INTO {} ({}) VALUES ({})".format( + table, ",".join(columns), ",".join("?" * len(columns))) + self.cur.executemany(sql, rows) + + def commit(self): + self.conn.commit() + + def rollback(self): + self.conn.rollback() + + def close(self): + self.conn.close() + + def table_names(self): + return {r[0] for r in self.cur.execute( + "SELECT name FROM sqlite_master WHERE type='table'")} + + +class PGAdapter: + placeholder = "%s" + + def __init__(self, dsn): + try: + import psycopg2 + from psycopg2.extras import execute_values + except ImportError: + try: + import psycopg as psycopg2 + from psycopg.extras import execute_values + except ImportError as exc: + raise SystemExit( + "需要 psycopg2:pip install psycopg2-binary") from exc + self.conn = psycopg2.connect(dsn) + self.conn.autocommit = False + self.cur = self.conn.cursor() + self._execute_values = execute_values + + def begin(self): + pass + + def execute(self, sql, params=()): + self.cur.execute(sql, params) + + def execute_script(self, script): + for stmt in script.split(";"): + if stmt.strip(): + self.cur.execute(stmt) + + def insert_many(self, table, columns, rows): + base = "INSERT INTO {} ({}) VALUES %s".format( + table, ",".join(columns)) + self._execute_values(self.cur, base, rows, page_size=1000) + + def commit(self): + self.conn.commit() + + def rollback(self): + self.conn.rollback() + + def close(self): + self.conn.close() + + def table_names(self): + self.cur.execute( + "SELECT table_name FROM information_schema.tables " + "WHERE table_schema='public'") + return {r[0] for r in self.cur.fetchall()} + + +class MySQLAdapter: + placeholder = "%s" + + def __init__(self, dsn): + try: + import pymysql + except ImportError: + try: + import mysql.connector as pymysql + except ImportError as exc: + raise SystemExit( + "需要 MySQL 驱动:pip install pymysql") from exc + parsed = urlparse(dsn) + if parsed.scheme not in ("mysql", "mysql+pymysql", ""): + raise SystemExit("MySQL DSN 应形如 mysql://user:pass@host:3306/db") + self.conn = pymysql.connect( + host=parsed.hostname or "127.0.0.1", + port=parsed.port or 3306, + user=unquote(parsed.username or "root"), + password=unquote(parsed.password or ""), + database=(parsed.path or "/").lstrip("/"), + charset="utf8mb4", + autocommit=False, + ) + self.cur = self.conn.cursor() + + def begin(self): + pass + + def execute(self, sql, params=()): + self.cur.execute(sql, params) + + def execute_script(self, script): + for stmt in script.split(";"): + if stmt.strip(): + self.cur.execute(stmt) + + def insert_many(self, table, columns, rows): + sql = "INSERT INTO {} ({}) VALUES ({})".format( + table, ",".join(columns), ",".join(["%s"] * len(columns))) + self.cur.executemany(sql, rows) + + def commit(self): + self.conn.commit() + + def rollback(self): + self.conn.rollback() + + def close(self): + self.conn.close() + + def table_names(self): + self.cur.execute( + "SELECT table_name FROM information_schema.tables " + "WHERE table_schema = DATABASE()") + return {r[0] for r in self.cur.fetchall()} + + +NON_COLUMN_KEYWORDS = { + "PRIMARY", "FOREIGN", "UNIQUE", "CHECK", "CONSTRAINT", "KEY", "INDEX", +} + + +def annotate_ddl(ddl, dialect): + """为每个字段追加注释;MySQL 同时把行内 REFERENCES 提升为表级外键。""" + out = [] + table = None + fks = [] + col_re = re.compile(r"^(\s+)([a-z_][a-z0-9_]*)\s+(.+?)(,?)\s*$") + for line in ddl.splitlines(): + m = re.match(r"\s*CREATE TABLE\s+(\w+)\s*\(", line, re.I) + if m: + table = m.group(1) + fks = [] + elif table and line.strip().startswith(")"): + if fks: + if out and not out[-1].rstrip().endswith(","): + out[-1] = out[-1].rstrip() + "," + out.append(" " + ",\n ".join(fks)) + table = None + + mc = col_re.match(line) if table else None + if mc: + col, body, comma = mc.group(2), mc.group(3).rstrip(), mc.group(4) + if col.upper() not in NON_COLUMN_KEYWORDS: + comment = COLUMN_COMMENTS.get((table, col)) + if comment is None: + raise ValueError(f"缺少字段注释: {table}.{col}") + esc = comment.replace("'", "''") + if dialect == "mysql": + ref = re.search( + r"\s+REFERENCES\s+(\w+)\s*\(([^)]+)\)\s*$", body, re.I) + if ref: + body = body[:ref.start()].rstrip() + fks.append("FOREIGN KEY ({}) REFERENCES {}({})".format( + col, ref.group(1), ref.group(2))) + chk = re.search(r"\s+CHECK\s*\((.+)\)\s*$", body, re.I) + if chk: + body = body[:chk.start()].rstrip() + fks.append("CHECK ({})".format(chk.group(1))) + line = "{}{} {} COMMENT '{}'{}".format( + mc.group(1), col, body, esc, comma) + else: + line = line.rstrip() + if line.endswith(","): + line = line[:-1].rstrip() + f", -- {comment}" + else: + line = line + f" -- {comment}" + out.append(line) + return "\n".join(out) + + +def pg_comment_statements(): + stmts = [] + for table in TABLES: + for col in COLUMNS[table]: + comment = COLUMN_COMMENTS.get((table, col)) + if comment is None: + raise ValueError(f"缺少字段注释: {table}.{col}") + esc = comment.replace("'", "''") + stmts.append(f"COMMENT ON COLUMN {table}.{col} IS '{esc}'") + return stmts + + +def sqlite_ddl(): + ddl = SCHEMA_PG + ddl = re.sub(r"\bDEFERRABLE INITIALLY DEFERRED\b", "", ddl) + ddl = re.sub(r"\bjsonb\b", "TEXT", ddl) + ddl = re.sub(r"\bboolean\b", "INTEGER", ddl) + ddl = re.sub(r"\bbigint\b", "INTEGER", ddl) + ddl = re.sub(r"\bsmallint\b", "INTEGER", ddl) + ddl = re.sub(r"\bDEFAULT FALSE\b", "DEFAULT 0", ddl, flags=re.I) + ddl = re.sub(r"\bDEFAULT TRUE\b", "DEFAULT 1", ddl, flags=re.I) + return annotate_ddl(ddl, "sqlite") + + +def mysql_ddl(): + ddl = SCHEMA_PG + ddl = re.sub(r"\bDEFERRABLE INITIALLY DEFERRED\b", "", ddl) + ddl = re.sub(r"\bjsonb\b", "JSON", ddl) + ddl = re.sub(r"\bboolean\b", "TINYINT(1)", ddl) + ddl = re.sub(r"\bDEFAULT FALSE\b", "DEFAULT 0", ddl, flags=re.I) + ddl = re.sub(r"\bDEFAULT TRUE\b", "DEFAULT 1", ddl, flags=re.I) + parts = [] + for stmt in ddl.split(";"): + if not stmt.strip(): + continue + stmt = stmt.rstrip() + if stmt.lstrip().upper().startswith("CREATE TABLE"): + # 必须用 utf8mb4_bin:源数据存在仅大小写不同的唯一键组合 + # (如 confusable_word 同一 entry 下 italic / Italic), + # _ci 排序规则会把二者视为重复,与 SQLite/PostgreSQL 语义不一致。 + stmt += " ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" + parts.append(stmt) + ddl = ";\n".join(parts) + ";" + return annotate_ddl(ddl, "mysql") + + +# --------------------------------------------------------------------------- +# 批量写入 +# --------------------------------------------------------------------------- + +class Writer: + def __init__(self, adapter, batch_size=10000): + self.adapter = adapter + self.batch_size = batch_size + self.buffers = defaultdict(list) + self.counts = defaultdict(int) + + def add(self, table, row): + self.buffers[table].append(row) + if len(self.buffers[table]) >= self.batch_size: + # 按依赖顺序统一刷盘,兼容 MySQL 的即时外键检查 + self.flush_all() + + def flush(self, table): + buf = self.buffers[table] + if not buf: + return + self.adapter.insert_many(table, COLUMNS[table], buf) + self.counts[table] += len(buf) + buf.clear() + + def flush_all(self): + for table in TABLES: + self.flush(table) + + +def nid(name): + nid.value[name] += 1 + return nid.value[name] + + +nid.value = defaultdict(int) + + +# --------------------------------------------------------------------------- +# 导入:公共维表 +# --------------------------------------------------------------------------- + +def import_exams(w, data_dir): + seen = set() + path = os.path.join(data_dir, "wordroot-export", "word_family_exam.csv") + if os.path.exists(path): + for r in read_csv(path): + code = (r.get("exam") or "").strip() + eid = _to_int(r.get("id")) + if not code or eid is None: + continue + name = (r.get("tag") or "").strip() or code + usable = 1 if eid in GRADE_USABLE else 0 + w.add("exam", (eid, code, name, usable, 0)) + seen.add(eid) + for code, (eid, name, pseudo) in EXAM_CODES.items(): + if eid not in seen: + usable = 1 if eid in GRADE_USABLE else 0 + w.add("exam", (eid, code, name, usable, 1 if pseudo else 0)) + w.flush("exam") + print(f" exam: {w.counts['exam']}") + + +# --------------------------------------------------------------------------- +# 导入:词书 +# --------------------------------------------------------------------------- + +def scan_books(data_dir, limit_books=None): + """返回 books(meta), canonical(entry -> (ts,key,word_hint)), phrase_book_ids""" + pattern = os.path.join(data_dir, "wordbooks", "decrypted", "LangEasyLexisV3_*.csv") + files = sorted(glob.glob(pattern)) + if limit_books: + files = files[:limit_books] + books = {} + canonical = {} + for path in files: + code = os.path.basename(path)[len("LangEasyLexisV3_"):-4] + first = None + with open(path, encoding="utf-8-sig", newline="") as f: + for row in csv.DictReader(f): + if first is None: + first = row + eid = _to_int(row.get("id")) + key = (row.get("zpk") or "").strip() + if eid is None or not key: + continue + ts = _to_int(key.rsplit("_", 1)[-1], 0) + hint = (row.get("word") or "").strip() or None + prev = canonical.get(eid) + if prev is None or ts > prev[0]: + canonical[eid] = (ts, key, hint) + if first is not None: + books[code] = first + return books, canonical + + +def import_books(w, data_dir, limit_books=None): + catalog_path = os.path.join(data_dir, "book-catalog.json") + catalog = [] + if os.path.exists(catalog_path): + with open(catalog_path, encoding="utf-8-sig") as f: + catalog = json.load(f) + by_code = {c.get("code"): c for c in catalog} + + books, canonical = scan_books(data_dir, limit_books) + book_ids = set() + phrase_pairs = [] + + for code, first in books.items(): + meta = by_code.get(code, {}) + book_id = _to_int(first.get("book_id")) + book_ids.add(book_id) + exam_id = exam_id_of(first.get("book_exam") or meta.get("exam")) + name = (first.get("book_name") or meta.get("name") or code).strip() + declared = _to_int(first.get("book_count"), meta.get("count")) + w.add("book", ( + book_id, + code, + name, + _text(meta.get("desc")), + exam_id, + declared, + _text(meta.get("cover")), + _text(meta.get("url")), + _to_int(meta.get("version")), + _to_int(meta.get("bookVersion")), + _text(meta.get("sortBy")), + _to_int(meta.get("sortTime")), + _to_int(meta.get("forSale")), + _to_int(meta.get("purchased")), + _to_int(meta.get("opt")), + _to_int(meta.get("src")), + _to_int(meta.get("type")), + _text(meta.get("labels")), + _text(meta.get("publishLog")), + json.dumps(meta.get("pkgPage"), ensure_ascii=False) if meta.get("pkgPage") is not None else None, + json.dumps(meta.get("externalLink"), ensure_ascii=False) if meta.get("externalLink") is not None else None, + 1, + )) + for pos, pid in enumerate(meta.get("phraseBookIds") or []): + pid = _to_int(pid) + if pid is not None: + phrase_pairs.append((book_id, pid, pos)) + + # 目录中存在但磁盘无清单的词书 + for code, meta in by_code.items(): + if code in books: + continue + book_id = _to_int(meta.get("id")) + if book_id is None: + continue + book_ids.add(book_id) + w.add("book", ( + book_id, code, (meta.get("name") or code), + _text(meta.get("desc")), + exam_id_of(meta.get("exam")), + _to_int(meta.get("count")), + _text(meta.get("cover")), _text(meta.get("url")), + _to_int(meta.get("version")), _to_int(meta.get("bookVersion")), + _text(meta.get("sortBy")), _to_int(meta.get("sortTime")), + _to_int(meta.get("forSale")), _to_int(meta.get("purchased")), + _to_int(meta.get("opt")), _to_int(meta.get("src")), + _to_int(meta.get("type")), _text(meta.get("labels")), + _text(meta.get("publishLog")), + json.dumps(meta.get("pkgPage"), ensure_ascii=False) if meta.get("pkgPage") is not None else None, + json.dumps(meta.get("externalLink"), ensure_ascii=False) if meta.get("externalLink") is not None else None, + 0, + )) + for pos, pid in enumerate(meta.get("phraseBookIds") or []): + pid = _to_int(pid) + if pid is not None: + phrase_pairs.append((book_id, pid, pos)) + + w.flush("book") + seen_pp = set() + for pair in phrase_pairs: + if pair[1] in book_ids and pair[:2] not in seen_pp: + seen_pp.add(pair[:2]) + w.add("book_phrase_book", pair) + w.flush("book_phrase_book") + print(f" book: {w.counts['book']} book_phrase_book: {w.counts['book_phrase_book']}") + return books, canonical + + +def import_book_entries(w, data_dir, limit_books=None): + pattern = os.path.join(data_dir, "wordbooks", "decrypted", "LangEasyLexisV3_*.csv") + files = sorted(glob.glob(pattern)) + if limit_books: + files = files[:limit_books] + seq = 0 + for path in files: + for r in read_csv(path): + seq += 1 + w.add("book_entry", ( + seq, + _to_int(r.get("book_id")), + _to_int(r.get("id")), + (r.get("word") or "").strip(), + _text((r.get("main_word") or "").strip()), + _to_int(r.get("seq_book")), + _to_int(r.get("seq_az")), + _to_int(r.get("seq_freq")), + _to_int(r.get("seq_rand")), + _text((r.get("zpk") or "").strip()), + )) + w.flush("book_entry") + print(f" book_entry: {w.counts['book_entry']}") + + +# --------------------------------------------------------------------------- +# 导入:zpk 词条 +# --------------------------------------------------------------------------- + +def _load_example(d): + raw = d.get("example") + if isinstance(raw, dict): + return raw + if isinstance(raw, str) and raw.strip(): + try: + return json.loads(raw) + except json.JSONDecodeError: + return {} + return {} + + +def _as_list(value): + if value is None: + return [] + if isinstance(value, (list, tuple)): + return list(value) + return [value] + + +def import_word_entries(w, data_dir, canonical): + zpk_dir = os.path.join(data_dir, "zpk", "decrypted") + total = len(canonical) + done = 0 + t0 = time.time() + for eid in sorted(canonical): + ts, key, hint = canonical[eid] + path = os.path.join(zpk_dir, key + ".json") + stem = key.split("_") + variant_code = _to_int(stem[2]) if len(stem) > 2 else None + if not os.path.exists(path): + # 缺失文件:写入占位词条,保证 book_entry 外键成立 + w.add("word_entry", ( + eid, hint or str(eid), (hint or str(eid)).lower(), + None, None, None, None, None, 0, 0, key, variant_code, ts)) + done += 1 + continue + try: + with open(path, encoding="utf-8") as f: + d = json.load(f) + except (json.JSONDecodeError, OSError) as exc: + print(f" ! 跳过损坏 JSON {key}: {exc}") + w.add("word_entry", ( + eid, hint or str(eid), (hint or str(eid)).lower(), + None, None, None, None, None, 0, 0, key, variant_code, ts)) + done += 1 + continue + + meta = d.get("meta") or {} + ex = _load_example(d) + word = (d.get("word") or hint or str(eid)).strip() + w.add("word_entry", ( + eid, word, (d.get("index_word") or word).strip(), + _text(d.get("interpret")), + _text(d.get("us_pron")), _text(d.get("uk_pron")), + _text(meta.get("data_version")), + exam_id_of(meta.get("book_exam")), + _to_int(d.get("book_occurrences"), 0), + 1 if ex.get("is_type_99") else 0, + key, variant_code, ts, + )) + + for pos, sp in enumerate(d.get("spell") or []): + w.add("word_spelling_variant", (eid, pos, str(sp))) + + # 义项 + 释义对象 + meaning_map = {} + relations = [] + for group in d.get("interpret_v2") or []: + pos_name = group.get("p") + for sense in group.get("d") or []: + sid = nid("sense") + w.add("word_sense", ( + sid, eid, _to_int(sense.get("id"), 0), pos_name, + _text(sense.get("i")), + _text(",".join(str(x) for x in _as_list(sense.get("s")))), + _to_int(sense.get("h")), + )) + seen_exam = set() + for code in _as_list(sense.get("t")): + xid = exam_id_of(code) + if xid is not None and xid not in seen_exam: + seen_exam.add(xid) + w.add("sense_exam", (sid, xid)) + for oid in _as_list(sense.get("oids")): + oid = _to_int(oid) + if oid is None: + continue + mid = nid("meaning") + w.add("sense_object", (mid, eid, oid, sid)) + meaning_map.setdefault(oid, mid) + for field, rel in (("synids", "synonym"), + ("antonymids", "antonym"), + ("refids", "reference")): + for apos, target in enumerate(_as_list(sense.get(field))): + target = _to_int(target) + if target is not None: + relations.append((sid, rel, apos, target)) + + for sid, rel, apos, target in relations: + mid = meaning_map.get(target) + if mid is not None: + w.add("sense_relation", (sid, rel, apos, mid)) + + # 短语 + phrase_map = {} + for pg in d.get("phrase") or []: + gtype = _to_int(pg.get("t"), 0) + for x in pg.get("p") or []: + pid = nid("phrase") + src_pid = _to_int(x.get("pid")) + if src_pid is not None: + phrase_map.setdefault(src_pid, pid) + oid = _to_int(x.get("oid")) + mid = meaning_map.get(oid) if oid is not None else None + w.add("phrase", ( + pid, eid, src_pid, mid, gtype, + _text(x.get("en")) or "", _text(x.get("cn")), + _text(x.get("fid")), _text(x.get("bd")), + )) + seen_exam = set() + for code in _as_list(x.get("exams")): + xid = exam_id_of(code) + if xid is not None and xid not in seen_exam: + seen_exam.add(xid) + w.add("phrase_exam", (pid, xid)) + for code, pids in (pg.get("pseqs") or {}).items(): + xid = exam_id_of(code) + if xid is None: + continue + for pos, src_pid in enumerate(pids): + pid = phrase_map.get(_to_int(src_pid)) + if pid is not None: + w.add("phrase_exam_order", (eid, xid, gtype, pos, pid)) + + # 例句块 + for block in ex.get("data") or []: + kind = _to_int(block.get("type"), 99) + info = block.get("i") or {} + exam_ids = [exam_id_of(c) for c in _as_list(block.get("exams"))] + exam_ids = list(dict.fromkeys(x for x in exam_ids if x is not None)) + source_codes = ",".join( + str(c) for c in _as_list(block.get("exams")) if c) + if str(kind) == "99": + egid = nid("example_group") + w.add("example_group", ( + egid, eid, None, 99, None, None, None, + None, None, None, None, source_codes)) + for sent in block.get("g") or []: + if not isinstance(sent, dict): + continue + w.add("example_sentence", _sentence_row(sent, egid, None, True)) + continue + + oid = _to_int(block.get("oid")) + mid = meaning_map.get(oid) if oid is not None else None + egid = nid("example_group") + w.add("example_group", ( + egid, eid, mid, kind, + _text(info.get("p")), _text(info.get("t")), _text(info.get("u")), + _text(info.get("e")), _text(info.get("c")), + _text(info.get("yu")), _text(info.get("yo")), source_codes, + )) + for xid in exam_ids: + w.add("example_group_exam", (egid, xid)) + for upos, usage in enumerate(block.get("g") or []): + uid = nid("usage") + src_uid = _to_int(usage.get("uid")) + pattern_id = phrase_map.get(src_uid) if src_uid else None + w.add("example_usage", ( + uid, egid, pattern_id, _text(usage.get("u")), upos)) + for sent in usage.get("s") or []: + if not isinstance(sent, dict): + continue + w.add("example_sentence", _sentence_row(sent, egid, uid, False)) + + # 考试例句 + for code, arr in (d.get("exam_sentences") or {}).items(): + xid = exam_id_of(code) + if xid is None: + continue + for pos, item in enumerate(arr or []): + if not isinstance(item, dict): + continue + w.add("entry_exam_sentence", ( + eid, xid, pos, + _text(item.get("en")) or "", + _text(item.get("cn")), _text(item.get("src")), + )) + + # 义项顺序 + for code, obj in (ex.get("oseqs") or {}).items(): + xid = exam_id_of(code) + if xid is None or not isinstance(obj, dict): + continue + for scope in ("core", "ext"): + seen_meaning = set() + for pos, oid in enumerate(obj.get(scope) or []): + mid = meaning_map.get(_to_int(oid)) + if mid is not None and mid not in seen_meaning: + seen_meaning.add(mid) + w.add("entry_exam_meaning_order", + (eid, xid, scope, pos, mid)) + + # 易混词 + seen_confuse = set() + cv = d.get("confuse_v2") + if cv: + for pos, item in enumerate(cv): + if not isinstance(item, dict): + continue + text = (item.get("w") or "").strip() + if not text or text in seen_confuse: + continue + seen_confuse.add(text) + w.add("confusable_word", ( + nid("confusable"), eid, pos, text, + _text(item.get("i")), _to_int(item.get("p")), _text(item.get("sp")))) + else: + conf = d.get("confuse") or [] + exps = d.get("confuse_exps") or [] + for pos, text in enumerate(conf): + text = str(text).strip() + if not text or text in seen_confuse: + continue + seen_confuse.add(text) + gloss = exps[pos] if pos < len(exps) else None + w.add("confusable_word", ( + nid("confusable"), eid, pos, text, _text(gloss), None, None)) + + done += 1 + if done % 2000 == 0: + elapsed = time.time() - t0 + print(f" zpk: {done}/{total} ({elapsed:.0f}s)") + w.flush_all() + print(f" word_entry: {w.counts['word_entry']} " + f"word_sense: {w.counts['word_sense']} " + f"phrase: {w.counts['phrase']} " + f"example_sentence: {w.counts['example_sentence']}") + + +def _sentence_row(sent, egid, uid, listening): + return ( + nid("sentence"), + _text((sent.get("sid") or "")), + _text((sent.get("eid") or "")), + egid, uid, _to_int(sent.get("s")), + _text(sent.get("e")) or "", + _text(sent.get("c")), + _text(sent.get("fid")), _text(sent.get("u")), _text(sent.get("i")), + _text(sent.get("b")), + 1 if listening else 0, + ) + + +# --------------------------------------------------------------------------- +# 导入:词根词族 +# --------------------------------------------------------------------------- + +def import_wordroot(w, data_dir): + base = os.path.join(data_dir, "wordroot-export") + + # 词素 + morpheme_map = {} + for typ, fname in (("prefix", "prefix.csv"), ("root", "root.csv"), + ("suffix", "suffix.csv")): + path = os.path.join(base, fname) + if not os.path.exists(path): + continue + for r in read_csv(path): + src = _to_int(r.get("id")) + if src is None: + continue + mid = nid("morpheme") + morpheme_map[(typ, src)] = mid + form, en, zh, note = parse_affix(r.get("display")) + w.add("morpheme", (mid, typ, src, form, r.get("display") or "", en, zh, note)) + w.flush("morpheme") + + def morph(typ, value): + src = _to_int(value) + if src is None or src <= 0: + return None + return morpheme_map.get((typ, src)) + + # 卡片 + card_path = os.path.join(base, "card_info.csv") + if os.path.exists(card_path): + for r in read_csv(card_path): + cid = _to_int(r.get("id")) + if cid is None: + continue + w.add("card", ( + cid, (r.get("word") or "").strip(), + _text(r.get("hightlight")), _text(r.get("memory")), + _text(r.get("example")), + morph("prefix", r.get("prefix")), + morph("root", r.get("root")), + morph("suffix", r.get("suffix")), + _to_int(r.get("default_f_id")), + _to_bool(r.get("hidden")), + )) + seen_exam = set() + for g in (r.get("grade") or "").split(","): + xid = _to_int(g) + if xid is not None and xid not in seen_exam: + seen_exam.add(xid) + w.add("card_exam", (cid, xid)) + w.flush("card") + + # 词族 + family_path = os.path.join(base, "word_family.csv") + family_ids = set() + family_rows = list(read_csv(family_path)) if os.path.exists(family_path) else [] + for r in family_rows: + fid = _to_int(r.get("id")) + if fid is not None: + family_ids.add(fid) + # 先插入自指根行,满足自外键 + ordered = [r for r in family_rows if _to_int(r.get("id")) == _to_int(r.get("top_id"))] + ordered += [r for r in family_rows if _to_int(r.get("id")) != _to_int(r.get("top_id"))] + for r in ordered: + fid = _to_int(r.get("id")) + if fid is None: + continue + w.add("word_family", ( + fid, _to_int(r.get("top_id"), fid), (r.get("word") or "").strip(), + _text(r.get("hightlight")), _text(r.get("interpret")), + morph("prefix", r.get("prefix")), morph("suffix", r.get("suffix")), + _to_int(r.get("group1")), _to_int(r.get("group2")), _to_int(r.get("group3")), + )) + seen_exam = set() + for g in (r.get("grade") or "").split(","): + xid = _to_int(g) + if xid is not None and xid not in seen_exam: + seen_exam.add(xid) + w.add("word_family_exam", (fid, xid)) + try: + j = json.loads(r.get("json") or "{}") + except json.JSONDecodeError: + j = {} + for key, value in j.items(): + xid = _to_int(key) + pos = _to_int(value) + if xid is not None and pos is not None: + w.add("word_family_exam_order", (fid, xid, pos)) + w.flush("word_family") + + # 变体 + var_path = os.path.join(base, "word_family_variant.csv") + variant_ids = [] + if os.path.exists(var_path): + for r in read_csv(var_path): + vid = _to_int(r.get("id")) + fid = _to_int(r.get("word_family_id")) + if vid is None or fid not in family_ids: + continue + variant_ids.append(vid) + w.add("word_family_variant", ( + vid, (r.get("variant") or "").strip(), fid, + _to_int(r.get("top_id"), fid), _text(r.get("hightlight")), + _text(r.get("interpret")), + morph("prefix", r.get("prefix")), morph("suffix", r.get("suffix")), + )) + try: + j = json.loads(r.get("json") or "{}") + except json.JSONDecodeError: + j = {} + for key, words in j.items(): + xid = _to_int(key) + if xid is None or not isinstance(words, list): + continue + for pos, word in enumerate(words): + text = str(word) + w.add("word_family_variant_member", ( + vid, xid, pos, text, 1 if text == "*" else 0)) + w.flush("word_family_variant") + + # 小组 + group_path = os.path.join(base, "word_family_mini_group.csv") + if os.path.exists(group_path): + for r in read_csv(group_path): + gid = _to_int(r.get("id")) + if gid is None: + continue + tokens = [x.strip() for x in (r.get("word_ids") or "").split(",") if x.strip()] + w.add("word_family_mini_group", (gid, len(tokens))) + seen = set() + for pos, tok in enumerate(tokens): + fid = _to_int(tok) + if fid is None or fid not in family_ids or fid in seen: + continue + seen.add(fid) + w.add("word_family_mini_group_member", (gid, fid, pos)) + w.flush("word_family_mini_group") + + # 卡片—分组 + wg_path = os.path.join(base, "word_group.csv") + if os.path.exists(wg_path): + for r in read_csv(wg_path): + lid = _to_int(r.get("id")) + cid = _to_int(r.get("card_info_id")) + if lid is None or cid is None: + continue + w.add("card_family_link", ( + lid, cid, _to_int(r.get("group_id"), 0), + _to_int(r.get("f_id")), _to_bool(r.get("hidden")), + )) + w.flush("card_family_link") + + print(f" morpheme: {w.counts['morpheme']} card: {w.counts['card']} " + f"word_family: {w.counts['word_family']} " + f"variant: {w.counts['word_family_variant']} " + f"mini_group: {w.counts['word_family_mini_group']}") + + +# --------------------------------------------------------------------------- +# 导入:近义词 +# --------------------------------------------------------------------------- + +def import_synonyms(w, data_dir): + base = os.path.join(data_dir, "wordroot-export") + words_path = os.path.join(base, "synonym_words.csv") + word_ids = set() + if os.path.exists(words_path): + for r in read_csv(words_path): + wid = _to_int(r.get("id")) + if wid is None: + continue + word_ids.add(wid) + w.add("synonym_word", (wid, (r.get("word") or "").strip())) + w.flush("synonym_word") + + data_path = os.path.join(base, "synonym_data.csv") + if not os.path.exists(data_path): + print(" synonym_data.csv 不存在,跳过") + return + for r in read_csv(data_path): + sid = _to_int(r.get("id")) + wid = _to_int(r.get("word_id")) + if sid is None or wid is None: + continue + w.add("synonym_sense", ( + sid, wid, _text(r.get("pos")), _text(r.get("interpret")))) + for column, xid in SYNONYM_EXAM_COLUMNS.items(): + raw = (r.get(column) or "").strip() + if not raw: + continue + try: + cell = json.loads(raw) + except json.JSONDecodeError: + continue + syn = [x for x in str(cell.get("syn") or "").split(",") if x.strip()] + ant = [x for x in str(cell.get("ant") or "").split(",") if x.strip()] + free = _to_int(cell.get("free")) + if free is not None and free not in word_ids: + free = None + syn_ids = [] + seen = set() + for x in syn: + v = _to_int(x) + if v in word_ids and v not in seen: + seen.add(v) + syn_ids.append(v) + w.add("synonym_group", (sid, xid, free, len(syn_ids))) + for pos, v in enumerate(syn_ids): + w.add("synonym_group_member", (sid, xid, pos, v)) + for pos, x in enumerate(ant): + v = _to_int(x) + if v in word_ids: + w.add("synonym_antonym", (sid, xid, pos, v)) + w.flush_all() + print(f" synonym_word: {w.counts['synonym_word']} " + f"synonym_sense: {w.counts['synonym_sense']} " + f"synonym_group: {w.counts['synonym_group']}") + + +# --------------------------------------------------------------------------- +# 主流程 +# --------------------------------------------------------------------------- + +def drop_schema(adapter): + if isinstance(adapter, SQLiteAdapter): + adapter.execute("PRAGMA foreign_keys=OFF") + for table in reversed(TABLES): + adapter.execute(f"DROP TABLE IF EXISTS {table}") + elif isinstance(adapter, MySQLAdapter): + adapter.execute("SET FOREIGN_KEY_CHECKS=0") + for table in reversed(TABLES): + adapter.execute(f"DROP TABLE IF EXISTS {table}") + adapter.execute("SET FOREIGN_KEY_CHECKS=1") + else: + for table in reversed(TABLES): + adapter.execute(f"DROP TABLE IF EXISTS {table} CASCADE") + + +def create_schema(adapter): + if isinstance(adapter, SQLiteAdapter): + script = sqlite_ddl() + elif isinstance(adapter, MySQLAdapter): + script = mysql_ddl() + else: + script = annotate_ddl(SCHEMA_PG, "pg") + adapter.execute_script(script) + if isinstance(adapter, PGAdapter): + for stmt in pg_comment_statements(): + adapter.execute(stmt) + + +def create_indexes(adapter): + for stmt in INDEXES.split(";"): + if stmt.strip(): + adapter.execute(stmt) + + +def print_counts(adapter): + print("\n导入结果:") + total = 0 + for table in TABLES: + try: + adapter.execute(f"SELECT COUNT(*) FROM {table}") + n = adapter.cur.fetchone()[0] + except Exception: + n = -1 + total += max(n, 0) + print(f" {table:<34} {n:>10,}") + print(f" {'TOTAL':<34} {total:>10,}") + + +def validate_sqlite(adapter): + try: + rows = adapter.cur.execute("PRAGMA foreign_key_check").fetchall() + except Exception as exc: + print(f" 外键校验失败: {exc}") + return + print(f"\n外键校验: {'通过' if not rows else f'发现 {len(rows)} 条违规'}") + for row in rows[:10]: + print(" ", row) + + +def main(argv=None): + here = os.path.dirname(os.path.abspath(__file__)) + default_data = os.path.join(here, "output") + if not os.path.isdir(default_data): + default_data = os.path.join(os.getcwd(), "output") + + ap = argparse.ArgumentParser(description="Lexis 词库一键建表并导入") + ap.add_argument("--db", choices=("sqlite", "postgres", "mysql"), default="sqlite") + ap.add_argument("--sqlite-path", default=os.path.join(here, "lexis.db")) + ap.add_argument("--dsn", help="PostgreSQL / MySQL DSN") + ap.add_argument("--data-dir", default=default_data) + ap.add_argument("--recreate", action="store_true", help="删除并重建所有表") + ap.add_argument("--batch-size", type=int, default=10000) + ap.add_argument("--limit-books", type=int, default=None, + help="仅导入前 N 本词书(调试用)") + args = ap.parse_args(argv) + + data_dir = os.path.abspath(args.data_dir) + if not os.path.isdir(data_dir): + raise SystemExit(f"数据目录不存在: {data_dir}") + + nid.value.clear() + + print(f"数据目录: {data_dir}") + if args.db == "sqlite": + adapter = SQLiteAdapter(args.sqlite_path) + print(f"目标: SQLite {os.path.abspath(args.sqlite_path)}") + elif args.db == "mysql": + if not args.dsn: + raise SystemExit("--db mysql 时必须提供 --dsn") + adapter = MySQLAdapter(args.dsn) + print("目标: MySQL") + else: + if not args.dsn: + raise SystemExit("--db postgres 时必须提供 --dsn") + adapter = PGAdapter(args.dsn) + print("目标: PostgreSQL") + + existing = adapter.table_names() & set(TABLES) + if existing and not args.recreate: + raise SystemExit( + "数据库中已存在本模型的表;如需重建请加 --recreate") + if existing and args.recreate: + print("删除旧表...") + drop_schema(adapter) + + print("建表...") + create_schema(adapter) + if isinstance(adapter, MySQLAdapter): + adapter.execute("SET FOREIGN_KEY_CHECKS=0") + adapter.execute("SET UNIQUE_CHECKS=0") + adapter.begin() + + w = Writer(adapter, batch_size=args.batch_size) + t0 = time.time() + try: + print("导入考试维表...") + import_exams(w, data_dir) + + print("扫描词书清单...") + books, canonical = import_books(w, data_dir, args.limit_books) + print(f" 词书 {len(books)} 本,词条 {len(canonical)} 个") + + print("导入词条词典(每个 entry 取最新 zpk 版本)...") + import_word_entries(w, data_dir, canonical) + + print("导入词书词条...") + import_book_entries(w, data_dir, args.limit_books) + + print("导入词根词族...") + import_wordroot(w, data_dir) + + print("导入近反义词...") + import_synonyms(w, data_dir) + + adapter.commit() + except Exception: + adapter.rollback() + raise + + if isinstance(adapter, MySQLAdapter): + adapter.execute("SET FOREIGN_KEY_CHECKS=1") + adapter.execute("SET UNIQUE_CHECKS=1") + + print(f"\n导入耗时 {time.time() - t0:.1f}s,创建索引...") + adapter.begin() + create_indexes(adapter) + adapter.commit() + + print_counts(adapter) + if isinstance(adapter, SQLiteAdapter): + validate_sqlite(adapter) + adapter.close() + print("\n完成。") + + +if __name__ == "__main__": + main() diff --git a/importer/run.py b/importer/run.py new file mode 100644 index 0000000..810e9dc --- /dev/null +++ b/importer/run.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""逐个调用 Lexis 三步导出流程。 + + 步骤1:下载词书目录 + 步骤2:逐个下载所有词书、解密并导出单词 + 步骤3:逐个下载所有词书所有单词的 ZPK 并解密为 JSONL + +所有产物统一输出到 output/,每步产出均为可读明文。 + +用法: + python run.py + python run.py --wordbook-workers 16 --zpk-workers 96 --zpk-timeout 6 + +任一步骤失败(非零退出码)即停止,不再执行后续步骤。 +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") +if hasattr(sys.stderr, "reconfigure"): + sys.stderr.reconfigure(encoding="utf-8", errors="replace") + +ROOT = Path(__file__).resolve().parent + + +def run_step(name: str, step_args: list[str]) -> int: + print(f"\n{'=' * 64}\n[INFO] {name}:status=started\n{'=' * 64}", flush=True) + process = subprocess.Popen([sys.executable, *step_args], cwd=ROOT) + try: + returncode = process.wait() + except KeyboardInterrupt: + print(f"\n[{name}] 收到 Ctrl+C,正在终止子进程...", file=sys.stderr, flush=True) + process.terminate() + try: + returncode = process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + returncode = process.wait() + return 130 + if returncode != 0: + print(f"\n[{name}] 退出码 {returncode}", file=sys.stderr, flush=True) + return returncode + + +def main() -> int: + parser = argparse.ArgumentParser(description="逐个执行 Lexis 三步导出流程") + parser.add_argument("--wordbook-workers", type=int, default=16, help="步骤2 下载词书的并发数") + parser.add_argument("--zpk-workers", type=int, default=96, help="步骤3 下载 ZPK 的并发数") + parser.add_argument("--zpk-timeout", type=float, default=6, help="步骤3 单个 CDN 请求超时秒数") + parser.add_argument("--progress-every", type=int, default=1000, help="步骤3 进度打印间隔") + args = parser.parse_args() + + steps = [ + ("步骤 1/3:下载词书目录", ["src/fetch-book-catalog.py"]), + ( + "步骤 2/3:逐个下载所有词书(下载 -> 解密 -> 导出单词)", + ["src/download-wordbooks.py", "--workers", str(args.wordbook_workers)], + ), + ( + "步骤 3/3:逐个下载所有词书所有单词的 ZPK 并解密为 JSONL", + [ + "src/download-zpk.py", + "--workers", str(args.zpk_workers), + "--timeout", str(args.zpk_timeout), + "--progress-every", str(args.progress_every), + ], + ), + ] + + for name, step_args in steps: + code = run_step(name, step_args) + if code != 0: + print(f"\n流程中断:'{name}' 失败(退出码 {code})", file=sys.stderr) + return code + + print("\n[INFO] 全部流程执行完成:output=output/", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/importer/src/decrypt-sqlcipher2-db.py b/importer/src/decrypt-sqlcipher2-db.py new file mode 100644 index 0000000..aaf16b5 --- /dev/null +++ b/importer/src/decrypt-sqlcipher2-db.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +from pathlib import Path + +from Crypto.Cipher import AES + + +KEY = b"arsenal iscool" +PAGE_SIZE = 1024 +RESERVE = 48 +KDF_ITER = 4000 +SQLITE_HEADER = b"SQLite format 3\x00" + + +def derive_key(salt: bytes) -> bytes: + return hashlib.pbkdf2_hmac("sha1", KEY, salt, KDF_ITER, 32) + + +def decrypt_db(src: Path, dst: Path) -> None: + blob = src.read_bytes() + if len(blob) % PAGE_SIZE: + raise ValueError(f"input size is not aligned to {PAGE_SIZE}: {src}") + + key = derive_key(blob[:16]) + out = bytearray() + pages = len(blob) // PAGE_SIZE + for page_no in range(pages): + page = blob[page_no * PAGE_SIZE : (page_no + 1) * PAGE_SIZE] + iv = page[PAGE_SIZE - RESERVE : PAGE_SIZE - RESERVE + 16] + if page_no == 0: + enc_start = 16 + prefix = SQLITE_HEADER + else: + enc_start = 0 + prefix = b"" + enc_end = PAGE_SIZE - RESERVE + dec = AES.new(key, AES.MODE_CBC, iv).decrypt(page[enc_start:enc_end]) + plain = prefix + dec + if len(plain) > PAGE_SIZE: + raise ValueError("decrypted page overflow") + out.extend(plain) + out.extend(b"\x00" * (PAGE_SIZE - len(plain))) + + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_bytes(out) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("src", type=Path) + parser.add_argument("dst", type=Path) + args = parser.parse_args() + decrypt_db(args.src, args.dst) + print(args.dst) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/importer/src/decrypt-zpk-aes.py b/importer/src/decrypt-zpk-aes.py new file mode 100644 index 0000000..5e74ea8 --- /dev/null +++ b/importer/src/decrypt-zpk-aes.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Decrypt Lexis .zpk files offline. + +ZPK format: + outer protobuf: field 1 length-delimited AES ciphertext + cipher: AES-256-CBC + PKCS7/PKCS5 padding + key: md5(word).hexdigest().encode("ascii") # 32 bytes + iv: b"IscoolInitVector" + +The word -> zpk mapping can be supplied by output/unique-zpk-index.csv. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +from pathlib import Path + +from Crypto.Cipher import AES +from Crypto.Util.Padding import unpad + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_INDEX = ROOT / "output" / "unique-zpk-index.csv" +DEFAULT_ZPK_DIR = ROOT / "output" / "zpk" / "original" +DEFAULT_OUTPUT = ROOT / "output" / "zpk" / "decrypted" / "zpk-decrypted.jsonl" +IV = b"IscoolInitVector" + + +def read_varint(data: bytes, offset: int) -> tuple[int, int]: + value = 0 + shift = 0 + while offset < len(data): + b = data[offset] + offset += 1 + value |= (b & 0x7F) << shift + if b < 0x80: + return value, offset + shift += 7 + raise ValueError("unterminated varint") + + +def read_field1_payload(data: bytes) -> bytes: + if not data or data[0] != 0x0A: + raise ValueError("bad zpk outer protobuf header") + size, offset = read_varint(data, 1) + payload = data[offset : offset + size] + if len(payload) != size: + raise ValueError(f"truncated payload: expected={size} actual={len(payload)}") + return payload + + +def decrypt_zpk_bytes(raw: bytes, word: str) -> dict: + ciphertext = read_field1_payload(raw) + key = hashlib.md5(word.encode("utf-8")).hexdigest().encode("ascii") + plaintext = AES.new(key, AES.MODE_CBC, IV).decrypt(ciphertext) + plaintext = unpad(plaintext, 16) + inner = read_field1_payload(plaintext) + return json.loads(inner.decode("utf-8")) + + +def load_index(path: Path) -> list[dict[str, str]]: + with path.open("r", encoding="utf-8-sig", newline="") as fh: + return [row for row in csv.DictReader(fh) if row.get("zpk") and row.get("first_word")] + + +def flatten(obj: dict) -> dict: + return { + "word": obj.get("word", ""), + "us_pron": obj.get("us_pron", ""), + "uk_pron": obj.get("uk_pron", ""), + "interpret": obj.get("interpret", ""), + "definition_count": sum(len(g.get("d") or []) for g in (obj.get("interpret_v2") or []) if isinstance(g, dict)), + "phrase_count": sum(len(g.get("p") or []) for g in (obj.get("phrase") or []) if isinstance(g, dict)), + "raw": obj, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--index", type=Path, default=DEFAULT_INDEX) + parser.add_argument("--zpk-dir", type=Path, default=DEFAULT_ZPK_DIR) + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + parser.add_argument("--limit", type=int, default=0) + parser.add_argument("--raw", action="store_true", help="include full raw JSON object") + args = parser.parse_args() + + rows = load_index(args.index) + if args.limit: + rows = rows[: args.limit] + + args.output.parent.mkdir(parents=True, exist_ok=True) + ok = missing = failed = 0 + with args.output.open("w", encoding="utf-8", newline="\n") as out: + for row in rows: + zpk = row["zpk"] + word = row["first_word"] + path = args.zpk_dir / f"{zpk}.zpk" + if not path.exists(): + missing += 1 + continue + try: + obj = decrypt_zpk_bytes(path.read_bytes(), word) + rec = dict(obj) + rec.update( + { + "zpk": zpk, + "zpk_path": str(path), + "index_word": word, + "book_occurrences": row.get("book_occurrences", ""), + } + ) + if args.raw: + rec["raw"] = obj + out.write(json.dumps(rec, ensure_ascii=False, separators=(",", ":")) + "\n") + ok += 1 + except Exception as exc: + failed += 1 + if failed <= 20: + print(f"[fail] {zpk} word={word}: {exc}") + + print(f"ok={ok} missing={missing} failed={failed} -> {args.output}") + return 0 if failed == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/importer/src/download-wordbooks.py b/importer/src/download-wordbooks.py new file mode 100644 index 0000000..37672bd --- /dev/null +++ b/importer/src/download-wordbooks.py @@ -0,0 +1,459 @@ +#!/usr/bin/env python3 +"""步骤2:逐个下载所有词书,解包、解密 SQLCipher2 DB 并导出单词与 zpk 字段。 + +每本词书走完整链路:下载 zip -> 校验 MD5 -> 解包 -> 解密 -> 导出单词 CSV。 +解密逻辑复用独立工具 decrypt-sqlcipher2-db.py(可单独调用)。 + +已下载的 DB 自动跳过(可用 --force 重下),已解密的 DB 按 mtime 复用(可用 --force-decrypt 重解)。 + +输出(默认统一到 output/): +- output/databases/LangEasyLexisV3_.db 原始加密 DB(中间缓存) +- output/decrypted-databases/LangEasyLexisV3_.sqlite 解密后 DB(可读) +- output/wordbook-words/.csv 每本词书单词(可读) +- output/all-wordbook-words.csv 全量合并词表(可读,步骤3 的输入) +- output/wordbook-download-report.csv 下载报告 +- output/wordbook-word-summary.csv 导出报告 +""" + +from __future__ import annotations + +import argparse +import base64 +import csv +import hashlib +import importlib.util +import json +import os +import re +import sqlite3 +import sys +import tempfile +import time +import zipfile +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Iterable +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + + +ROOT = Path(__file__).resolve().parents[1] +OUT_DIR = ROOT / "output" +DEFAULT_CATALOG = OUT_DIR / "book-catalog.json" +WORDBOOK_DIR = OUT_DIR / "wordbooks" +DB_DIR = WORDBOOK_DIR / "original" +DECRYPTED_DIR = WORDBOOK_DIR / "decrypted" + +DECRYPTOR_PATH = ROOT / "src" / "decrypt-sqlcipher2-db.py" + +BASE_STATIC_URLS = [ + "http://static.beingfine.cn", + "https://hwstatic.beingfine.cn", + "https://ccastatic.beingfine.cn", +] +REFERER = "https://app.bbdc.cn" +USER_AGENT = "LangEasyLexis/5.11.1" + +WORD_FIELDS = ( + "book_id", + "book_code", + "book_name", + "book_count", + "book_exam", + "book_version", + "id", + "word", + "main_word", + "seq_book", + "seq_az", + "seq_freq", + "seq_rand", + "zpk", +) + +SUMMARY_FIELDS = ( + "book_id", + "book_code", + "book_name", + "book_count", + "exported_count", + "status", + "table", + "local_db", + "decrypted_db", + "export_csv", + "error", +) + +DOWNLOAD_REPORT_FIELDS = ("code", "status", "bytes", "url") + + +def load_decrypt_db(): + """加载独立解密工具 decrypt-sqlcipher2-db.py 的 decrypt_db 函数。""" + spec = importlib.util.spec_from_file_location("decrypt_sqlcipher2_db", DECRYPTOR_PATH) + if spec is None or spec.loader is None: + raise RuntimeError(f"Cannot load decryptor: {DECRYPTOR_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.decrypt_db + + +def load_books(path: Path) -> list[dict]: + with path.open("r", encoding="utf-8") as f: + data = json.load(f) + if isinstance(data, dict) and isinstance(data.get("books"), list): + return data["books"] + if isinstance(data, list): + return data + raise ValueError(f"Unsupported catalog shape: {path}") + + +def book_url(book: dict, base_url: str) -> str: + code = book["code"] + version = str(book.get("version") or 0) + db_name = f"LangEasyLexisV3_{code}.db" + return f"{base_url}/dicts/v3/{code}/{version}/{db_name}.zip" + + +def request_url(url: str): + req = Request(url, headers={"Referer": REFERER, "User-Agent": USER_AGENT}) + return urlopen(req, timeout=45) + + +def md5_b64(path: Path) -> str: + h = hashlib.md5() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return base64.b64encode(h.digest()).decode("ascii") + + +def safe_code_filename(code: object) -> str: + text = str(code or "UNKNOWN") + return re.sub(r"[^A-Za-z0-9_.-]+", "_", text).strip("._") or "UNKNOWN" + + +def quote_identifier(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +def list_tables(conn: sqlite3.Connection) -> list[str]: + return [ + row[0] + for row in conn.execute( + "select name from sqlite_master where type='table' and name not like 'sqlite_%' order by name" + ) + ] + + +def table_columns(conn: sqlite3.Connection, table: str) -> set[str]: + return {row[1] for row in conn.execute(f"pragma table_info({quote_identifier(table)})")} + + +def choose_word_table(conn: sqlite3.Connection, code: str) -> str: + tables = list_tables(conn) + preferred = f"LangEasyLexisV3_{code}" + if preferred in tables: + return preferred + + for table in tables: + if {"id", "word"}.issubset(table_columns(conn, table)): + return table + + raise RuntimeError(f"no word table found; tables={tables}") + + +def iter_words(conn: sqlite3.Connection, table: str) -> Iterable[sqlite3.Row]: + conn.row_factory = sqlite3.Row + cols = table_columns(conn, table) + required = {"id", "word", "main_word", "seq_book", "seq_az", "seq_freq", "seq_rand", "zpk"} + missing = sorted(required - cols) + if missing: + raise RuntimeError(f"table {table} missing columns: {missing}") + + query = ( + f"select id, word, main_word, seq_book, seq_az, seq_freq, seq_rand, zpk " + f"from {quote_identifier(table)} order by seq_book, id" + ) + return conn.execute(query) + + +def download_book_db(book: dict, force: bool) -> dict: + """下载单本词书 DB zip 并解包,返回下载状态 dict。""" + code = book["code"] + db_path = DB_DIR / f"LangEasyLexisV3_{code}.db" + + if db_path.exists() and not force: + return {"status": "skip-db-exists", "url": "", "bytes": db_path.stat().st_size} + + last_error = "" + for base in BASE_STATIC_URLS: + url = book_url(book, base) + try: + with request_url(url) as resp: + status = getattr(resp, "status", 200) + if status != 200: + last_error = f"http-{status}" + continue + + tmp_file = tempfile.NamedTemporaryFile(prefix=f"lexis-{code}-", suffix=".zip", delete=False) + tmp_path = Path(tmp_file.name) + with tmp_file as out: + while True: + chunk = resp.read(1024 * 1024) + if not chunk: + break + out.write(chunk) + + content_md5 = resp.headers.get("Content-MD5") + if content_md5 and md5_b64(tmp_path) != content_md5: + tmp_path.unlink(missing_ok=True) + last_error = "md5-mismatch" + continue + + DB_DIR.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(tmp_path) as zf: + names = zf.namelist() + wanted = f"LangEasyLexisV3_{code}.db" + member = wanted if wanted in names else names[0] + zf.extract(member, DB_DIR) + extracted = DB_DIR / member + if extracted != db_path: + extracted.replace(db_path) + + tmp_path.unlink(missing_ok=True) + return {"status": "downloaded", "url": url, "bytes": db_path.stat().st_size} + except HTTPError as e: + last_error = f"http-{e.code}" + except (URLError, TimeoutError, zipfile.BadZipFile, OSError) as e: + last_error = type(e).__name__ + finally: + if "tmp_path" in locals(): + tmp_path.unlink(missing_ok=True) + + return {"status": f"failed:{last_error}", "url": "", "bytes": 0} + + +def export_book_words(row: dict, decrypt_db, force_decrypt: bool) -> dict: + """解密并导出一本词书的单词,返回导出 summary dict。""" + code = str(row.get("code") or "") + src = DB_DIR / f"LangEasyLexisV3_{code}.db" + if not src.exists(): + return summary(row, "missing_db", "", src, Path(), Path(), 0, f"not found: {src}") + + export_csv = DECRYPTED_DIR / f"LangEasyLexisV3_{safe_code_filename(code)}.csv" + DECRYPTED_DIR.mkdir(parents=True, exist_ok=True) + + # 解密 DB:已存在且比源新则复用 + # SQLite 仅作为临时解密载体,不作为 output 产物。 + # Windows 可能在 SQLite 连接关闭后仍短暂持有文件句柄;清理失败不能影响导出结果。 + with tempfile.TemporaryDirectory(prefix="lexis-decrypted-", ignore_cleanup_errors=True) as temp_dir: + decrypted = Path(temp_dir) / src.with_suffix(".sqlite").name + decrypt_db(src, decrypted) + with sqlite3.connect(decrypted) as conn: + table = choose_word_table(conn, code) + count = 0 + with export_csv.open("w", encoding="utf-8-sig", newline="") as fh: + writer = csv.DictWriter(fh, fieldnames=WORD_FIELDS) + writer.writeheader() + for word_row in iter_words(conn, table): + writer.writerow( + { + "book_id": row.get("id", ""), + "book_code": code, + "book_name": row.get("name", ""), + "book_count": row.get("count", ""), + "book_exam": row.get("exam", ""), + "book_version": row.get("version", ""), + "id": word_row["id"], + "word": word_row["word"], + "main_word": word_row["main_word"], + "seq_book": word_row["seq_book"], + "seq_az": word_row["seq_az"], + "seq_freq": word_row["seq_freq"], + "seq_rand": word_row["seq_rand"], + "zpk": word_row["zpk"], + } + ) + count += 1 + + return summary(row, "ok", table, src, decrypted, export_csv, count, "") + + +def summary( + row: dict, + status: str, + table: str, + src: Path, + decrypted: Path, + export_csv: Path, + exported_count: int, + error: str, +) -> dict: + return { + "book_id": row.get("id", ""), + "book_code": row.get("code", ""), + "book_name": row.get("name", ""), + "book_count": row.get("count", ""), + "exported_count": exported_count, + "status": status, + "table": table, + "local_db": str(src) if src else "", + "decrypted_db": str(decrypted) if decrypted else "", + "export_csv": str(export_csv) if export_csv else "", + "error": error, + } + + +def process_one(book: dict, decrypt_db, force_download: bool, force_decrypt: bool) -> dict: + """处理一本词书:下载 + 解密 + 导出,返回合并结果 dict。""" + code = book["code"] + result = { + "code": code, + "dl_status": "", + "dl_url": "", + "dl_bytes": 0, + "export": None, + } + + if book.get("url") == "*": + result["dl_status"] = "skip-virtual-book" + return result + + dl = download_book_db(book, force_download) + result["dl_status"] = dl["status"] + result["dl_url"] = dl["url"] + result["dl_bytes"] = dl["bytes"] + + if dl["status"] not in {"downloaded", "skip-db-exists"}: + result["export"] = summary(book, "download_failed", "", DB_DIR / f"LangEasyLexisV3_{code}.db", Path(), Path(), 0, dl["status"]) + return result + + result["export"] = export_book_words(book, decrypt_db, force_decrypt) + return result + + +def write_download_report(results: list[dict], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="", encoding="utf-8-sig") as f: + writer = csv.DictWriter(f, fieldnames=DOWNLOAD_REPORT_FIELDS) + writer.writeheader() + for r in results: + writer.writerow( + {"code": r["code"], "status": r["dl_status"], "bytes": r["dl_bytes"], "url": r["dl_url"]} + ) + + +def write_summary(summaries: list[dict], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8-sig", newline="") as fh: + writer = csv.DictWriter(fh, fieldnames=SUMMARY_FIELDS) + writer.writeheader() + writer.writerows(summaries) + + +def write_combined(summaries: list[dict], path: Path) -> int: + total = 0 + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8-sig", newline="") as out_fh: + writer = csv.DictWriter(out_fh, fieldnames=WORD_FIELDS) + writer.writeheader() + for item in summaries: + if item["status"] != "ok": + continue + with Path(item["export_csv"]).open("r", encoding="utf-8-sig", newline="") as in_fh: + reader = csv.DictReader(in_fh) + for row in reader: + writer.writerow(row) + total += 1 + return total + + +def main() -> int: + parser = argparse.ArgumentParser(description="逐个下载所有词书:下载 -> 解密 -> 导出单词") + parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) + parser.add_argument("--force", action="store_true", help="重新下载已存在的词书 DB") + parser.add_argument("--force-decrypt", action="store_true", help="重新解密已存在的解密 DB") + parser.add_argument("--limit", type=int, default=0, help="最多处理 N 本词书(冒烟测试)") + parser.add_argument("--sleep", type=float, default=0.0, help="单 worker 模式下每本之间 sleep 秒数") + parser.add_argument("--workers", type=int, default=1, help="并行下载词书的 worker 数") + args = parser.parse_args() + + if not args.catalog.exists(): + print(f"词书目录不存在:{args.catalog},请先运行 src/fetch-book-catalog.py", file=sys.stderr) + return 1 + + decrypt_db = load_decrypt_db() + books = [row for row in load_books(args.catalog) if row.get("url") != "*"] + if args.limit: + books = books[: args.limit] + + total = len(books) + results: list[dict] = [] + done = 0 + workers = max(1, args.workers) + + if workers == 1: + for book in books: + result = process_one(book, decrypt_db, args.force, args.force_decrypt) + results.append(result) + done += 1 + exp = result["export"] + print( + f"[INFO] [步骤 2/3] 处理进度:current={done}/{total},percent={done / total * 100:.1f}%," + f"code={result['code']},download={result['dl_status']}," + f"decrypt={exp['status'] if exp else '-'},words={exp['exported_count'] if exp else '-'}", + flush=True, + ) + if args.sleep: + time.sleep(args.sleep) + else: + executor = ThreadPoolExecutor(max_workers=workers) + futures = [executor.submit(process_one, book, decrypt_db, args.force, args.force_decrypt) for book in books] + try: + for future in as_completed(futures): + result = future.result() + results.append(result) + done += 1 + exp = result["export"] + print( + f"[INFO] [步骤 2/3] 处理进度:current={done}/{total},percent={done / total * 100:.1f}%," + f"code={result['code']},download={result['dl_status']}," + f"decrypt={exp['status'] if exp else '-'},words={exp['exported_count'] if exp else '-'}", + flush=True, + ) + except KeyboardInterrupt: + for future in futures: + future.cancel() + executor.shutdown(wait=False, cancel_futures=True) + print("\n收到 Ctrl+C,已取消未开始的词书任务。", file=sys.stderr, flush=True) + os._exit(130) + else: + executor.shutdown(wait=True) + + summaries = [r["export"] for r in results if r["export"] is not None] + + ok_dl = sum(1 for r in results if r["dl_status"] in {"downloaded", "skip-db-exists"}) + ok_export = sum(1 for s in summaries if s["status"] == "ok") + failed_dl = len(results) - ok_dl + failed_export = len(summaries) - ok_export + exported_total = sum(int(s["exported_count"] or 0) for s in summaries) + + print(f"[INFO] [步骤 2/3] 处理完成:total={len(results)},success={ok_export},failed={failed_export},words={exported_total}") + failures = [] + for result in results: + export = result.get("export") or {} + if result["dl_status"] not in {"downloaded", "skip-db-exists"}: + failures.append(f"{result['code']}: 下载 {result['dl_status']}") + elif export.get("status") != "ok": + failures.append(f"{result['code']}: 解密/导出 {export.get('error') or export.get('status')}") + for failure in failures[:20]: + print(f"失败项:{failure}", file=sys.stderr) + if len(failures) > 20: + print(f"失败项:另有 {len(failures) - 20} 项未显示", file=sys.stderr) + # 单本失败不阻断后续流程;步骤 3 继续处理已成功导出的词书。 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/importer/src/download-zpk.py b/importer/src/download-zpk.py new file mode 100644 index 0000000..fbd3f03 --- /dev/null +++ b/importer/src/download-zpk.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +"""步骤3:从全量词表去重后,逐个下载所有词书所有单词的 ZPK 并解密为可读 JSONL。 + +解密逻辑复用独立工具 decrypt-zpk-aes.py(可单独调用)。 + +输入:output/all-wordbook-words.csv(步骤2 产出,含 word、zpk 字段) +去重:同一 zpk 名只下载一次,记录首个出现的 word 作为 first_word(ZPK 解密 key = md5(word))。 + +CDN:{base}/r/{zpkName}.zpk + +输出(默认统一到 output/): +- output/unique-zpk-index.csv 去重后的 zpk 索引(可读) +- output/zpk-cache/0/{zpkName}.zpk 加密 ZPK(中间缓存) +- output/zpk-jsonl/zpk-decrypted.jsonl 解密后 JSONL(可读,最终产出) +- output/zpk-download-report.csv 下载/解密报告 + +已解密的 zpk(记录在 zpk-decrypted.jsonl 中)自动跳过;已下载的 zpk 自动跳过下载,仅重新解密。 +""" + +from __future__ import annotations + +import argparse +import base64 +import csv +import hashlib +import http.client +import importlib.util +import json +import queue +import sys +import threading +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlparse + +ROOT = Path(__file__).resolve().parents[1] +OUT_DIR = ROOT / "output" +WORDS_CSV = OUT_DIR / "wordbooks" / "decrypted" +ZPK_DIR = OUT_DIR / "zpk" / "original" +ZPK_JSONL = OUT_DIR / "zpk" / "decrypted" + +DECRYPTOR_PATH = ROOT / "src" / "decrypt-zpk-aes.py" + +BASE_STATIC_URLS = [ + "http://static.beingfine.cn", + "https://hwstatic.beingfine.cn", + "https://ccastatic.beingfine.cn", +] +REFERER = "https://app.bbdc.cn" +USER_AGENT = "LangEasyLexis/5.11.1" +REQUEST_TIMEOUT = 8.0 +STOP_EVENT = threading.Event() + +UNIQUE_FIELDS = ( + "zpk", + "zpk_bid", + "zpk_word_id", + "zpk_version", + "zpk_updated_unix", + "zpk_updated_utc", + "first_word", + "book_occurrences", +) + + +@dataclass(frozen=True) +class CdnEndpoint: + base_url: str + scheme: str + host: str + port: int | None + + @classmethod + def parse(cls, base_url: str) -> "CdnEndpoint": + parsed = urlparse(base_url) + return cls(base_url=base_url, scheme=parsed.scheme, host=parsed.hostname or "", port=parsed.port) + + +class CdnClient: + def __init__(self, endpoints: list[CdnEndpoint], timeout: float) -> None: + self.endpoints = endpoints + self.timeout = timeout + self.connections: dict[str, http.client.HTTPConnection] = {} + + def close(self) -> None: + for conn in self.connections.values(): + conn.close() + self.connections.clear() + + def _conn(self, endpoint: CdnEndpoint) -> http.client.HTTPConnection: + key = endpoint.base_url + conn = self.connections.get(key) + if conn is not None: + return conn + conn_cls = http.client.HTTPSConnection if endpoint.scheme == "https" else http.client.HTTPConnection + conn = conn_cls(endpoint.host, endpoint.port, timeout=self.timeout) + self.connections[key] = conn + return conn + + def get(self, endpoint: CdnEndpoint, name: str) -> tuple[str, http.client.HTTPResponse]: + url_path = f"/r/{name}.zpk" + headers = {"Connection": "keep-alive", "Referer": REFERER, "User-Agent": USER_AGENT} + conn = self._conn(endpoint) + try: + conn.request("GET", url_path, headers=headers) + return f"{endpoint.base_url}{url_path}", conn.getresponse() + except (http.client.HTTPException, OSError): + conn.close() + self.connections.pop(endpoint.base_url, None) + raise + + def drop(self, endpoint: CdnEndpoint) -> None: + conn = self.connections.pop(endpoint.base_url, None) + if conn is not None: + conn.close() + + +class JsonlSink: + """线程安全地为每个 ZPK 写一个独立 JSON 文件。""" + + def __init__(self, path: Path) -> None: + self.lock = threading.Lock() + path.parent.mkdir(parents=True, exist_ok=True) + self.dir = path + self.dir.mkdir(parents=True, exist_ok=True) + + def write(self, name: str, line: str) -> None: + with self.lock: + target = self.dir / f"{name}.json" + target.write_text(line + "\n", encoding="utf-8") + + def close(self) -> None: + return + + +def load_decrypt_zpk(): + """加载独立解密工具 decrypt-zpk-aes.py 的解密函数。""" + spec = importlib.util.spec_from_file_location("decrypt_zpk_aes", DECRYPTOR_PATH) + if spec is None or spec.loader is None: + raise RuntimeError(f"Cannot load decryptor: {DECRYPTOR_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.decrypt_zpk_bytes, module.flatten + + +def build_endpoints(cdn_mode: str, worker_index: int = 0) -> list[CdnEndpoint]: + base_urls = BASE_STATIC_URLS + if cdn_mode == "spread" and base_urls: + offset = worker_index % len(base_urls) + base_urls = base_urls[offset:] + base_urls[:offset] + return [CdnEndpoint.parse(base) for base in base_urls] + + +def parse_zpk(zpk: str) -> dict[str, str]: + parts = (zpk or "").split("_") + if len(parts) != 4 or not all(part.isdigit() for part in parts): + return { + "zpk_bid": "", + "zpk_word_id": "", + "zpk_version": "", + "zpk_updated_unix": "", + "zpk_updated_utc": "", + } + + updated = int(parts[3]) + try: + updated_utc = datetime.fromtimestamp(updated, tz=timezone.utc).isoformat().replace("+00:00", "Z") + except (OverflowError, OSError, ValueError): + updated_utc = "" + + return { + "zpk_bid": parts[0], + "zpk_word_id": parts[1], + "zpk_version": parts[2], + "zpk_updated_unix": parts[3], + "zpk_updated_utc": updated_utc, + } + + +def build_unique_index(words_path: Path) -> list[dict[str, str]]: + """从全量词表在内存中去重 ZPK 名,不生成索引文件。""" + unique: dict[str, dict[str, str | int]] = {} + paths = [words_path] if words_path.is_file() else sorted(words_path.glob("*.csv")) + for path in paths: + with path.open("r", encoding="utf-8-sig", newline="") as fh: + reader = csv.DictReader(fh) + for row in reader: + zpk = row.get("zpk", "") + if not zpk: + continue + if zpk not in unique: + unique[zpk] = { + "zpk": zpk, + **parse_zpk(zpk), + "first_word": row.get("word", ""), + "book_occurrences": 0, + } + unique[zpk]["book_occurrences"] = int(unique[zpk]["book_occurrences"]) + 1 + + return list(unique.values()) + + +def load_already_decrypted(path: Path) -> set[str]: + """读取现有 JSONL 中已解密的 zpk 名集合,用于断点续传跳过。""" + if not path.exists(): + return set() + if path.is_dir(): + return {p.stem for p in path.glob("*.json")} + seen = set() + with path.open("r", encoding="utf-8", newline="") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + zpk = obj.get("zpk") + if zpk: + seen.add(zpk) + return seen + + +def read_and_save_response(resp: http.client.HTTPResponse, dest: Path) -> int: + tmp = dest.with_name(f"{dest.name}.{threading.get_ident()}.tmp") + h = hashlib.md5() + size = 0 + content_md5 = resp.getheader("Content-MD5") + + dest.parent.mkdir(parents=True, exist_ok=True) + with tmp.open("wb") as out: + while not STOP_EVENT.is_set(): + chunk = resp.read(256 * 1024) + if not chunk: + break + out.write(chunk) + h.update(chunk) + size += len(chunk) + + if STOP_EVENT.is_set(): + tmp.unlink(missing_ok=True) + raise InterruptedError + + if content_md5 and base64.b64encode(h.digest()).decode("ascii") != content_md5: + tmp.unlink(missing_ok=True) + raise ValueError("md5-mismatch") + + tmp.replace(dest) + return size + + +def ensure_downloaded(name: str, zpk_dir: Path, client: CdnClient) -> dict: + """确保 zpk 文件已下载,返回 (status, url, bytes)。""" + dest = zpk_dir / f"{name}.zpk" + if dest.exists() and dest.stat().st_size > 0: + return {"status": "skip-download", "url": "", "bytes": dest.stat().st_size} + + last_error = "" + for endpoint in client.endpoints: + if STOP_EVENT.is_set(): + return {"status": "cancelled", "url": "", "bytes": 0} + + try: + url, resp = client.get(endpoint, name) + try: + if resp.status != 200: + last_error = f"http-{resp.status}" + resp.read() + if resp.status >= 500: + client.drop(endpoint) + continue + size = read_and_save_response(resp, dest) + return {"status": "downloaded", "url": url, "bytes": size} + finally: + resp.close() + except InterruptedError: + return {"status": "cancelled", "url": "", "bytes": 0} + except ValueError as e: + last_error = str(e) + client.drop(endpoint) + except (http.client.HTTPException, OSError, TimeoutError) as e: + last_error = type(e).__name__ + client.drop(endpoint) + + return {"status": f"failed:{last_error}", "url": "", "bytes": 0} + + +def process_one( + row: dict, + zpk_dir: Path, + client: CdnClient, + sink: JsonlSink, + decrypt_zpk_bytes, + flatten, + include_raw: bool = True, +) -> dict: + """下载 + 解密单个 zpk,解密结果写入 JSONL。""" + name = row["zpk"] + word = row.get("first_word", "") + + dl = ensure_downloaded(name, zpk_dir, client) + if dl["status"].startswith("failed:"): + return {"name": name, "status": dl["status"], "bytes": dl["bytes"], "url": dl["url"]} + if dl["status"] == "cancelled": + return {"name": name, "status": "cancelled", "bytes": 0, "url": ""} + + # 下载成功或已存在,开始解密 + path = zpk_dir / f"{name}.zpk" + try: + obj = decrypt_zpk_bytes(path.read_bytes(), word) + # 解密输出必须保留 ZPK 内层 JSON 的全部字段;flatten 仅用于旧版兼容。 + rec = dict(obj) + rec.update( + { + "zpk": name, + "zpk_path": str(path), + "index_word": word, + "book_occurrences": row.get("book_occurrences", ""), + } + ) + if include_raw: + rec["raw"] = obj + sink.write(name, json.dumps(rec, ensure_ascii=False, separators=(",", ":"))) + return {"name": name, "status": "decrypted", "bytes": dl["bytes"], "url": dl["url"]} + except Exception as exc: + return {"name": name, "status": f"failed:decrypt-{type(exc).__name__}", "bytes": dl["bytes"], "url": dl["url"]} + + +def progress_line(done: int, total: int, result: dict, started_at: float, bytes_done: int) -> str: + elapsed = max(0.001, time.monotonic() - started_at) + files_per_sec = done / elapsed + mib_per_sec = bytes_done / 1024 / 1024 / elapsed + return ( + f"[INFO] [步骤 3/3] 处理进度:current={done}/{total},percent={done / total * 100:.1f}%," + f"name={result['name']},status={result['status']},rate={files_per_sec:.1f}/s,speed={mib_per_sec:.2f}MiB/s" + ) + + +def run_parallel( + rows: list[dict], + zpk_dir: Path, + sink: JsonlSink, + workers: int, + progress_every: int, + cdn_mode: str, + decrypt_zpk_bytes, + flatten, + include_raw: bool, +) -> list[dict]: + work: queue.Queue[dict | None] = queue.Queue(maxsize=max(1, workers * 4)) + results: list[dict] = [] + results_lock = threading.Lock() + started_at = time.monotonic() + done = 0 + bytes_done = 0 + total = len(rows) + + def record(result: dict) -> None: + nonlocal done, bytes_done + with results_lock: + results.append(result) + done += 1 + if result["status"] in {"decrypted"}: + bytes_done += int(result["bytes"]) + if done % progress_every == 0 or done == total or STOP_EVENT.is_set(): + print(progress_line(done, total, result, started_at, bytes_done), flush=True) + + def worker(worker_index: int) -> None: + client = CdnClient(build_endpoints(cdn_mode, worker_index), REQUEST_TIMEOUT) + try: + while not STOP_EVENT.is_set(): + try: + row = work.get(timeout=0.25) + except queue.Empty: + continue + try: + if row is None: + return + record(process_one(row, zpk_dir, client, sink, decrypt_zpk_bytes, flatten, include_raw)) + finally: + work.task_done() + finally: + client.close() + + threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(workers)] + for thread in threads: + thread.start() + + try: + for row in rows: + while not STOP_EVENT.is_set(): + try: + work.put(row, timeout=0.25) + break + except queue.Full: + continue + if STOP_EVENT.is_set(): + break + for _ in threads: + while not STOP_EVENT.is_set(): + try: + work.put(None, timeout=0.25) + break + except queue.Full: + continue + work.join() + except KeyboardInterrupt: + STOP_EVENT.set() + print(f"\n[中断] ZPK 已处理 {done}/{total},已保留已完成结果。", flush=True) + finally: + for thread in threads: + thread.join(timeout=REQUEST_TIMEOUT + 1) + + return results + + +def write_report(rows: list[dict], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="", encoding="utf-8-sig") as f: + writer = csv.DictWriter(f, fieldnames=["name", "status", "bytes", "url"]) + writer.writeheader() + writer.writerows(rows) + + +def main() -> int: + global REQUEST_TIMEOUT + parser = argparse.ArgumentParser(description="从全量词表去重下载所有 ZPK 并解密为 JSONL。") + parser.add_argument("--words", type=Path, default=WORDS_CSV, help="词书解密目录或 CSV 文件") + parser.add_argument("--out-dir", type=Path, default=ZPK_DIR) + parser.add_argument("--jsonl", type=Path, default=ZPK_JSONL, help="解密输出目录,每个 ZPK 一个 JSON") + parser.add_argument("--workers", type=int, default=64) + parser.add_argument("--timeout", type=float, default=8, help="单个 CDN 请求超时秒数;Ctrl+C 最多等待活跃请求到此超时") + parser.add_argument("--progress-every", type=int, default=500) + parser.add_argument( + "--cdn-mode", + choices=("spread", "primary"), + default="primary", + help="primary 会始终优先第一个 CDN;spread 会把 worker 分散到 3 个 CDN", + ) + parser.add_argument("--limit", type=int, default=0, help="最多处理 N 个(冒烟测试)") + parser.add_argument("--force", action="store_true", help="已解密也重新下载解密") + parser.add_argument("--raw", action="store_true", default=True, help="保留完整原始 JSON 对象(默认开启)") + args = parser.parse_args() + + REQUEST_TIMEOUT = max(1.0, args.timeout) + zpk_dir = args.out_dir + + if not args.words.exists(): + print(f"词表不存在:{args.words},请先运行 src/download-wordbooks.py", file=sys.stderr) + return 1 + + decrypt_zpk_bytes, flatten = load_decrypt_zpk() + index_rows = build_unique_index(args.words) + print(f"[INFO] [步骤 3/3] 索引准备完成:total={len(index_rows)}", flush=True) + + already_decrypted = set() if args.force else load_already_decrypted(args.jsonl) + rows = [r for r in index_rows if r["zpk"] not in already_decrypted] + print(f"[INFO] [步骤 3/3] 断点状态:completed={len(already_decrypted)},pending={len(rows)}", flush=True) + + if args.limit: + rows = rows[: args.limit] + + total = len(rows) + print(f"[INFO] [步骤 3/3] 开始下载并解密:total={total}", flush=True) + + sink = JsonlSink(args.jsonl) + workers = max(1, args.workers) + try: + if workers == 1: + endpoint_client = CdnClient(build_endpoints(args.cdn_mode), REQUEST_TIMEOUT) + results = [] + started_at = time.monotonic() + bytes_done = 0 + try: + for done, row in enumerate(rows, 1): + r = process_one(row, zpk_dir, endpoint_client, sink, decrypt_zpk_bytes, flatten, args.raw) + results.append(r) + if r["status"] == "decrypted": + bytes_done += int(r["bytes"]) + print(progress_line(done, total, r, started_at, bytes_done), flush=True) + except KeyboardInterrupt: + STOP_EVENT.set() + print(f"\n[中断] ZPK 已处理 {len(results)}/{total},已保留已完成结果。", flush=True) + finally: + endpoint_client.close() + else: + results = run_parallel( + rows, zpk_dir, sink, workers, max(1, args.progress_every), args.cdn_mode, + decrypt_zpk_bytes, flatten, args.raw, + ) + finally: + sink.close() + + ok = sum(1 for r in results if r["status"] == "decrypted") + failed = len(results) - ok + cancelled = sum(1 for r in results if r["status"] == "cancelled") + print(f"[INFO] [步骤 3/3] 处理完成:success={ok},failed={failed},skipped={cancelled}", flush=True) + if STOP_EVENT.is_set(): + return 130 + return 0 if failed == 0 else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/importer/src/export-synonym-derivative.py b/importer/src/export-synonym-derivative.py new file mode 100644 index 0000000..2cf19da --- /dev/null +++ b/importer/src/export-synonym-derivative.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Export synonym data from bbdb and derivative/family data from wordroot.""" + +from __future__ import annotations + +import csv +import argparse +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "output" / "synonym-derivative-export" +PACKAGE = "cn.com.langeasy.LangEasyLexis" +KEYS = { + "bbdb": "North_London_bbdb_Forever_20230710", + "wordroot": "North_wordroot_London_Forever_20220601", +} + + +def connect(path: Path, key: str): + try: + import sqlcipher3.dbapi2 as dbapi + except ImportError as exc: + raise SystemExit("请先安装 sqlcipher3:python -m pip install sqlcipher3") from exc + conn = dbapi.connect(str(path)) + conn.execute(f"PRAGMA key='{key}'") + conn.execute("SELECT count(*) FROM sqlite_master").fetchone() + return conn + + +def adb_export(serial: str | None, remote_name: str, local_path: Path) -> None: + target = ["adb"] + (["-s", serial] if serial else []) + target += ["exec-out", "cat", f"/data/data/{PACKAGE}/databases/{remote_name}"] + with local_path.open("wb") as fh: + result = subprocess.run(target, stdout=fh, stderr=subprocess.PIPE) + if result.returncode != 0: + target = ["adb"] + (["-s", serial] if serial else []) + target += ["exec-out", "run-as", PACKAGE, "cat", f"databases/{remote_name}"] + with local_path.open("wb") as fh: + result = subprocess.run(target, stdout=fh, stderr=subprocess.PIPE) + if result.returncode != 0 or local_path.stat().st_size == 0: + detail = result.stderr.decode(errors="replace").strip() + raise SystemExit(f"ADB 导出失败: {remote_name}: {detail}") + + +def dump_query(conn, sql: str, name: str) -> int: + cur = conn.execute(sql) + path = OUT / name + with path.open("w", newline="", encoding="utf-8-sig") as fh: + writer = csv.writer(fh) + writer.writerow([d[0] for d in cur.description]) + rows = cur.fetchall() + writer.writerows(rows) + return len(rows) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--serial", help="adb 设备序列号;默认使用当前设备") + args = parser.parse_args() + OUT.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="lexis-adb-export-") as tmp: + bb_path = Path(tmp) / "bbdb.db" + wr_path = Path(tmp) / "wordroot.db" + adb_export(args.serial, "bbdb.db", bb_path) + adb_export(args.serial, "wordroot.db", wr_path) + bb = connect(bb_path, KEYS["bbdb"]) + wr = connect(wr_path, KEYS["wordroot"]) + try: + counts = {} + counts["synonym_words.csv"] = dump_query(bb, "SELECT id, word FROM t_synonym_words ORDER BY id", "synonym_words.csv") + counts["synonym_data.csv"] = dump_query(bb, "SELECT * FROM t_synonym_datas ORDER BY word_id, pos, id", "synonym_data.csv") + counts["synonyms-joined.csv"] = dump_query(bb, """SELECT w.word, d.pos, d.interpret, d.zk, d.gk, d.cet4, d.cet6, d.ky, d.ielts, d.zsb, d.other, d.toefl FROM t_synonym_words w JOIN t_synonym_datas d ON d.word_id=w.id ORDER BY w.word, d.pos, d.id""", "synonyms-joined.csv") + counts["derivative-family.csv"] = dump_query(wr, """SELECT id, word, top_id, grade, interpret, prefix, suffix, group1, group2, group3, json FROM word_family ORDER BY top_id, id""", "derivative-family.csv") + counts["derivative-mini-groups.csv"] = dump_query(wr, "SELECT id, word_ids FROM word_family_mini_group ORDER BY id", "derivative-mini-groups.csv") + finally: + bb.close() + wr.close() + (OUT / "synonym-derivative-summary.txt").write_text( + "近义词与派生串记导出\n" + f"近义词单词: {counts['synonym_words.csv']}\n" + f"近义词释义记录: {counts['synonym_data.csv']}\n" + f"近义词关联记录: {counts['synonyms-joined.csv']}\n" + f"派生词族记录: {counts['derivative-family.csv']}\n" + f"派生小组记录: {counts['derivative-mini-groups.csv']}\n", + encoding="utf-8", + ) + print("已完成近义词和派生串记导出") + for name, count in counts.items(): + print(f"{name}: {count}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/importer/src/export-wordroot.py b/importer/src/export-wordroot.py new file mode 100644 index 0000000..be6946b --- /dev/null +++ b/importer/src/export-wordroot.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Export the encrypted word-root database into CSV files. + +The script supports pysqlcipher3/sqlcipher3 and the sqlcipher CLI. The +database is copied to a temporary file before opening so the source remains +unchanged. +""" + +from __future__ import annotations + +import argparse +import csv +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_DB = ROOT / "assets" / "wordroot.db" +DEFAULT_OUT = ROOT / "output" / "wordroot-export" +DB_KEY = "North_wordroot_London_Forever_20220601" + +LOCAL_RUNTIME = ROOT / ".runtime" / "sqlcipher3" +if LOCAL_RUNTIME.exists(): + sys.path.insert(0, str(LOCAL_RUNTIME)) + + +def open_python_cipher(db: Path): + for module_name in ("pysqlcipher3.dbapi2", "sqlcipher3.dbapi2"): + try: + module = __import__(module_name, fromlist=["connect"]) + except ImportError: + continue + conn = module.connect(str(db)) + conn.execute(f"PRAGMA key='{DB_KEY}'") + conn.execute("PRAGMA cipher_compatibility=4") + conn.execute("SELECT count(*) FROM sqlite_master").fetchone() + return conn + return None + + +def export_with_cli(db: Path, out_dir: Path) -> None: + sqlcipher = shutil.which("sqlcipher") + if not sqlcipher: + raise RuntimeError( + "未找到 SQLCipher。请安装 pysqlcipher3/sqlcipher3,或将 sqlcipher 加入 PATH。" + ) + out_dir.mkdir(parents=True, exist_ok=True) + script = ( + f"PRAGMA key='{DB_KEY}';\n" + "PRAGMA cipher_compatibility=4;\n" + ".mode csv\n" + ".headers on\n" + ) + tables = subprocess.run( + [sqlcipher, str(db)], input=script + "SELECT name FROM sqlite_master WHERE type='table';\n", + text=True, capture_output=True, check=True, + ).stdout.splitlines() + for table in (line.strip() for line in tables): + if not table or table == "name" or table.startswith("---"): + continue + safe = "".join(c if c.isalnum() or c in "-_" else "_" for c in table) + query = script + f'.output "{out_dir / (safe + ".csv")}"\nSELECT * FROM "{table}";\n.output stdout\n' + subprocess.run([sqlcipher, str(db)], input=query, text=True, check=True) + + +def export_with_python(conn, out_dir: Path) -> list[str]: + out_dir.mkdir(parents=True, exist_ok=True) + names = [row[0] for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name" + )] + for table in names: + safe = "".join(c if c.isalnum() or c in "-_" else "_" for c in table) + cur = conn.execute(f'SELECT * FROM "{table}"') + with (out_dir / f"{safe}.csv").open("w", newline="", encoding="utf-8-sig") as fh: + writer = csv.writer(fh) + writer.writerow([col[0] for col in cur.description]) + writer.writerows(cur) + return names + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--db", type=Path, default=DEFAULT_DB) + parser.add_argument("--output", type=Path, default=DEFAULT_OUT) + args = parser.parse_args() + db = args.db.resolve() + if not db.exists(): + raise SystemExit(f"数据库不存在: {db}") + + with tempfile.TemporaryDirectory(prefix="lexis-wordroot-") as tmp: + local_db = Path(tmp) / "wordroot.db" + shutil.copy2(db, local_db) + conn = open_python_cipher(local_db) + if conn is not None: + try: + tables = export_with_python(conn, args.output.resolve()) + finally: + conn.close() + else: + export_with_cli(local_db, args.output.resolve()) + tables = [p.stem for p in args.output.resolve().glob("*.csv")] + + manifest = args.output.resolve() / "README.txt" + manifest.write_text( + "wordroot.db 导出结果\n" + f"源数据库: {db}\n" + f"表数量: {len(tables)}\n" + "CSV 使用 UTF-8-BOM 编码。\n", + encoding="utf-8", + ) + print(f"已导出 {len(tables)} 张表到: {args.output.resolve()}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/importer/src/fetch-book-catalog.py b/importer/src/fetch-book-catalog.py new file mode 100644 index 0000000..7d8a7ea --- /dev/null +++ b/importer/src/fetch-book-catalog.py @@ -0,0 +1,102 @@ +"""从服务端词书列表接口拉取词书目录,替代从设备偏好文件提取。 + +接口:GET https://api.beingfine.cn/v3/3/bb/wordbooks/for-guest/{timestamp} +该接口是游客态全量词书列表,无需登录,响应 data_body 为明文 JSON。 +""" + +import argparse +import json +import sys +import time +import urllib.parse +import urllib.request +from pathlib import Path + +if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + +API_PATH = "https://api.beingfine.cn/v3/3/bb/wordbooks/for-guest" +APP_ID = "600000001" +APP_VERSION = "5.11.1" +CHANNEL = "isCool" +TOKEN = "tk85bf4d7d70a4d5028187ba6dfb3649" +REFERER = "https://app.bbdc.cn" +USER_AGENT = "LangEasyLexis/5.11.1" + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "output" / "book-catalog.json" +FIELDS = ("id", "code", "name", "count", "exam", "url", "version", "bookVersion", "purchased", "forSale") + + +def compact_book(book): + return {key: book.get(key) for key in FIELDS if key in book} + + +def fetch_catalog(timeout=20): + """拉取游客态词书列表,返回 (books, raw_body)。""" + ts = int(time.time() * 1000) + url = f"{API_PATH}/{ts}" + params = { + "type": "3/bb/wordbooks/for-guest", + "app_id": APP_ID, + "udid": "", + "umid": "", + "version": APP_VERSION, + "user_id": "0", + "channel": CHANNEL, + "timestamp": str(ts), + "timezone": "480", + "pad": "0", + "token": TOKEN, + } + qs = "&".join(f"{k}={urllib.parse.quote(str(v), safe='')}" for k, v in params.items()) + req = urllib.request.Request( + url + "?" + qs, + headers={"Referer": REFERER, "User-Agent": USER_AGENT}, + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read().decode("utf-8") + + outer = json.loads(raw) + if outer.get("result_code") != 200: + raise RuntimeError(f"result_code={outer.get('result_code')} body={raw[:200]}") + data_body = outer.get("data_body") + if isinstance(data_body, str): + data_body = json.loads(data_body) + if not isinstance(data_body, dict): + raise RuntimeError(f"data_body 缺失或格式异常: {raw[:200]}") + books = data_body.get("books") or [] + return books, data_body + + +def main(): + parser = argparse.ArgumentParser(description="从服务端拉取 Lexis 词书目录") + parser.add_argument("--output", default=OUT, help="输出词书目录 JSON 路径") + args = parser.parse_args() + + print("[INFO] [步骤 1/3] 开始获取词书目录", flush=True) + started = time.monotonic() + books, full = fetch_catalog() + catalog = sorted( + books, + key=lambda item: (str(item.get("exam", "")), str(item.get("code", ""))), + ) + + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + with open(args.output, "w", encoding="utf-8") as f: + json.dump(catalog, f, ensure_ascii=False, indent=2) + f.write("\n") + + categories = {} + for item in catalog: + category = str(item.get("exam") or "未分类") + categories[category] = categories.get(category, 0) + 1 + elapsed = time.monotonic() - started + category_text = ", ".join(f"{name}: {count}" for name, count in sorted(categories.items())) + print(f"[INFO] [步骤 1/3] 词书目录获取完成:数量={len(catalog)},耗时={elapsed:.1f}s") + print(f"[INFO] [步骤 1/3] 分类统计:{category_text}") + print(f"[INFO] [步骤 1/3] 输出文件:{args.output}") + + +if __name__ == "__main__": + main()