Files
word-market/importer/import_lexis.py
T
voidandClaude Sonnet 4.6 f030334243 feat(importer): 添加 Lexis 词书导出工具链
三步主流程(下载目录 -> 下载词书并解密导出 -> 下载并解密 ZPK),
以及可独立调用的 SQLCipher2 / ZPK 解密工具与词根、近义词导出脚本。

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-11-08 21:15:43 +08:00

1956 lines
77 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Lexis 词库一键建表 + 导入脚本
用法(默认 SQLite,无需任何服务):
python import_lexis.py --recreate
python import_lexis.py --sqlite-path D:/data/lexis.db --recreate
PostgreSQL:
pip install psycopg2-binary
python import_lexis.py --db postgres --dsn "postgresql://user:pass@host:5432/lexis" --recreate
MySQL:
pip install pymysql
python import_lexis.py --db mysql --dsn "mysql://user:pass@host:3306/lexis" --recreate
只导入解密数据,自动跳过 wordbooks/original 与 zpk/original。
"""
from __future__ import annotations
import argparse
import csv
import glob
import json
import os
import re
import sqlite3
import sys
import time
from collections import defaultdict
from urllib.parse import urlparse, unquote
# ---------------------------------------------------------------------------
# 表定义
# ---------------------------------------------------------------------------
TABLES = [
"exam", "book", "book_phrase_book", "word_entry", "book_entry",
"word_spelling_variant", "word_sense", "sense_object", "sense_exam",
"sense_relation", "entry_exam_meaning_order", "phrase", "phrase_exam",
"phrase_exam_order", "example_group", "example_group_exam",
"example_usage", "example_sentence", "entry_exam_sentence",
"confusable_word", "morpheme", "card", "card_exam", "word_family",
"word_family_exam", "word_family_exam_order", "word_family_variant",
"word_family_variant_member", "word_family_mini_group",
"word_family_mini_group_member", "card_family_link", "synonym_word",
"synonym_sense", "synonym_group", "synonym_group_member", "synonym_antonym",
]
COLUMNS = {
"exam": ("exam_id", "exam_code", "exam_name_zh", "is_grade_usable", "is_pseudo"),
"book": ("book_id", "book_code", "book_name", "description", "exam_id",
"declared_entry_count", "cover_url", "package_url", "content_version",
"schema_version", "sort_by", "sort_at", "is_for_sale", "is_purchased",
"is_opt", "source_code", "catalog_type", "labels", "publish_log",
"pkg_page", "external_link", "has_wordbook_file"),
"book_phrase_book": ("book_id", "phrase_book_id", "position"),
"word_entry": ("entry_id", "headword", "index_word", "definition_text",
"us_pronunciation", "uk_pronunciation", "data_version",
"book_exam_id", "occurrence_count", "has_listening_examples",
"source_zpk_key", "source_variant_code", "source_timestamp"),
"book_entry": ("book_entry_id", "book_id", "entry_id", "display_word",
"headword_text", "seq_book", "seq_az", "seq_freq", "seq_rand",
"source_zpk_key"),
"word_spelling_variant": ("entry_id", "position", "variant"),
"word_sense": ("sense_id", "entry_id", "sense_no", "part_of_speech",
"gloss_zh", "secondary_exam_codes", "flag_h"),
"sense_object": ("meaning_id", "entry_id", "source_meaning_id", "sense_id"),
"sense_exam": ("sense_id", "exam_id"),
"sense_relation": ("sense_id", "relation_type", "position", "target_meaning_id"),
"entry_exam_meaning_order": ("entry_id", "exam_id", "scope", "position", "meaning_id"),
"phrase": ("phrase_id", "entry_id", "source_phrase_id", "meaning_id",
"group_type", "text_en", "text_zh", "audio_ref", "extra_code"),
"phrase_exam": ("phrase_id", "exam_id"),
"phrase_exam_order": ("entry_id", "exam_id", "group_type", "position", "phrase_id"),
"example_group": ("example_group_id", "entry_id", "meaning_id", "group_kind",
"part_of_speech", "entry_kind_code", "head_form",
"definition_en", "definition_zh", "youdao_code_1",
"youdao_code_2", "source_exam_codes"),
"example_group_exam": ("example_group_id", "exam_id"),
"example_usage": ("usage_id", "example_group_id", "pattern_id",
"pattern_text", "position"),
"example_sentence": ("sentence_id", "source_sentence_key",
"source_sentence_ext_key", "example_group_id", "usage_id",
"position", "text_en", "text_zh", "audio_ref", "audio_url",
"image_path", "source_title", "is_listening_corpus"),
"entry_exam_sentence": ("entry_id", "exam_id", "position", "text_en",
"text_zh", "source_desc"),
"confusable_word": ("confusable_id", "entry_id", "position", "word_text",
"gloss_zh", "pos_code", "spelling_flag"),
"morpheme": ("morpheme_id", "morpheme_type", "source_morpheme_id", "form",
"display_text", "meaning_en", "meaning_zh", "note"),
"card": ("card_id", "word", "highlighted_word", "memory_note", "example_text",
"prefix_morpheme_id", "root_morpheme_id", "suffix_morpheme_id",
"default_family_ref", "is_hidden"),
"card_exam": ("card_id", "exam_id"),
"word_family": ("family_entry_id", "top_family_id", "word", "highlighted_word",
"interpretation", "prefix_morpheme_id", "suffix_morpheme_id",
"group_level_1", "group_level_2", "group_level_3"),
"word_family_exam": ("family_entry_id", "exam_id"),
"word_family_exam_order": ("family_entry_id", "exam_id", "position"),
"word_family_variant": ("variant_id", "variant_word", "family_entry_id",
"top_family_id", "highlighted_word", "interpretation",
"prefix_morpheme_id", "suffix_morpheme_id"),
"word_family_variant_member": ("variant_id", "exam_id", "position",
"member_word", "is_placeholder"),
"word_family_mini_group": ("mini_group_id", "size"),
"word_family_mini_group_member": ("mini_group_id", "family_entry_id", "position"),
"card_family_link": ("link_id", "card_id", "group_no", "family_ref", "is_hidden"),
"synonym_word": ("word_id", "word"),
"synonym_sense": ("sense_id", "word_id", "part_of_speech", "gloss_zh"),
"synonym_group": ("sense_id", "exam_id", "canonical_word_id", "member_count"),
"synonym_group_member": ("sense_id", "exam_id", "position", "member_word_id"),
"synonym_antonym": ("sense_id", "exam_id", "position", "antonym_word_id"),
}
# 字段注释:DDL 生成时强制校验,缺一不可
COLUMN_COMMENTS = {
# exam
("exam", "exam_id"): "主键;0=def 默认视图,1-13 真实考试,14=XX 未分类,15=OTHER 其他",
("exam", "exam_code"): "考试业务编码,仅作唯一键,不参与外键",
("exam", "exam_name_zh"): "考试中文名称",
("exam", "is_grade_usable"): "是否可用于年级/考试编码(仅 1-10 为真)",
("exam", "is_pseudo"): "是否为伪考试视图(def/XX/OTHER)",
# book
("book", "book_id"): "词书主键,对应目录 id",
("book", "book_code"): "词书业务唯一编码,取自文件名后缀",
("book", "book_name"): "词书名称",
("book", "description"): "词书描述",
("book", "exam_id"): "关联考试 id,空表示非考试类",
("book", "declared_entry_count"): "源端声明的词条数",
("book", "cover_url"): "封面图片地址",
("book", "package_url"): "词书数据包地址",
("book", "content_version"): "内容版本",
("book", "schema_version"): "目录结构版本",
("book", "sort_by"): "排序方式",
("book", "sort_at"): "排序时间,Unix 秒",
("book", "is_for_sale"): "是否售卖",
("book", "is_purchased"): "是否已购买",
("book", "is_opt"): "是否可选项",
("book", "source_code"): "来源编码",
("book", "catalog_type"): "目录类型",
("book", "labels"): "标签文本",
("book", "publish_log"): "发布日志",
("book", "pkg_page"): "包页面配置,JSON",
("book", "external_link"): "外部链接配置,JSON",
("book", "has_wordbook_file"): "是否存在词书清单文件",
# book_phrase_book
("book_phrase_book", "book_id"): "词书 id",
("book_phrase_book", "phrase_book_id"): "内嵌短语书的词书 id",
("book_phrase_book", "position"): "内嵌顺序",
# word_entry
("word_entry", "entry_id"): "词条主键,取自 zpk 文件名第 2 段",
("word_entry", "headword"): "词条原形",
("word_entry", "index_word"): "归一化索引词",
("word_entry", "definition_text"): "原始释义文本(词性 + 中文)",
("word_entry", "us_pronunciation"): "美式音标",
("word_entry", "uk_pronunciation"): "英式音标",
("word_entry", "data_version"): "源数据版本",
("word_entry", "book_exam_id"): "所属考试 id",
("word_entry", "occurrence_count"): "源端声明的收录词书数",
("word_entry", "has_listening_examples"): "是否包含听力语料例句",
("word_entry", "source_zpk_key"): "源 zpk 文件键,仅溯源",
("word_entry", "source_variant_code"): "源文件名中的变体编码",
("word_entry", "source_timestamp"): "源文件名中的时间戳,Unix 秒",
# book_entry
("book_entry", "book_entry_id"): "词书词条关联主键",
("book_entry", "book_id"): "词书 id",
("book_entry", "entry_id"): "词条 id",
("book_entry", "display_word"): "词书中的展示词形",
("book_entry", "headword_text"): "主词/词根提示,弱文本引用",
("book_entry", "seq_book"): "词书内顺序",
("book_entry", "seq_az"): "字母序",
("book_entry", "seq_freq"): "词频排名,0 表示未知",
("book_entry", "seq_rand"): "随机顺序",
("book_entry", "source_zpk_key"): "源 zpk 文件键,仅溯源",
# word_spelling_variant
("word_spelling_variant", "entry_id"): "词条 id",
("word_spelling_variant", "position"): "变体顺序",
("word_spelling_variant", "variant"): "拼写变体文本",
# word_sense
("word_sense", "sense_id"): "义项主键(代理)",
("word_sense", "entry_id"): "词条 id",
("word_sense", "sense_no"): "词条内义项序号,源值不连续",
("word_sense", "part_of_speech"): "词性",
("word_sense", "gloss_zh"): "中文释义",
("word_sense", "secondary_exam_codes"): "源字段 s,次要考试编码列表,逗号分隔",
("word_sense", "flag_h"): "源字段 h,数值标记",
# sense_object
("sense_object", "meaning_id"): "释义对象主键(代理)",
("sense_object", "entry_id"): "词条 id",
("sense_object", "source_meaning_id"): "源释义对象 id(oid),仅词条内唯一",
("sense_object", "sense_id"): "所属义项 id",
# sense_exam
("sense_exam", "sense_id"): "义项 id",
("sense_exam", "exam_id"): "考试 id",
# sense_relation
("sense_relation", "sense_id"): "义项 id",
("sense_relation", "relation_type"): "关系类型:synonym/antonym/reference",
("sense_relation", "position"): "关系内顺序",
("sense_relation", "target_meaning_id"): "目标释义对象 id",
# entry_exam_meaning_order
("entry_exam_meaning_order", "entry_id"): "词条 id",
("entry_exam_meaning_order", "exam_id"): "考试 id",
("entry_exam_meaning_order", "scope"): "释义范围:core 核心 / ext 扩展",
("entry_exam_meaning_order", "position"): "范围内顺序",
("entry_exam_meaning_order", "meaning_id"): "释义对象 id",
# phrase
("phrase", "phrase_id"): "短语主键(代理)",
("phrase", "entry_id"): "词条 id",
("phrase", "source_phrase_id"): "源短语 id(pid),仅词条内唯一",
("phrase", "meaning_id"): "所属释义对象 id",
("phrase", "group_type"): "短语组类型,源值 1/2",
("phrase", "text_en"): "英文短语",
("phrase", "text_zh"): "中文释义",
("phrase", "audio_ref"): "发音资源标识,非外键",
("phrase", "extra_code"): "源附加编码 bd",
# phrase_exam
("phrase_exam", "phrase_id"): "短语 id",
("phrase_exam", "exam_id"): "考试 id",
# phrase_exam_order
("phrase_exam_order", "entry_id"): "词条 id",
("phrase_exam_order", "exam_id"): "考试 id",
("phrase_exam_order", "group_type"): "短语组类型",
("phrase_exam_order", "position"): "分考试顺序",
("phrase_exam_order", "phrase_id"): "短语 id",
# example_group
("example_group", "example_group_id"): "例句块主键(代理)",
("example_group", "entry_id"): "词条 id",
("example_group", "meaning_id"): "所属释义对象 id,type 99 为空",
("example_group", "group_kind"): "例句块类型:1 核心,2 扩展,3 短语动词,4 习语,99 听力语料",
("example_group", "part_of_speech"): "词性",
("example_group", "entry_kind_code"): "词条类型码:NORMAL/PHRV/IDM/PHR",
("example_group", "head_form"): "词头形式",
("example_group", "definition_en"): "英文释义",
("example_group", "definition_zh"): "中文释义",
("example_group", "youdao_code_1"): "源附加编码 yu",
("example_group", "youdao_code_2"): "源附加编码 yo",
("example_group", "source_exam_codes"): "源考试编码快照,规范值见 example_group_exam",
# example_group_exam
("example_group_exam", "example_group_id"): "例句块 id",
("example_group_exam", "exam_id"): "考试 id",
# example_usage
("example_usage", "usage_id"): "用法模式主键(代理)",
("example_usage", "example_group_id"): "例句块 id",
("example_usage", "pattern_id"): "关联短语 id(源 uid 解析)",
("example_usage", "pattern_text"): "用法模式文本",
("example_usage", "position"): "组内顺序",
# example_sentence
("example_sentence", "sentence_id"): "句子主键(代理)",
("example_sentence", "source_sentence_key"): "源句子标识 sid,不唯一",
("example_sentence", "source_sentence_ext_key"): "源句子扩展标识 eid,不唯一",
("example_sentence", "example_group_id"): "例句块 id",
("example_sentence", "usage_id"): "用法模式 id,type 99 为空",
("example_sentence", "position"): "组内顺序",
("example_sentence", "text_en"): "英文句子,含高亮标签",
("example_sentence", "text_zh"): "中文翻译",
("example_sentence", "audio_ref"): "发音资源标识,非外键",
("example_sentence", "audio_url"): "音频地址",
("example_sentence", "image_path"): "图片路径",
("example_sentence", "source_title"): "来源书名或剧名",
("example_sentence", "is_listening_corpus"): "是否听力语料例句",
# entry_exam_sentence
("entry_exam_sentence", "entry_id"): "词条 id",
("entry_exam_sentence", "exam_id"): "考试 id",
("entry_exam_sentence", "position"): "顺序",
("entry_exam_sentence", "text_en"): "英文例句",
("entry_exam_sentence", "text_zh"): "中文翻译",
("entry_exam_sentence", "source_desc"): "来源说明",
# confusable_word
("confusable_word", "confusable_id"): "易混词主键(代理)",
("confusable_word", "entry_id"): "词条 id",
("confusable_word", "position"): "顺序",
("confusable_word", "word_text"): "易混词文本",
("confusable_word", "gloss_zh"): "中文释义",
("confusable_word", "pos_code"): "源词性编码",
("confusable_word", "spelling_flag"): "拼写标记",
# morpheme
("morpheme", "morpheme_id"): "词素主键(统一代理)",
("morpheme", "morpheme_type"): "词素类型:prefix/root/suffix",
("morpheme", "source_morpheme_id"): "源表内 id",
("morpheme", "form"): "词素形式",
("morpheme", "display_text"): "原始展示文本",
("morpheme", "meaning_en"): "英文释义",
("morpheme", "meaning_zh"): "中文释义",
("morpheme", "note"): "备注",
# card
("card", "card_id"): "卡片主键",
("card", "word"): "单词",
("card", "highlighted_word"): "高亮 HTML 快照",
("card", "memory_note"): "记忆提示",
("card", "example_text"): "例句文本",
("card", "prefix_morpheme_id"): "前缀词素 id",
("card", "root_morpheme_id"): "词根词素 id",
("card", "suffix_morpheme_id"): "后缀词素 id",
("card", "default_family_ref"): "旧版词族引用,不可解析,无外键",
("card", "is_hidden"): "是否隐藏",
# card_exam
("card_exam", "card_id"): "卡片 id",
("card_exam", "exam_id"): "考试 id",
# word_family
("word_family", "family_entry_id"): "词族词条主键",
("word_family", "top_family_id"): "顶层词族 id,自引用",
("word_family", "word"): "单词",
("word_family", "highlighted_word"): "高亮 HTML 快照",
("word_family", "interpretation"): "词性 + 中文释义",
("word_family", "prefix_morpheme_id"): "前缀词素 id",
("word_family", "suffix_morpheme_id"): "后缀词素 id",
("word_family", "group_level_1"): "分组层级 1",
("word_family", "group_level_2"): "分组层级 2",
("word_family", "group_level_3"): "分组层级 3",
# word_family_exam
("word_family_exam", "family_entry_id"): "词族词条 id",
("word_family_exam", "exam_id"): "考试 id",
# word_family_exam_order
("word_family_exam_order", "family_entry_id"): "词族词条 id",
("word_family_exam_order", "exam_id"): "考试 id",
("word_family_exam_order", "position"): "分考试顺序",
# word_family_variant
("word_family_variant", "variant_id"): "变体主键",
("word_family_variant", "variant_word"): "变体词",
("word_family_variant", "family_entry_id"): "所属词族词条 id",
("word_family_variant", "top_family_id"): "顶层词族 id",
("word_family_variant", "highlighted_word"): "高亮 HTML 快照",
("word_family_variant", "interpretation"): "词性 + 中文释义",
("word_family_variant", "prefix_morpheme_id"): "前缀词素 id",
("word_family_variant", "suffix_morpheme_id"): "后缀词素 id",
# word_family_variant_member
("word_family_variant_member", "variant_id"): "变体 id",
("word_family_variant_member", "exam_id"): "考试 id",
("word_family_variant_member", "position"): "顺序",
("word_family_variant_member", "member_word"): "成员词",
("word_family_variant_member", "is_placeholder"): "是否为 * 占位符",
# word_family_mini_group
("word_family_mini_group", "mini_group_id"): "小组主键",
("word_family_mini_group", "size"): "小组成员数",
# word_family_mini_group_member
("word_family_mini_group_member", "mini_group_id"): "小组 id",
("word_family_mini_group_member", "family_entry_id"): "词族词条 id",
("word_family_mini_group_member", "position"): "组内顺序",
# card_family_link
("card_family_link", "link_id"): "关联主键",
("card_family_link", "card_id"): "卡片 id",
("card_family_link", "group_no"): "分组编号(1-50)",
("card_family_link", "family_ref"): "旧版词族引用,不可解析,无外键",
("card_family_link", "is_hidden"): "是否隐藏",
# synonym_word
("synonym_word", "word_id"): "近义词词头主键",
("synonym_word", "word"): "单词",
# synonym_sense
("synonym_sense", "sense_id"): "近义词义项主键",
("synonym_sense", "word_id"): "词头 id",
("synonym_sense", "part_of_speech"): "词性",
("synonym_sense", "gloss_zh"): "中文释义",
# synonym_group
("synonym_group", "sense_id"): "近义词义项 id",
("synonym_group", "exam_id"): "考试 id",
("synonym_group", "canonical_word_id"): "规范词 id(源 free),可为空",
("synonym_group", "member_count"): "近义词成员数",
# synonym_group_member
("synonym_group_member", "sense_id"): "近义词义项 id",
("synonym_group_member", "exam_id"): "考试 id",
("synonym_group_member", "position"): "组内顺序",
("synonym_group_member", "member_word_id"): "成员词 id",
# synonym_antonym
("synonym_antonym", "sense_id"): "近义词义项 id",
("synonym_antonym", "exam_id"): "考试 id",
("synonym_antonym", "position"): "组内顺序",
("synonym_antonym", "antonym_word_id"): "反义词 id",
}
# PostgreSQL DDL;SQLite 通过类型替换复用
SCHEMA_PG = """
CREATE TABLE exam (
exam_id bigint PRIMARY KEY,
exam_code varchar(16) NOT NULL UNIQUE,
exam_name_zh varchar(32) NOT NULL,
is_grade_usable boolean NOT NULL DEFAULT FALSE,
is_pseudo boolean NOT NULL DEFAULT FALSE
);
CREATE TABLE book (
book_id bigint PRIMARY KEY,
book_code varchar(32) NOT NULL UNIQUE,
book_name varchar(128) NOT NULL,
description text,
exam_id bigint REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED,
declared_entry_count integer,
cover_url text,
package_url text,
content_version integer,
schema_version integer,
sort_by varchar(16),
sort_at bigint,
is_for_sale smallint,
is_purchased smallint,
is_opt smallint,
source_code smallint,
catalog_type smallint,
labels text,
publish_log text,
pkg_page jsonb,
external_link jsonb,
has_wordbook_file boolean NOT NULL DEFAULT TRUE
);
CREATE TABLE book_phrase_book (
book_id bigint NOT NULL REFERENCES book(book_id) DEFERRABLE INITIALLY DEFERRED,
phrase_book_id bigint NOT NULL REFERENCES book(book_id) DEFERRABLE INITIALLY DEFERRED,
position smallint NOT NULL,
PRIMARY KEY (book_id, phrase_book_id)
);
CREATE TABLE word_entry (
entry_id bigint PRIMARY KEY,
headword varchar(256) NOT NULL,
index_word varchar(256) NOT NULL,
definition_text text,
us_pronunciation varchar(128),
uk_pronunciation varchar(128),
data_version varchar(8),
book_exam_id bigint REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED,
occurrence_count integer,
has_listening_examples boolean NOT NULL DEFAULT FALSE,
source_zpk_key varchar(64) UNIQUE,
source_variant_code integer,
source_timestamp bigint
);
CREATE TABLE book_entry (
book_entry_id bigint PRIMARY KEY,
book_id bigint NOT NULL REFERENCES book(book_id) DEFERRABLE INITIALLY DEFERRED,
entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED,
display_word varchar(256) NOT NULL,
headword_text varchar(256),
seq_book integer,
seq_az integer,
seq_freq integer,
seq_rand integer,
source_zpk_key varchar(64),
UNIQUE (book_id, entry_id)
);
CREATE TABLE word_spelling_variant (
entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED,
position smallint NOT NULL,
variant varchar(256) NOT NULL,
PRIMARY KEY (entry_id, position)
);
CREATE TABLE word_sense (
sense_id bigint PRIMARY KEY,
entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED,
sense_no integer NOT NULL,
part_of_speech varchar(32),
gloss_zh text,
secondary_exam_codes text,
flag_h smallint
);
CREATE TABLE sense_object (
meaning_id bigint PRIMARY KEY,
entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED,
source_meaning_id bigint NOT NULL,
sense_id bigint REFERENCES word_sense(sense_id) DEFERRABLE INITIALLY DEFERRED
);
CREATE TABLE sense_exam (
sense_id bigint NOT NULL REFERENCES word_sense(sense_id) DEFERRABLE INITIALLY DEFERRED,
exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED,
PRIMARY KEY (sense_id, exam_id)
);
CREATE TABLE sense_relation (
sense_id bigint NOT NULL REFERENCES word_sense(sense_id) DEFERRABLE INITIALLY DEFERRED,
relation_type varchar(16) NOT NULL,
position smallint NOT NULL,
target_meaning_id bigint NOT NULL REFERENCES sense_object(meaning_id) DEFERRABLE INITIALLY DEFERRED,
PRIMARY KEY (sense_id, relation_type, position)
);
CREATE TABLE entry_exam_meaning_order (
entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED,
exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED,
scope varchar(8) NOT NULL CHECK (scope IN ('core','ext')),
position smallint NOT NULL,
meaning_id bigint NOT NULL REFERENCES sense_object(meaning_id) DEFERRABLE INITIALLY DEFERRED,
PRIMARY KEY (entry_id, exam_id, scope, position)
);
CREATE TABLE phrase (
phrase_id bigint PRIMARY KEY,
entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED,
source_phrase_id bigint,
meaning_id bigint REFERENCES sense_object(meaning_id) DEFERRABLE INITIALLY DEFERRED,
group_type smallint NOT NULL,
text_en text NOT NULL,
text_zh text,
audio_ref varchar(64),
extra_code varchar(16),
UNIQUE (entry_id, source_phrase_id)
);
CREATE TABLE phrase_exam (
phrase_id bigint NOT NULL REFERENCES phrase(phrase_id) DEFERRABLE INITIALLY DEFERRED,
exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED,
PRIMARY KEY (phrase_id, exam_id)
);
CREATE TABLE phrase_exam_order (
entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED,
exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED,
group_type smallint NOT NULL,
position smallint NOT NULL,
phrase_id bigint NOT NULL REFERENCES phrase(phrase_id) DEFERRABLE INITIALLY DEFERRED,
PRIMARY KEY (entry_id, exam_id, group_type, position)
);
CREATE TABLE example_group (
example_group_id bigint PRIMARY KEY,
entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED,
meaning_id bigint REFERENCES sense_object(meaning_id) DEFERRABLE INITIALLY DEFERRED,
group_kind smallint NOT NULL,
part_of_speech varchar(32),
entry_kind_code varchar(16),
head_form varchar(256),
definition_en text,
definition_zh text,
youdao_code_1 varchar(64),
youdao_code_2 varchar(64),
source_exam_codes text
);
CREATE TABLE example_group_exam (
example_group_id bigint NOT NULL REFERENCES example_group(example_group_id) DEFERRABLE INITIALLY DEFERRED,
exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED,
PRIMARY KEY (example_group_id, exam_id)
);
CREATE TABLE example_usage (
usage_id bigint PRIMARY KEY,
example_group_id bigint NOT NULL REFERENCES example_group(example_group_id) DEFERRABLE INITIALLY DEFERRED,
pattern_id bigint REFERENCES phrase(phrase_id) DEFERRABLE INITIALLY DEFERRED,
pattern_text text,
position smallint NOT NULL
);
CREATE TABLE example_sentence (
sentence_id bigint PRIMARY KEY,
source_sentence_key varchar(64),
source_sentence_ext_key varchar(64),
example_group_id bigint NOT NULL REFERENCES example_group(example_group_id) DEFERRABLE INITIALLY DEFERRED,
usage_id bigint REFERENCES example_usage(usage_id) DEFERRABLE INITIALLY DEFERRED,
position smallint,
text_en text NOT NULL,
text_zh text,
audio_ref varchar(64),
audio_url text,
image_path text,
source_title varchar(256),
is_listening_corpus boolean NOT NULL DEFAULT FALSE
);
CREATE TABLE entry_exam_sentence (
entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED,
exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED,
position smallint NOT NULL,
text_en text NOT NULL,
text_zh text,
source_desc text,
PRIMARY KEY (entry_id, exam_id, position)
);
CREATE TABLE confusable_word (
confusable_id bigint PRIMARY KEY,
entry_id bigint NOT NULL REFERENCES word_entry(entry_id) DEFERRABLE INITIALLY DEFERRED,
position smallint NOT NULL,
word_text varchar(256) NOT NULL,
gloss_zh text,
pos_code smallint,
spelling_flag varchar(8),
UNIQUE (entry_id, word_text)
);
CREATE TABLE morpheme (
morpheme_id bigint PRIMARY KEY,
morpheme_type varchar(8) NOT NULL CHECK (morpheme_type IN ('prefix','root','suffix')),
source_morpheme_id bigint NOT NULL,
form varchar(64),
display_text varchar(256) NOT NULL,
meaning_en varchar(256),
meaning_zh varchar(256),
note varchar(256),
UNIQUE (morpheme_type, source_morpheme_id)
);
CREATE TABLE card (
card_id bigint PRIMARY KEY,
word varchar(256) NOT NULL,
highlighted_word text,
memory_note text,
example_text text,
prefix_morpheme_id bigint REFERENCES morpheme(morpheme_id) DEFERRABLE INITIALLY DEFERRED,
root_morpheme_id bigint REFERENCES morpheme(morpheme_id) DEFERRABLE INITIALLY DEFERRED,
suffix_morpheme_id bigint REFERENCES morpheme(morpheme_id) DEFERRABLE INITIALLY DEFERRED,
default_family_ref bigint,
is_hidden boolean NOT NULL DEFAULT FALSE
);
CREATE TABLE card_exam (
card_id bigint NOT NULL REFERENCES card(card_id) DEFERRABLE INITIALLY DEFERRED,
exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED,
PRIMARY KEY (card_id, exam_id)
);
CREATE TABLE word_family (
family_entry_id bigint PRIMARY KEY,
top_family_id bigint NOT NULL REFERENCES word_family(family_entry_id) DEFERRABLE INITIALLY DEFERRED,
word varchar(256) NOT NULL,
highlighted_word text,
interpretation text,
prefix_morpheme_id bigint REFERENCES morpheme(morpheme_id) DEFERRABLE INITIALLY DEFERRED,
suffix_morpheme_id bigint REFERENCES morpheme(morpheme_id) DEFERRABLE INITIALLY DEFERRED,
group_level_1 smallint,
group_level_2 smallint,
group_level_3 smallint
);
CREATE TABLE word_family_exam (
family_entry_id bigint NOT NULL REFERENCES word_family(family_entry_id) DEFERRABLE INITIALLY DEFERRED,
exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED,
PRIMARY KEY (family_entry_id, exam_id)
);
CREATE TABLE word_family_exam_order (
family_entry_id bigint NOT NULL REFERENCES word_family(family_entry_id) DEFERRABLE INITIALLY DEFERRED,
exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED,
position integer NOT NULL,
PRIMARY KEY (family_entry_id, exam_id)
);
CREATE TABLE word_family_variant (
variant_id bigint PRIMARY KEY,
variant_word varchar(256) NOT NULL,
family_entry_id bigint NOT NULL REFERENCES word_family(family_entry_id) DEFERRABLE INITIALLY DEFERRED,
top_family_id bigint NOT NULL REFERENCES word_family(family_entry_id) DEFERRABLE INITIALLY DEFERRED,
highlighted_word text,
interpretation text,
prefix_morpheme_id bigint REFERENCES morpheme(morpheme_id) DEFERRABLE INITIALLY DEFERRED,
suffix_morpheme_id bigint REFERENCES morpheme(morpheme_id) DEFERRABLE INITIALLY DEFERRED
);
CREATE TABLE word_family_variant_member (
variant_id bigint NOT NULL REFERENCES word_family_variant(variant_id) DEFERRABLE INITIALLY DEFERRED,
exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED,
position smallint NOT NULL,
member_word varchar(256) NOT NULL,
is_placeholder boolean NOT NULL DEFAULT FALSE,
PRIMARY KEY (variant_id, exam_id, position)
);
CREATE TABLE word_family_mini_group (
mini_group_id bigint PRIMARY KEY,
size smallint NOT NULL
);
CREATE TABLE word_family_mini_group_member (
mini_group_id bigint NOT NULL REFERENCES word_family_mini_group(mini_group_id) DEFERRABLE INITIALLY DEFERRED,
family_entry_id bigint NOT NULL REFERENCES word_family(family_entry_id) DEFERRABLE INITIALLY DEFERRED,
position smallint NOT NULL,
PRIMARY KEY (mini_group_id, family_entry_id)
);
CREATE TABLE card_family_link (
link_id bigint PRIMARY KEY,
card_id bigint NOT NULL REFERENCES card(card_id) DEFERRABLE INITIALLY DEFERRED,
group_no smallint NOT NULL,
family_ref bigint,
is_hidden boolean NOT NULL DEFAULT FALSE
);
CREATE TABLE synonym_word (
word_id bigint PRIMARY KEY,
word varchar(256) NOT NULL
);
CREATE TABLE synonym_sense (
sense_id bigint PRIMARY KEY,
word_id bigint NOT NULL REFERENCES synonym_word(word_id) DEFERRABLE INITIALLY DEFERRED,
part_of_speech varchar(32),
gloss_zh text
);
CREATE TABLE synonym_group (
sense_id bigint NOT NULL REFERENCES synonym_sense(sense_id) DEFERRABLE INITIALLY DEFERRED,
exam_id bigint NOT NULL REFERENCES exam(exam_id) DEFERRABLE INITIALLY DEFERRED,
canonical_word_id bigint REFERENCES synonym_word(word_id) DEFERRABLE INITIALLY DEFERRED,
member_count smallint NOT NULL,
PRIMARY KEY (sense_id, exam_id)
);
CREATE TABLE synonym_group_member (
sense_id bigint NOT NULL,
exam_id bigint NOT NULL,
position smallint NOT NULL,
member_word_id bigint NOT NULL REFERENCES synonym_word(word_id) DEFERRABLE INITIALLY DEFERRED,
PRIMARY KEY (sense_id, exam_id, position),
FOREIGN KEY (sense_id, exam_id) REFERENCES synonym_group(sense_id, exam_id) DEFERRABLE INITIALLY DEFERRED
);
CREATE TABLE synonym_antonym (
sense_id bigint NOT NULL,
exam_id bigint NOT NULL,
position smallint NOT NULL,
antonym_word_id bigint NOT NULL REFERENCES synonym_word(word_id) DEFERRABLE INITIALLY DEFERRED,
PRIMARY KEY (sense_id, exam_id, position),
FOREIGN KEY (sense_id, exam_id) REFERENCES synonym_group(sense_id, exam_id) DEFERRABLE INITIALLY DEFERRED
);
"""
INDEXES = """
CREATE INDEX idx_book_entry_book_seq ON book_entry (book_id, seq_book);
CREATE INDEX idx_book_entry_entry ON book_entry (entry_id);
CREATE INDEX idx_word_entry_index_word ON word_entry (index_word);
CREATE INDEX idx_word_sense_entry ON word_sense (entry_id, sense_no);
CREATE INDEX idx_sense_object_entry ON sense_object (entry_id, source_meaning_id);
CREATE INDEX idx_phrase_entry ON phrase (entry_id);
CREATE INDEX idx_phrase_meaning ON phrase (meaning_id);
CREATE INDEX idx_example_group_entry ON example_group (entry_id, group_kind);
CREATE INDEX idx_example_group_meaning ON example_group (meaning_id);
CREATE INDEX idx_example_sentence_group ON example_sentence (example_group_id);
CREATE INDEX idx_example_sentence_usage ON example_sentence (usage_id);
CREATE INDEX idx_example_sentence_src ON example_sentence (source_sentence_key);
CREATE INDEX idx_entry_exam_sentence_entry ON entry_exam_sentence (entry_id);
CREATE INDEX idx_confusable_entry ON confusable_word (entry_id);
CREATE INDEX idx_morpheme_form ON morpheme (morpheme_type, form);
CREATE INDEX idx_word_family_word ON word_family (word);
CREATE INDEX idx_word_family_top ON word_family (top_family_id);
CREATE INDEX idx_mini_member_family ON word_family_mini_group_member (family_entry_id);
CREATE INDEX idx_card_word ON card (word);
CREATE INDEX idx_card_family_link_card ON card_family_link (card_id);
CREATE INDEX idx_synonym_word_text ON synonym_word (word);
CREATE INDEX idx_syn_group_member_word ON synonym_group_member (member_word_id);
CREATE INDEX idx_syn_antonym_word ON synonym_antonym (antonym_word_id);
"""
EXAM_CODES = {
"def": (0, "默认视图", True),
"ZK": (1, "中考", False),
"GK": (2, "高考", False),
"CET4": (3, "四级", False),
"CET6": (4, "六级", False),
"KY": (5, "考研", False),
"IELTS": (6, "雅思", False),
"TOEFL": (7, "托福", False),
"PRO4": (8, "专四", False),
"PRO8": (9, "专八", False),
"ZSB": (10, "专升本", False),
"KB": (11, "考博", False),
"SAT": (12, "SAT", False),
"GRE": (13, "GRE", False),
"XX": (14, "未分类", True),
"OTHER": (15, "其他", True),
}
GRADE_USABLE = set(range(1, 11))
SYNONYM_EXAM_COLUMNS = {
"zk": 1, "gk": 2, "cet4": 3, "cet6": 4, "ky": 5,
"ielts": 6, "toefl": 7, "zsb": 10, "other": 15,
}
# ---------------------------------------------------------------------------
# 工具函数
# ---------------------------------------------------------------------------
def _to_int(value, default=None):
if value is None:
return default
if isinstance(value, bool):
return int(value)
if isinstance(value, int):
return value
s = str(value).strip()
if not s:
return default
try:
return int(s)
except ValueError:
try:
return int(float(s))
except ValueError:
return default
def _to_bool(value):
return 1 if str(value).strip().lower() in ("1", "true", "yes", "y") else 0
def _text(value):
if value is None:
return None
s = str(value)
return s if s != "" else None
def read_csv(path):
with open(path, encoding="utf-8-sig", newline="") as f:
yield from csv.DictReader(f)
def exam_id_of(code):
if code is None:
return None
s = str(code).strip()
if not s:
return None
if s in EXAM_CODES:
return EXAM_CODES[s][0]
up = s.upper()
if up in EXAM_CODES:
return EXAM_CODES[up][0]
return None
def parse_affix(display):
"""'duct = to lead 引导' -> (form, meaning_en, meaning_zh, note)"""
text = (display or "").strip()
if "=" in text:
left, right = text.split("=", 1)
form = left.strip()
right = right.strip()
else:
form, right = text, ""
# 以第一个 CJK 字符为界
m = re.search(r"[\u3400-\u9fff]", right)
if m:
en = right[:m.start()].strip() or None
zh = right[m.start():].strip() or None
note = None
if ":" in zh:
zh, note = zh.split(":", 1)
zh, note = zh.strip(), note.strip() or None
elif ":" in zh:
zh, note = zh.split(":", 1)
zh, note = zh.strip(), note.strip() or None
return form, en, zh, note
return form, right or None, None, None
# ---------------------------------------------------------------------------
# 数据库适配
# ---------------------------------------------------------------------------
class SQLiteAdapter:
placeholder = "?"
def __init__(self, path):
self.conn = sqlite3.connect(path)
self.conn.execute("PRAGMA journal_mode=OFF")
self.conn.execute("PRAGMA synchronous=OFF")
self.conn.execute("PRAGMA temp_store=MEMORY")
self.conn.execute("PRAGMA cache_size=-200000")
self.conn.execute("PRAGMA foreign_keys=OFF")
self.cur = self.conn.cursor()
def begin(self):
self.conn.execute("BEGIN")
def execute(self, sql, params=()):
self.cur.execute(sql, params)
def execute_script(self, script):
self.cur.executescript(script)
def insert_many(self, table, columns, rows):
sql = "INSERT INTO {} ({}) VALUES ({})".format(
table, ",".join(columns), ",".join("?" * len(columns)))
self.cur.executemany(sql, rows)
def commit(self):
self.conn.commit()
def rollback(self):
self.conn.rollback()
def close(self):
self.conn.close()
def table_names(self):
return {r[0] for r in self.cur.execute(
"SELECT name FROM sqlite_master WHERE type='table'")}
class PGAdapter:
placeholder = "%s"
def __init__(self, dsn):
try:
import psycopg2
from psycopg2.extras import execute_values
except ImportError:
try:
import psycopg as psycopg2
from psycopg.extras import execute_values
except ImportError as exc:
raise SystemExit(
"需要 psycopg2:pip install psycopg2-binary") from exc
self.conn = psycopg2.connect(dsn)
self.conn.autocommit = False
self.cur = self.conn.cursor()
self._execute_values = execute_values
def begin(self):
pass
def execute(self, sql, params=()):
self.cur.execute(sql, params)
def execute_script(self, script):
for stmt in script.split(";"):
if stmt.strip():
self.cur.execute(stmt)
def insert_many(self, table, columns, rows):
base = "INSERT INTO {} ({}) VALUES %s".format(
table, ",".join(columns))
self._execute_values(self.cur, base, rows, page_size=1000)
def commit(self):
self.conn.commit()
def rollback(self):
self.conn.rollback()
def close(self):
self.conn.close()
def table_names(self):
self.cur.execute(
"SELECT table_name FROM information_schema.tables "
"WHERE table_schema='public'")
return {r[0] for r in self.cur.fetchall()}
class MySQLAdapter:
placeholder = "%s"
def __init__(self, dsn):
try:
import pymysql
except ImportError:
try:
import mysql.connector as pymysql
except ImportError as exc:
raise SystemExit(
"需要 MySQL 驱动:pip install pymysql") from exc
parsed = urlparse(dsn)
if parsed.scheme not in ("mysql", "mysql+pymysql", ""):
raise SystemExit("MySQL DSN 应形如 mysql://user:pass@host:3306/db")
self.conn = pymysql.connect(
host=parsed.hostname or "127.0.0.1",
port=parsed.port or 3306,
user=unquote(parsed.username or "root"),
password=unquote(parsed.password or ""),
database=(parsed.path or "/").lstrip("/"),
charset="utf8mb4",
autocommit=False,
)
self.cur = self.conn.cursor()
def begin(self):
pass
def execute(self, sql, params=()):
self.cur.execute(sql, params)
def execute_script(self, script):
for stmt in script.split(";"):
if stmt.strip():
self.cur.execute(stmt)
def insert_many(self, table, columns, rows):
sql = "INSERT INTO {} ({}) VALUES ({})".format(
table, ",".join(columns), ",".join(["%s"] * len(columns)))
self.cur.executemany(sql, rows)
def commit(self):
self.conn.commit()
def rollback(self):
self.conn.rollback()
def close(self):
self.conn.close()
def table_names(self):
self.cur.execute(
"SELECT table_name FROM information_schema.tables "
"WHERE table_schema = DATABASE()")
return {r[0] for r in self.cur.fetchall()}
NON_COLUMN_KEYWORDS = {
"PRIMARY", "FOREIGN", "UNIQUE", "CHECK", "CONSTRAINT", "KEY", "INDEX",
}
def annotate_ddl(ddl, dialect):
"""为每个字段追加注释;MySQL 同时把行内 REFERENCES 提升为表级外键。"""
out = []
table = None
fks = []
col_re = re.compile(r"^(\s+)([a-z_][a-z0-9_]*)\s+(.+?)(,?)\s*$")
for line in ddl.splitlines():
m = re.match(r"\s*CREATE TABLE\s+(\w+)\s*\(", line, re.I)
if m:
table = m.group(1)
fks = []
elif table and line.strip().startswith(")"):
if fks:
if out and not out[-1].rstrip().endswith(","):
out[-1] = out[-1].rstrip() + ","
out.append(" " + ",\n ".join(fks))
table = None
mc = col_re.match(line) if table else None
if mc:
col, body, comma = mc.group(2), mc.group(3).rstrip(), mc.group(4)
if col.upper() not in NON_COLUMN_KEYWORDS:
comment = COLUMN_COMMENTS.get((table, col))
if comment is None:
raise ValueError(f"缺少字段注释: {table}.{col}")
esc = comment.replace("'", "''")
if dialect == "mysql":
ref = re.search(
r"\s+REFERENCES\s+(\w+)\s*\(([^)]+)\)\s*$", body, re.I)
if ref:
body = body[:ref.start()].rstrip()
fks.append("FOREIGN KEY ({}) REFERENCES {}({})".format(
col, ref.group(1), ref.group(2)))
chk = re.search(r"\s+CHECK\s*\((.+)\)\s*$", body, re.I)
if chk:
body = body[:chk.start()].rstrip()
fks.append("CHECK ({})".format(chk.group(1)))
line = "{}{} {} COMMENT '{}'{}".format(
mc.group(1), col, body, esc, comma)
else:
line = line.rstrip()
if line.endswith(","):
line = line[:-1].rstrip() + f", -- {comment}"
else:
line = line + f" -- {comment}"
out.append(line)
return "\n".join(out)
def pg_comment_statements():
stmts = []
for table in TABLES:
for col in COLUMNS[table]:
comment = COLUMN_COMMENTS.get((table, col))
if comment is None:
raise ValueError(f"缺少字段注释: {table}.{col}")
esc = comment.replace("'", "''")
stmts.append(f"COMMENT ON COLUMN {table}.{col} IS '{esc}'")
return stmts
def sqlite_ddl():
ddl = SCHEMA_PG
ddl = re.sub(r"\bDEFERRABLE INITIALLY DEFERRED\b", "", ddl)
ddl = re.sub(r"\bjsonb\b", "TEXT", ddl)
ddl = re.sub(r"\bboolean\b", "INTEGER", ddl)
ddl = re.sub(r"\bbigint\b", "INTEGER", ddl)
ddl = re.sub(r"\bsmallint\b", "INTEGER", ddl)
ddl = re.sub(r"\bDEFAULT FALSE\b", "DEFAULT 0", ddl, flags=re.I)
ddl = re.sub(r"\bDEFAULT TRUE\b", "DEFAULT 1", ddl, flags=re.I)
return annotate_ddl(ddl, "sqlite")
def mysql_ddl():
ddl = SCHEMA_PG
ddl = re.sub(r"\bDEFERRABLE INITIALLY DEFERRED\b", "", ddl)
ddl = re.sub(r"\bjsonb\b", "JSON", ddl)
ddl = re.sub(r"\bboolean\b", "TINYINT(1)", ddl)
ddl = re.sub(r"\bDEFAULT FALSE\b", "DEFAULT 0", ddl, flags=re.I)
ddl = re.sub(r"\bDEFAULT TRUE\b", "DEFAULT 1", ddl, flags=re.I)
parts = []
for stmt in ddl.split(";"):
if not stmt.strip():
continue
stmt = stmt.rstrip()
if stmt.lstrip().upper().startswith("CREATE TABLE"):
# 必须用 utf8mb4_bin:源数据存在仅大小写不同的唯一键组合
# (如 confusable_word 同一 entry 下 italic / Italic),
# _ci 排序规则会把二者视为重复,与 SQLite/PostgreSQL 语义不一致。
stmt += " ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"
parts.append(stmt)
ddl = ";\n".join(parts) + ";"
return annotate_ddl(ddl, "mysql")
# ---------------------------------------------------------------------------
# 批量写入
# ---------------------------------------------------------------------------
class Writer:
def __init__(self, adapter, batch_size=10000):
self.adapter = adapter
self.batch_size = batch_size
self.buffers = defaultdict(list)
self.counts = defaultdict(int)
def add(self, table, row):
self.buffers[table].append(row)
if len(self.buffers[table]) >= self.batch_size:
# 按依赖顺序统一刷盘,兼容 MySQL 的即时外键检查
self.flush_all()
def flush(self, table):
buf = self.buffers[table]
if not buf:
return
self.adapter.insert_many(table, COLUMNS[table], buf)
self.counts[table] += len(buf)
buf.clear()
def flush_all(self):
for table in TABLES:
self.flush(table)
def nid(name):
nid.value[name] += 1
return nid.value[name]
nid.value = defaultdict(int)
# ---------------------------------------------------------------------------
# 导入:公共维表
# ---------------------------------------------------------------------------
def import_exams(w, data_dir):
seen = set()
path = os.path.join(data_dir, "wordroot-export", "word_family_exam.csv")
if os.path.exists(path):
for r in read_csv(path):
code = (r.get("exam") or "").strip()
eid = _to_int(r.get("id"))
if not code or eid is None:
continue
name = (r.get("tag") or "").strip() or code
usable = 1 if eid in GRADE_USABLE else 0
w.add("exam", (eid, code, name, usable, 0))
seen.add(eid)
for code, (eid, name, pseudo) in EXAM_CODES.items():
if eid not in seen:
usable = 1 if eid in GRADE_USABLE else 0
w.add("exam", (eid, code, name, usable, 1 if pseudo else 0))
w.flush("exam")
print(f" exam: {w.counts['exam']}")
# ---------------------------------------------------------------------------
# 导入:词书
# ---------------------------------------------------------------------------
def scan_books(data_dir, limit_books=None):
"""返回 books(meta), canonical(entry -> (ts,key,word_hint)), phrase_book_ids"""
pattern = os.path.join(data_dir, "wordbooks", "decrypted", "LangEasyLexisV3_*.csv")
files = sorted(glob.glob(pattern))
if limit_books:
files = files[:limit_books]
books = {}
canonical = {}
for path in files:
code = os.path.basename(path)[len("LangEasyLexisV3_"):-4]
first = None
with open(path, encoding="utf-8-sig", newline="") as f:
for row in csv.DictReader(f):
if first is None:
first = row
eid = _to_int(row.get("id"))
key = (row.get("zpk") or "").strip()
if eid is None or not key:
continue
ts = _to_int(key.rsplit("_", 1)[-1], 0)
hint = (row.get("word") or "").strip() or None
prev = canonical.get(eid)
if prev is None or ts > prev[0]:
canonical[eid] = (ts, key, hint)
if first is not None:
books[code] = first
return books, canonical
def import_books(w, data_dir, limit_books=None):
catalog_path = os.path.join(data_dir, "book-catalog.json")
catalog = []
if os.path.exists(catalog_path):
with open(catalog_path, encoding="utf-8-sig") as f:
catalog = json.load(f)
by_code = {c.get("code"): c for c in catalog}
books, canonical = scan_books(data_dir, limit_books)
book_ids = set()
phrase_pairs = []
for code, first in books.items():
meta = by_code.get(code, {})
book_id = _to_int(first.get("book_id"))
book_ids.add(book_id)
exam_id = exam_id_of(first.get("book_exam") or meta.get("exam"))
name = (first.get("book_name") or meta.get("name") or code).strip()
declared = _to_int(first.get("book_count"), meta.get("count"))
w.add("book", (
book_id,
code,
name,
_text(meta.get("desc")),
exam_id,
declared,
_text(meta.get("cover")),
_text(meta.get("url")),
_to_int(meta.get("version")),
_to_int(meta.get("bookVersion")),
_text(meta.get("sortBy")),
_to_int(meta.get("sortTime")),
_to_int(meta.get("forSale")),
_to_int(meta.get("purchased")),
_to_int(meta.get("opt")),
_to_int(meta.get("src")),
_to_int(meta.get("type")),
_text(meta.get("labels")),
_text(meta.get("publishLog")),
json.dumps(meta.get("pkgPage"), ensure_ascii=False) if meta.get("pkgPage") is not None else None,
json.dumps(meta.get("externalLink"), ensure_ascii=False) if meta.get("externalLink") is not None else None,
1,
))
for pos, pid in enumerate(meta.get("phraseBookIds") or []):
pid = _to_int(pid)
if pid is not None:
phrase_pairs.append((book_id, pid, pos))
# 目录中存在但磁盘无清单的词书
for code, meta in by_code.items():
if code in books:
continue
book_id = _to_int(meta.get("id"))
if book_id is None:
continue
book_ids.add(book_id)
w.add("book", (
book_id, code, (meta.get("name") or code),
_text(meta.get("desc")),
exam_id_of(meta.get("exam")),
_to_int(meta.get("count")),
_text(meta.get("cover")), _text(meta.get("url")),
_to_int(meta.get("version")), _to_int(meta.get("bookVersion")),
_text(meta.get("sortBy")), _to_int(meta.get("sortTime")),
_to_int(meta.get("forSale")), _to_int(meta.get("purchased")),
_to_int(meta.get("opt")), _to_int(meta.get("src")),
_to_int(meta.get("type")), _text(meta.get("labels")),
_text(meta.get("publishLog")),
json.dumps(meta.get("pkgPage"), ensure_ascii=False) if meta.get("pkgPage") is not None else None,
json.dumps(meta.get("externalLink"), ensure_ascii=False) if meta.get("externalLink") is not None else None,
0,
))
for pos, pid in enumerate(meta.get("phraseBookIds") or []):
pid = _to_int(pid)
if pid is not None:
phrase_pairs.append((book_id, pid, pos))
w.flush("book")
seen_pp = set()
for pair in phrase_pairs:
if pair[1] in book_ids and pair[:2] not in seen_pp:
seen_pp.add(pair[:2])
w.add("book_phrase_book", pair)
w.flush("book_phrase_book")
print(f" book: {w.counts['book']} book_phrase_book: {w.counts['book_phrase_book']}")
return books, canonical
def import_book_entries(w, data_dir, limit_books=None):
pattern = os.path.join(data_dir, "wordbooks", "decrypted", "LangEasyLexisV3_*.csv")
files = sorted(glob.glob(pattern))
if limit_books:
files = files[:limit_books]
seq = 0
for path in files:
for r in read_csv(path):
seq += 1
w.add("book_entry", (
seq,
_to_int(r.get("book_id")),
_to_int(r.get("id")),
(r.get("word") or "").strip(),
_text((r.get("main_word") or "").strip()),
_to_int(r.get("seq_book")),
_to_int(r.get("seq_az")),
_to_int(r.get("seq_freq")),
_to_int(r.get("seq_rand")),
_text((r.get("zpk") or "").strip()),
))
w.flush("book_entry")
print(f" book_entry: {w.counts['book_entry']}")
# ---------------------------------------------------------------------------
# 导入:zpk 词条
# ---------------------------------------------------------------------------
def _load_example(d):
raw = d.get("example")
if isinstance(raw, dict):
return raw
if isinstance(raw, str) and raw.strip():
try:
return json.loads(raw)
except json.JSONDecodeError:
return {}
return {}
def _as_list(value):
if value is None:
return []
if isinstance(value, (list, tuple)):
return list(value)
return [value]
def import_word_entries(w, data_dir, canonical):
zpk_dir = os.path.join(data_dir, "zpk", "decrypted")
total = len(canonical)
done = 0
t0 = time.time()
for eid in sorted(canonical):
ts, key, hint = canonical[eid]
path = os.path.join(zpk_dir, key + ".json")
stem = key.split("_")
variant_code = _to_int(stem[2]) if len(stem) > 2 else None
if not os.path.exists(path):
# 缺失文件:写入占位词条,保证 book_entry 外键成立
w.add("word_entry", (
eid, hint or str(eid), (hint or str(eid)).lower(),
None, None, None, None, None, 0, 0, key, variant_code, ts))
done += 1
continue
try:
with open(path, encoding="utf-8") as f:
d = json.load(f)
except (json.JSONDecodeError, OSError) as exc:
print(f" ! 跳过损坏 JSON {key}: {exc}")
w.add("word_entry", (
eid, hint or str(eid), (hint or str(eid)).lower(),
None, None, None, None, None, 0, 0, key, variant_code, ts))
done += 1
continue
meta = d.get("meta") or {}
ex = _load_example(d)
word = (d.get("word") or hint or str(eid)).strip()
w.add("word_entry", (
eid, word, (d.get("index_word") or word).strip(),
_text(d.get("interpret")),
_text(d.get("us_pron")), _text(d.get("uk_pron")),
_text(meta.get("data_version")),
exam_id_of(meta.get("book_exam")),
_to_int(d.get("book_occurrences"), 0),
1 if ex.get("is_type_99") else 0,
key, variant_code, ts,
))
for pos, sp in enumerate(d.get("spell") or []):
w.add("word_spelling_variant", (eid, pos, str(sp)))
# 义项 + 释义对象
meaning_map = {}
relations = []
for group in d.get("interpret_v2") or []:
pos_name = group.get("p")
for sense in group.get("d") or []:
sid = nid("sense")
w.add("word_sense", (
sid, eid, _to_int(sense.get("id"), 0), pos_name,
_text(sense.get("i")),
_text(",".join(str(x) for x in _as_list(sense.get("s")))),
_to_int(sense.get("h")),
))
seen_exam = set()
for code in _as_list(sense.get("t")):
xid = exam_id_of(code)
if xid is not None and xid not in seen_exam:
seen_exam.add(xid)
w.add("sense_exam", (sid, xid))
for oid in _as_list(sense.get("oids")):
oid = _to_int(oid)
if oid is None:
continue
mid = nid("meaning")
w.add("sense_object", (mid, eid, oid, sid))
meaning_map.setdefault(oid, mid)
for field, rel in (("synids", "synonym"),
("antonymids", "antonym"),
("refids", "reference")):
for apos, target in enumerate(_as_list(sense.get(field))):
target = _to_int(target)
if target is not None:
relations.append((sid, rel, apos, target))
for sid, rel, apos, target in relations:
mid = meaning_map.get(target)
if mid is not None:
w.add("sense_relation", (sid, rel, apos, mid))
# 短语
phrase_map = {}
for pg in d.get("phrase") or []:
gtype = _to_int(pg.get("t"), 0)
for x in pg.get("p") or []:
pid = nid("phrase")
src_pid = _to_int(x.get("pid"))
if src_pid is not None:
phrase_map.setdefault(src_pid, pid)
oid = _to_int(x.get("oid"))
mid = meaning_map.get(oid) if oid is not None else None
w.add("phrase", (
pid, eid, src_pid, mid, gtype,
_text(x.get("en")) or "", _text(x.get("cn")),
_text(x.get("fid")), _text(x.get("bd")),
))
seen_exam = set()
for code in _as_list(x.get("exams")):
xid = exam_id_of(code)
if xid is not None and xid not in seen_exam:
seen_exam.add(xid)
w.add("phrase_exam", (pid, xid))
for code, pids in (pg.get("pseqs") or {}).items():
xid = exam_id_of(code)
if xid is None:
continue
for pos, src_pid in enumerate(pids):
pid = phrase_map.get(_to_int(src_pid))
if pid is not None:
w.add("phrase_exam_order", (eid, xid, gtype, pos, pid))
# 例句块
for block in ex.get("data") or []:
kind = _to_int(block.get("type"), 99)
info = block.get("i") or {}
exam_ids = [exam_id_of(c) for c in _as_list(block.get("exams"))]
exam_ids = list(dict.fromkeys(x for x in exam_ids if x is not None))
source_codes = ",".join(
str(c) for c in _as_list(block.get("exams")) if c)
if str(kind) == "99":
egid = nid("example_group")
w.add("example_group", (
egid, eid, None, 99, None, None, None,
None, None, None, None, source_codes))
for sent in block.get("g") or []:
if not isinstance(sent, dict):
continue
w.add("example_sentence", _sentence_row(sent, egid, None, True))
continue
oid = _to_int(block.get("oid"))
mid = meaning_map.get(oid) if oid is not None else None
egid = nid("example_group")
w.add("example_group", (
egid, eid, mid, kind,
_text(info.get("p")), _text(info.get("t")), _text(info.get("u")),
_text(info.get("e")), _text(info.get("c")),
_text(info.get("yu")), _text(info.get("yo")), source_codes,
))
for xid in exam_ids:
w.add("example_group_exam", (egid, xid))
for upos, usage in enumerate(block.get("g") or []):
uid = nid("usage")
src_uid = _to_int(usage.get("uid"))
pattern_id = phrase_map.get(src_uid) if src_uid else None
w.add("example_usage", (
uid, egid, pattern_id, _text(usage.get("u")), upos))
for sent in usage.get("s") or []:
if not isinstance(sent, dict):
continue
w.add("example_sentence", _sentence_row(sent, egid, uid, False))
# 考试例句
for code, arr in (d.get("exam_sentences") or {}).items():
xid = exam_id_of(code)
if xid is None:
continue
for pos, item in enumerate(arr or []):
if not isinstance(item, dict):
continue
w.add("entry_exam_sentence", (
eid, xid, pos,
_text(item.get("en")) or "",
_text(item.get("cn")), _text(item.get("src")),
))
# 义项顺序
for code, obj in (ex.get("oseqs") or {}).items():
xid = exam_id_of(code)
if xid is None or not isinstance(obj, dict):
continue
for scope in ("core", "ext"):
seen_meaning = set()
for pos, oid in enumerate(obj.get(scope) or []):
mid = meaning_map.get(_to_int(oid))
if mid is not None and mid not in seen_meaning:
seen_meaning.add(mid)
w.add("entry_exam_meaning_order",
(eid, xid, scope, pos, mid))
# 易混词
seen_confuse = set()
cv = d.get("confuse_v2")
if cv:
for pos, item in enumerate(cv):
if not isinstance(item, dict):
continue
text = (item.get("w") or "").strip()
if not text or text in seen_confuse:
continue
seen_confuse.add(text)
w.add("confusable_word", (
nid("confusable"), eid, pos, text,
_text(item.get("i")), _to_int(item.get("p")), _text(item.get("sp"))))
else:
conf = d.get("confuse") or []
exps = d.get("confuse_exps") or []
for pos, text in enumerate(conf):
text = str(text).strip()
if not text or text in seen_confuse:
continue
seen_confuse.add(text)
gloss = exps[pos] if pos < len(exps) else None
w.add("confusable_word", (
nid("confusable"), eid, pos, text, _text(gloss), None, None))
done += 1
if done % 2000 == 0:
elapsed = time.time() - t0
print(f" zpk: {done}/{total} ({elapsed:.0f}s)")
w.flush_all()
print(f" word_entry: {w.counts['word_entry']} "
f"word_sense: {w.counts['word_sense']} "
f"phrase: {w.counts['phrase']} "
f"example_sentence: {w.counts['example_sentence']}")
def _sentence_row(sent, egid, uid, listening):
return (
nid("sentence"),
_text((sent.get("sid") or "")),
_text((sent.get("eid") or "")),
egid, uid, _to_int(sent.get("s")),
_text(sent.get("e")) or "",
_text(sent.get("c")),
_text(sent.get("fid")), _text(sent.get("u")), _text(sent.get("i")),
_text(sent.get("b")),
1 if listening else 0,
)
# ---------------------------------------------------------------------------
# 导入:词根词族
# ---------------------------------------------------------------------------
def import_wordroot(w, data_dir):
base = os.path.join(data_dir, "wordroot-export")
# 词素
morpheme_map = {}
for typ, fname in (("prefix", "prefix.csv"), ("root", "root.csv"),
("suffix", "suffix.csv")):
path = os.path.join(base, fname)
if not os.path.exists(path):
continue
for r in read_csv(path):
src = _to_int(r.get("id"))
if src is None:
continue
mid = nid("morpheme")
morpheme_map[(typ, src)] = mid
form, en, zh, note = parse_affix(r.get("display"))
w.add("morpheme", (mid, typ, src, form, r.get("display") or "", en, zh, note))
w.flush("morpheme")
def morph(typ, value):
src = _to_int(value)
if src is None or src <= 0:
return None
return morpheme_map.get((typ, src))
# 卡片
card_path = os.path.join(base, "card_info.csv")
if os.path.exists(card_path):
for r in read_csv(card_path):
cid = _to_int(r.get("id"))
if cid is None:
continue
w.add("card", (
cid, (r.get("word") or "").strip(),
_text(r.get("hightlight")), _text(r.get("memory")),
_text(r.get("example")),
morph("prefix", r.get("prefix")),
morph("root", r.get("root")),
morph("suffix", r.get("suffix")),
_to_int(r.get("default_f_id")),
_to_bool(r.get("hidden")),
))
seen_exam = set()
for g in (r.get("grade") or "").split(","):
xid = _to_int(g)
if xid is not None and xid not in seen_exam:
seen_exam.add(xid)
w.add("card_exam", (cid, xid))
w.flush("card")
# 词族
family_path = os.path.join(base, "word_family.csv")
family_ids = set()
family_rows = list(read_csv(family_path)) if os.path.exists(family_path) else []
for r in family_rows:
fid = _to_int(r.get("id"))
if fid is not None:
family_ids.add(fid)
# 先插入自指根行,满足自外键
ordered = [r for r in family_rows if _to_int(r.get("id")) == _to_int(r.get("top_id"))]
ordered += [r for r in family_rows if _to_int(r.get("id")) != _to_int(r.get("top_id"))]
for r in ordered:
fid = _to_int(r.get("id"))
if fid is None:
continue
w.add("word_family", (
fid, _to_int(r.get("top_id"), fid), (r.get("word") or "").strip(),
_text(r.get("hightlight")), _text(r.get("interpret")),
morph("prefix", r.get("prefix")), morph("suffix", r.get("suffix")),
_to_int(r.get("group1")), _to_int(r.get("group2")), _to_int(r.get("group3")),
))
seen_exam = set()
for g in (r.get("grade") or "").split(","):
xid = _to_int(g)
if xid is not None and xid not in seen_exam:
seen_exam.add(xid)
w.add("word_family_exam", (fid, xid))
try:
j = json.loads(r.get("json") or "{}")
except json.JSONDecodeError:
j = {}
for key, value in j.items():
xid = _to_int(key)
pos = _to_int(value)
if xid is not None and pos is not None:
w.add("word_family_exam_order", (fid, xid, pos))
w.flush("word_family")
# 变体
var_path = os.path.join(base, "word_family_variant.csv")
variant_ids = []
if os.path.exists(var_path):
for r in read_csv(var_path):
vid = _to_int(r.get("id"))
fid = _to_int(r.get("word_family_id"))
if vid is None or fid not in family_ids:
continue
variant_ids.append(vid)
w.add("word_family_variant", (
vid, (r.get("variant") or "").strip(), fid,
_to_int(r.get("top_id"), fid), _text(r.get("hightlight")),
_text(r.get("interpret")),
morph("prefix", r.get("prefix")), morph("suffix", r.get("suffix")),
))
try:
j = json.loads(r.get("json") or "{}")
except json.JSONDecodeError:
j = {}
for key, words in j.items():
xid = _to_int(key)
if xid is None or not isinstance(words, list):
continue
for pos, word in enumerate(words):
text = str(word)
w.add("word_family_variant_member", (
vid, xid, pos, text, 1 if text == "*" else 0))
w.flush("word_family_variant")
# 小组
group_path = os.path.join(base, "word_family_mini_group.csv")
if os.path.exists(group_path):
for r in read_csv(group_path):
gid = _to_int(r.get("id"))
if gid is None:
continue
tokens = [x.strip() for x in (r.get("word_ids") or "").split(",") if x.strip()]
w.add("word_family_mini_group", (gid, len(tokens)))
seen = set()
for pos, tok in enumerate(tokens):
fid = _to_int(tok)
if fid is None or fid not in family_ids or fid in seen:
continue
seen.add(fid)
w.add("word_family_mini_group_member", (gid, fid, pos))
w.flush("word_family_mini_group")
# 卡片—分组
wg_path = os.path.join(base, "word_group.csv")
if os.path.exists(wg_path):
for r in read_csv(wg_path):
lid = _to_int(r.get("id"))
cid = _to_int(r.get("card_info_id"))
if lid is None or cid is None:
continue
w.add("card_family_link", (
lid, cid, _to_int(r.get("group_id"), 0),
_to_int(r.get("f_id")), _to_bool(r.get("hidden")),
))
w.flush("card_family_link")
print(f" morpheme: {w.counts['morpheme']} card: {w.counts['card']} "
f"word_family: {w.counts['word_family']} "
f"variant: {w.counts['word_family_variant']} "
f"mini_group: {w.counts['word_family_mini_group']}")
# ---------------------------------------------------------------------------
# 导入:近义词
# ---------------------------------------------------------------------------
def import_synonyms(w, data_dir):
base = os.path.join(data_dir, "wordroot-export")
words_path = os.path.join(base, "synonym_words.csv")
word_ids = set()
if os.path.exists(words_path):
for r in read_csv(words_path):
wid = _to_int(r.get("id"))
if wid is None:
continue
word_ids.add(wid)
w.add("synonym_word", (wid, (r.get("word") or "").strip()))
w.flush("synonym_word")
data_path = os.path.join(base, "synonym_data.csv")
if not os.path.exists(data_path):
print(" synonym_data.csv 不存在,跳过")
return
for r in read_csv(data_path):
sid = _to_int(r.get("id"))
wid = _to_int(r.get("word_id"))
if sid is None or wid is None:
continue
w.add("synonym_sense", (
sid, wid, _text(r.get("pos")), _text(r.get("interpret"))))
for column, xid in SYNONYM_EXAM_COLUMNS.items():
raw = (r.get(column) or "").strip()
if not raw:
continue
try:
cell = json.loads(raw)
except json.JSONDecodeError:
continue
syn = [x for x in str(cell.get("syn") or "").split(",") if x.strip()]
ant = [x for x in str(cell.get("ant") or "").split(",") if x.strip()]
free = _to_int(cell.get("free"))
if free is not None and free not in word_ids:
free = None
syn_ids = []
seen = set()
for x in syn:
v = _to_int(x)
if v in word_ids and v not in seen:
seen.add(v)
syn_ids.append(v)
w.add("synonym_group", (sid, xid, free, len(syn_ids)))
for pos, v in enumerate(syn_ids):
w.add("synonym_group_member", (sid, xid, pos, v))
for pos, x in enumerate(ant):
v = _to_int(x)
if v in word_ids:
w.add("synonym_antonym", (sid, xid, pos, v))
w.flush_all()
print(f" synonym_word: {w.counts['synonym_word']} "
f"synonym_sense: {w.counts['synonym_sense']} "
f"synonym_group: {w.counts['synonym_group']}")
# ---------------------------------------------------------------------------
# 主流程
# ---------------------------------------------------------------------------
def drop_schema(adapter):
if isinstance(adapter, SQLiteAdapter):
adapter.execute("PRAGMA foreign_keys=OFF")
for table in reversed(TABLES):
adapter.execute(f"DROP TABLE IF EXISTS {table}")
elif isinstance(adapter, MySQLAdapter):
adapter.execute("SET FOREIGN_KEY_CHECKS=0")
for table in reversed(TABLES):
adapter.execute(f"DROP TABLE IF EXISTS {table}")
adapter.execute("SET FOREIGN_KEY_CHECKS=1")
else:
for table in reversed(TABLES):
adapter.execute(f"DROP TABLE IF EXISTS {table} CASCADE")
def create_schema(adapter):
if isinstance(adapter, SQLiteAdapter):
script = sqlite_ddl()
elif isinstance(adapter, MySQLAdapter):
script = mysql_ddl()
else:
script = annotate_ddl(SCHEMA_PG, "pg")
adapter.execute_script(script)
if isinstance(adapter, PGAdapter):
for stmt in pg_comment_statements():
adapter.execute(stmt)
def create_indexes(adapter):
for stmt in INDEXES.split(";"):
if stmt.strip():
adapter.execute(stmt)
def print_counts(adapter):
print("\n导入结果:")
total = 0
for table in TABLES:
try:
adapter.execute(f"SELECT COUNT(*) FROM {table}")
n = adapter.cur.fetchone()[0]
except Exception:
n = -1
total += max(n, 0)
print(f" {table:<34} {n:>10,}")
print(f" {'TOTAL':<34} {total:>10,}")
def validate_sqlite(adapter):
try:
rows = adapter.cur.execute("PRAGMA foreign_key_check").fetchall()
except Exception as exc:
print(f" 外键校验失败: {exc}")
return
print(f"\n外键校验: {'通过' if not rows else f'发现 {len(rows)} 条违规'}")
for row in rows[:10]:
print(" ", row)
def main(argv=None):
here = os.path.dirname(os.path.abspath(__file__))
default_data = os.path.join(here, "output")
if not os.path.isdir(default_data):
default_data = os.path.join(os.getcwd(), "output")
ap = argparse.ArgumentParser(description="Lexis 词库一键建表并导入")
ap.add_argument("--db", choices=("sqlite", "postgres", "mysql"), default="sqlite")
ap.add_argument("--sqlite-path", default=os.path.join(here, "lexis.db"))
ap.add_argument("--dsn", help="PostgreSQL / MySQL DSN")
ap.add_argument("--data-dir", default=default_data)
ap.add_argument("--recreate", action="store_true", help="删除并重建所有表")
ap.add_argument("--batch-size", type=int, default=10000)
ap.add_argument("--limit-books", type=int, default=None,
help="仅导入前 N 本词书(调试用)")
args = ap.parse_args(argv)
data_dir = os.path.abspath(args.data_dir)
if not os.path.isdir(data_dir):
raise SystemExit(f"数据目录不存在: {data_dir}")
nid.value.clear()
print(f"数据目录: {data_dir}")
if args.db == "sqlite":
adapter = SQLiteAdapter(args.sqlite_path)
print(f"目标: SQLite {os.path.abspath(args.sqlite_path)}")
elif args.db == "mysql":
if not args.dsn:
raise SystemExit("--db mysql 时必须提供 --dsn")
adapter = MySQLAdapter(args.dsn)
print("目标: MySQL")
else:
if not args.dsn:
raise SystemExit("--db postgres 时必须提供 --dsn")
adapter = PGAdapter(args.dsn)
print("目标: PostgreSQL")
existing = adapter.table_names() & set(TABLES)
if existing and not args.recreate:
raise SystemExit(
"数据库中已存在本模型的表;如需重建请加 --recreate")
if existing and args.recreate:
print("删除旧表...")
drop_schema(adapter)
print("建表...")
create_schema(adapter)
if isinstance(adapter, MySQLAdapter):
adapter.execute("SET FOREIGN_KEY_CHECKS=0")
adapter.execute("SET UNIQUE_CHECKS=0")
adapter.begin()
w = Writer(adapter, batch_size=args.batch_size)
t0 = time.time()
try:
print("导入考试维表...")
import_exams(w, data_dir)
print("扫描词书清单...")
books, canonical = import_books(w, data_dir, args.limit_books)
print(f" 词书 {len(books)} 本,词条 {len(canonical)} 个")
print("导入词条词典(每个 entry 取最新 zpk 版本)...")
import_word_entries(w, data_dir, canonical)
print("导入词书词条...")
import_book_entries(w, data_dir, args.limit_books)
print("导入词根词族...")
import_wordroot(w, data_dir)
print("导入近反义词...")
import_synonyms(w, data_dir)
adapter.commit()
except Exception:
adapter.rollback()
raise
if isinstance(adapter, MySQLAdapter):
adapter.execute("SET FOREIGN_KEY_CHECKS=1")
adapter.execute("SET UNIQUE_CHECKS=1")
print(f"\n导入耗时 {time.time() - t0:.1f}s,创建索引...")
adapter.begin()
create_indexes(adapter)
adapter.commit()
print_counts(adapter)
if isinstance(adapter, SQLiteAdapter):
validate_sqlite(adapter)
adapter.close()
print("\n完成。")
if __name__ == "__main__":
main()