84 lines
2.5 KiB
Python
84 lines
2.5 KiB
Python
|
"""词典导入共享工具。"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import re
|
|||
|
|
from typing import Optional
|
|||
|
|
|
|||
|
|
from sqlalchemy.dialects.mysql import insert as mysql_insert
|
|||
|
|
from sqlalchemy.orm import Session
|
|||
|
|
|
|||
|
|
from models import DictionaryEntry
|
|||
|
|
|
|||
|
|
_POS_PREFIX = re.compile(r"^[a-zA-Z]+\.\s*")
|
|||
|
|
_TAG_PREFIX = re.compile(r"^\[[^\]]+\]\s*")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def clean_translation(text: str, max_len: int = 256) -> str:
|
|||
|
|
if not text:
|
|||
|
|
return ""
|
|||
|
|
lines = [ln.strip() for ln in text.replace("\r", "").split("\n") if ln.strip()]
|
|||
|
|
parts: list[str] = []
|
|||
|
|
for ln in lines:
|
|||
|
|
ln = _TAG_PREFIX.sub("", ln)
|
|||
|
|
ln = _POS_PREFIX.sub("", ln)
|
|||
|
|
if ln:
|
|||
|
|
parts.append(ln)
|
|||
|
|
result = "；".join(parts) if parts else text.strip()
|
|||
|
|
return result[:max_len]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def normalize_row(raw: dict, source: Optional[str]) -> Optional[dict]:
|
|||
|
|
lemma_en = (raw.get("lemma_en") or raw.get("en") or raw.get("word") or "").strip().lower()
|
|||
|
|
zh = clean_translation(raw.get("zh") or raw.get("cn") or raw.get("translation") or "")
|
|||
|
|
if not lemma_en or not zh:
|
|||
|
|
return None
|
|||
|
|
if len(lemma_en) > 128:
|
|||
|
|
return None
|
|||
|
|
phonetic = (raw.get("phonetic") or "").strip() or None
|
|||
|
|
if phonetic and len(phonetic) > 128:
|
|||
|
|
phonetic = phonetic[:128]
|
|||
|
|
return {
|
|||
|
|
"lemma_en": lemma_en,
|
|||
|
|
"zh": zh,
|
|||
|
|
"phonetic": phonetic,
|
|||
|
|
"example_en": (raw.get("example_en") or "").strip() or None,
|
|||
|
|
"example_cn": (raw.get("example_cn") or "").strip() or None,
|
|||
|
|
"source": source or (raw.get("source") or "import"),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def upsert_batch(session: Session, rows: list[dict]) -> int:
|
|||
|
|
if not rows:
|
|||
|
|
return 0
|
|||
|
|
stmt = mysql_insert(DictionaryEntry).values(rows)
|
|||
|
|
stmt = stmt.on_duplicate_key_update(
|
|||
|
|
zh=stmt.inserted.zh,
|
|||
|
|
phonetic=stmt.inserted.phonetic,
|
|||
|
|
example_en=stmt.inserted.example_en,
|
|||
|
|
example_cn=stmt.inserted.example_cn,
|
|||
|
|
source=stmt.inserted.source,
|
|||
|
|
)
|
|||
|
|
session.execute(stmt)
|
|||
|
|
return len(rows)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def upsert_entries(session: Session, raw_rows: list[dict], source: Optional[str], batch_size: int = 2000) -> int:
|
|||
|
|
batch: list[dict] = []
|
|||
|
|
total = 0
|
|||
|
|
for raw in raw_rows:
|
|||
|
|
row = normalize_row(raw, source)
|
|||
|
|
if not row:
|
|||
|
|
continue
|
|||
|
|
batch.append(row)
|
|||
|
|
if len(batch) >= batch_size:
|
|||
|
|
upsert_batch(session, batch)
|
|||
|
|
session.commit()
|
|||
|
|
total += len(batch)
|
|||
|
|
batch.clear()
|
|||
|
|
if batch:
|
|||
|
|
upsert_batch(session, batch)
|
|||
|
|
session.commit()
|
|||
|
|
total += len(batch)
|
|||
|
|
return total
|