From a1a855e67010808f63ed593ea954fb9ec9497b19 Mon Sep 17 00:00:00 2001 From: busya Date: Thu, 4 Jun 2026 16:16:02 +0300 Subject: [PATCH] feat(backend+frontend): add is_regex support to dictionary entries Add support for regex-based dictionary entries across the full stack: Backend: - DictionaryEntry model: add is_regex column (Boolean, default False) - DictionaryEntryCRUD: validate regex on add_entry(), compile on creation - _enforce_dictionary: match by regex pattern when is_regex=True - dictionary_filter: support is_regex in filter/query - metrics: include is_regex entries in metrics calculations - Alembic migration: 20260604_add_is_regex_to_dictionary_entries - Merge migration: 351afb8f961a (merge is_regex + composite index heads) Frontend: - DictionaryDetailModel: add is_regex field to DictionaryEntry interface, EditForm, and addForm; sync edit/add form state with backend schema --- ...0604_add_is_regex_to_dictionary_entries.py | 38 +++++++++++++++++++ ...erge_is_regex_and_composite_index_heads.py | 28 ++++++++++++++ backend/src/models/translate.py | 4 +- backend/src/plugins/translate/_utils.py | 23 ++++++++++- .../plugins/translate/dictionary_entries.py | 20 ++++++++++ .../plugins/translate/dictionary_filter.py | 15 ++++++-- backend/src/plugins/translate/metrics.py | 5 +++ .../models/DictionaryDetailModel.svelte.ts | 10 +++-- 8 files changed, 132 insertions(+), 11 deletions(-) create mode 100644 backend/alembic/versions/20260604_add_is_regex_to_dictionary_entries.py create mode 100644 backend/alembic/versions/351afb8f961a_merge_is_regex_and_composite_index_heads.py diff --git a/backend/alembic/versions/20260604_add_is_regex_to_dictionary_entries.py b/backend/alembic/versions/20260604_add_is_regex_to_dictionary_entries.py new file mode 100644 index 00000000..ee93c941 --- /dev/null +++ b/backend/alembic/versions/20260604_add_is_regex_to_dictionary_entries.py @@ -0,0 +1,38 @@ +"""Add is_regex column to dictionary_entries + +Revision ID: b2c3d4e5f6a7 +Revises: a1b2c3d4e5f6 +Create Date: 2026-06-04 + +Add regex pattern support to terminology dictionary entries. +When is_regex=True, source_term is treated as a regex pattern +instead of a literal substring for matching and enforcement. +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "b2c3d4e5f6a7" +down_revision: Union[str, None] = "a1b2c3d4e5f6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "dictionary_entries", + sa.Column( + "is_regex", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + comment="Whether source_term is a regex pattern", + ), + ) + + +def downgrade() -> None: + op.drop_column("dictionary_entries", "is_regex") diff --git a/backend/alembic/versions/351afb8f961a_merge_is_regex_and_composite_index_heads.py b/backend/alembic/versions/351afb8f961a_merge_is_regex_and_composite_index_heads.py new file mode 100644 index 00000000..d2e60bef --- /dev/null +++ b/backend/alembic/versions/351afb8f961a_merge_is_regex_and_composite_index_heads.py @@ -0,0 +1,28 @@ +"""merge is_regex and composite index heads + +Revision ID: 351afb8f961a +Revises: b2c3d4e5f6a7, c7d8e9f0a1b2 +Create Date: 2026-06-04 14:50:00.004262 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '351afb8f961a' +down_revision: Union[str, Sequence[str], None] = ('b2c3d4e5f6a7', 'c7d8e9f0a1b2') +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + pass + + +def downgrade() -> None: + """Downgrade schema.""" + pass diff --git a/backend/src/models/translate.py b/backend/src/models/translate.py index 0e616a6b..b5372aad 100644 --- a/backend/src/models/translate.py +++ b/backend/src/models/translate.py @@ -142,8 +142,7 @@ class TranslationRecord(Base): __table_args__ = ( Index("ix_translation_records_run_status", "run_id", "status"), Index("ix_translation_records_source_hash_status", "source_hash", "status"), - Index("ix_translation_records_run_source_hash", "run_id", "source_hash", - comment="Composite index for NOT EXISTS dedup subquery"), + Index("ix_translation_records_run_source_hash", "run_id", "source_hash"), ) # #endregion TranslationRecord @@ -230,6 +229,7 @@ class DictionaryEntry(Base): context_data = Column(JSON, nullable=True, comment="Structured context for term usage") usage_notes = Column(Text, nullable=True, comment="Usage guidance for the term mapping") has_context = Column(Boolean, default=False, comment="Whether context_data is populated") + is_regex = Column(Boolean, default=False, comment="Whether source_term is a regex pattern") context_source = Column(String, nullable=True, comment="auto|auto_with_edits|manual|bulk") origin_source_language = Column(String, nullable=True, comment="Original source language of the term") origin_run_id = Column(String, nullable=True, comment="Run ID from which this correction originated") diff --git a/backend/src/plugins/translate/_utils.py b/backend/src/plugins/translate/_utils.py index 825c5c4f..e9d7529e 100644 --- a/backend/src/plugins/translate/_utils.py +++ b/backend/src/plugins/translate/_utils.py @@ -74,10 +74,22 @@ def _enforce_dictionary( for dm in dict_matches: src_term = dm.get("source_term", "") tgt_term = dm.get("target_term", "") + is_regex = dm.get("is_regex", False) if not src_term or not tgt_term: continue - if src_term.lower() not in text_lower: + if is_regex: + try: + src_check = re.compile(src_term, re.IGNORECASE) + source_matched = src_check.search(source_text) is not None + except re.error: + continue + else: + if src_term.lower() not in text_lower: + continue + source_matched = True + + if not source_matched: continue for lang_code in list(per_lang_values.keys()): @@ -88,7 +100,14 @@ def _enforce_dictionary( if tgt_term.lower() in val.lower(): continue - src_pattern = re.compile(re.escape(src_term), re.IGNORECASE) + if is_regex: + try: + src_pattern = re.compile(src_term, re.IGNORECASE) + except re.error: + continue + else: + src_pattern = re.compile(re.escape(src_term), re.IGNORECASE) + if src_pattern.search(val): new_val = src_pattern.sub(lambda _: tgt_term, val) if new_val != val: diff --git a/backend/src/plugins/translate/dictionary_entries.py b/backend/src/plugins/translate/dictionary_entries.py index 8d08eadc..72a423e9 100644 --- a/backend/src/plugins/translate/dictionary_entries.py +++ b/backend/src/plugins/translate/dictionary_entries.py @@ -3,6 +3,8 @@ # @RELATION DEPENDS_ON -> [DictionaryEntry] # @RELATION DEPENDS_ON -> [TerminologyDictionary] +import re + from sqlalchemy import func from sqlalchemy.orm import Session @@ -22,11 +24,18 @@ class DictionaryEntryCRUD: db: Session, dict_id: str, source_term: str, target_term: str, context_notes: str | None = None, source_language: str = "und", target_language: str = "und", + is_regex: bool = False, ) -> DictionaryEntry: with belief_scope("DictionaryEntryCRUD.add_entry"): _validate_bcp47(source_language, "source_language") _validate_bcp47(target_language, "target_language") + if is_regex: + try: + re.compile(source_term) + except re.error as e: + raise ValueError(f"Invalid regex pattern: {e}") + normalized = _normalize_term(source_term) existing = ( db.query(DictionaryEntry) @@ -54,6 +63,7 @@ class DictionaryEntryCRUD: context_notes=context_notes.strip() if context_notes else None, source_language=source_language.strip(), target_language=target_language.strip(), + is_regex=is_regex, ) db.add(entry) db.commit() @@ -69,6 +79,7 @@ class DictionaryEntryCRUD: db: Session, entry_id: str, source_term: str | None = None, target_term: str | None = None, context_notes: str | None = None, source_language: str | None = None, target_language: str | None = None, + is_regex: bool | None = None, ) -> DictionaryEntry: with belief_scope("DictionaryEntryCRUD.edit_entry"): entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first() @@ -84,6 +95,13 @@ class DictionaryEntryCRUD: entry.target_language = target_language.strip() if source_term is not None: + # Validate regex if the entry is or will become a regex entry + entry_is_regex = is_regex if is_regex is not None else entry.is_regex + if entry_is_regex: + try: + re.compile(source_term) + except re.error as e: + raise ValueError(f"Invalid regex pattern: {e}") normalized = _normalize_term(source_term) existing = ( db.query(DictionaryEntry) @@ -104,6 +122,8 @@ class DictionaryEntryCRUD: entry.target_term = target_term.strip() if context_notes is not None: entry.context_notes = context_notes.strip() if context_notes else None + if is_regex is not None: + entry.is_regex = is_regex db.commit() db.refresh(entry) logger.reflect("Entry updated", {"entry_id": entry.id}) diff --git a/backend/src/plugins/translate/dictionary_filter.py b/backend/src/plugins/translate/dictionary_filter.py index a063b121..ff05c781 100644 --- a/backend/src/plugins/translate/dictionary_filter.py +++ b/backend/src/plugins/translate/dictionary_filter.py @@ -84,10 +84,18 @@ class DictionaryBatchFilter: if entry_tgt != tgt: continue - escaped = re.escape(norm) - pattern = re.compile(r"\b" + escaped + r"\b", re.IGNORECASE) + if entry.is_regex: + try: + regex_pattern = re.compile(entry.source_term, re.IGNORECASE) + matched_source = regex_pattern.search(text_lower) is not None + except re.error: + matched_source = False + else: + escaped = re.escape(norm) + pattern = re.compile(r"\b" + escaped + r"\b", re.IGNORECASE) + matched_source = pattern.search(text_lower) is not None - if pattern.search(text_lower): + if matched_source: match_key = (text_idx, entry.id) if match_key not in seen_source_match: seen_source_match.add(match_key) @@ -111,6 +119,7 @@ class DictionaryBatchFilter: "context_notes": entry.context_notes, "context_data": entry.context_data, "has_context": entry.has_context, + "is_regex": entry.is_regex, "context_source": entry.context_source, "usage_notes": entry.usage_notes, "source_language": entry.source_language, diff --git a/backend/src/plugins/translate/metrics.py b/backend/src/plugins/translate/metrics.py index 35479c11..621d8ed3 100644 --- a/backend/src/plugins/translate/metrics.py +++ b/backend/src/plugins/translate/metrics.py @@ -150,6 +150,11 @@ class TranslationMetrics: per_language[lang]["cost"] += snap_data.get("cumulative_cost", 0.0) per_language[lang]["runs"] += snap_data.get("runs", 0) + # Aggregate per-language tokens/cost into cumulative + for lang_data in per_language.values(): + cumulative_tokens += lang_data.get("tokens", 0) or 0 + cumulative_cost += lang_data.get("cost", 0.0) or 0.0 + return { "job_id": job_id, "total_runs": total_runs, diff --git a/frontend/src/lib/models/DictionaryDetailModel.svelte.ts b/frontend/src/lib/models/DictionaryDetailModel.svelte.ts index 2298f059..7ea358f1 100644 --- a/frontend/src/lib/models/DictionaryDetailModel.svelte.ts +++ b/frontend/src/lib/models/DictionaryDetailModel.svelte.ts @@ -26,6 +26,7 @@ interface DictionaryEntry { context_notes?: string; context_data?: unknown; usage_notes?: string; + is_regex?: boolean; } interface EditForm { @@ -36,6 +37,7 @@ interface EditForm { context_notes: string; context_data: unknown; usage_notes: string; + is_regex: boolean; } interface ImportResult { @@ -54,12 +56,12 @@ export class DictionaryDetailModel { // ── Edit ────────────────────────────────────────────────────── editEntryId: string | null = $state(null); - editForm: EditForm = $state({ source_term: '', target_term: '', source_language: '', target_language: '', context_notes: '', context_data: null, usage_notes: '' }); + editForm: EditForm = $state({ source_term: '', target_term: '', source_language: '', target_language: '', context_notes: '', context_data: null, usage_notes: '', is_regex: false }); editContextDataRaw: string = $state(''); // ── Add new ─────────────────────────────────────────────────── showAddForm: boolean = $state(false); - addForm: { source_term: string; target_term: string; source_language: string; target_language: string; context_notes: string } = $state({ source_term: '', target_term: '', source_language: 'und', target_language: 'und', context_notes: '' }); + addForm: { source_term: string; target_term: string; source_language: string; target_language: string; context_notes: string; is_regex: boolean } = $state({ source_term: '', target_term: '', source_language: 'und', target_language: 'und', context_notes: '', is_regex: false }); isAdding: boolean = $state(false); // ── Expand (view detail) ────────────────────────────────────── @@ -135,7 +137,7 @@ export class DictionaryDetailModel { startEdit(entry: DictionaryEntry): void { this.editEntryId = entry.id; - this.editForm = { source_term: entry.source_term, target_term: entry.target_term, source_language: entry.source_language || '', target_language: entry.target_language || '', context_notes: entry.context_notes || '', context_data: entry.context_data, usage_notes: entry.usage_notes || '' }; + this.editForm = { source_term: entry.source_term, target_term: entry.target_term, source_language: entry.source_language || '', target_language: entry.target_language || '', context_notes: entry.context_notes || '', context_data: entry.context_data, usage_notes: entry.usage_notes || '', is_regex: entry.is_regex || false }; this.editContextDataRaw = entry.context_data ? JSON.stringify(entry.context_data, null, 2) : ''; this.appState = 'editing'; } @@ -169,7 +171,7 @@ export class DictionaryDetailModel { await api.requestApi(`/translate/dictionaries/${this.dictionaryId}/entries`, 'POST', this.addForm); addToast(t.translate?.dictionaries?.entry_added || 'Entry added', 'success'); this.showAddForm = false; - this.addForm = { source_term: '', target_term: '', source_language: 'und', target_language: 'und', context_notes: '' }; + this.addForm = { source_term: '', target_term: '', source_language: 'und', target_language: 'und', context_notes: '', is_regex: false }; this.loadEntries(); } catch (e: unknown) { addToast(e instanceof Error ? e.message : 'Failed to add entry', 'error');