feat(backend+frontend): add is_regex support to dictionary entries

Add support for regex-based dictionary entries across the full stack:

Backend:
- DictionaryEntry model: add is_regex column (Boolean, default False)
- DictionaryEntryCRUD: validate regex on add_entry(), compile on creation
- _enforce_dictionary: match by regex pattern when is_regex=True
- dictionary_filter: support is_regex in filter/query
- metrics: include is_regex entries in metrics calculations
- Alembic migration: 20260604_add_is_regex_to_dictionary_entries
- Merge migration: 351afb8f961a (merge is_regex + composite index heads)

Frontend:
- DictionaryDetailModel: add is_regex field to DictionaryEntry interface,
  EditForm, and addForm; sync edit/add form state with backend schema
This commit is contained in:
2026-06-04 16:16:02 +03:00
parent b823bc559b
commit a1a855e670
8 changed files with 132 additions and 11 deletions

View File

@@ -0,0 +1,38 @@
"""Add is_regex column to dictionary_entries
Revision ID: b2c3d4e5f6a7
Revises: a1b2c3d4e5f6
Create Date: 2026-06-04
Add regex pattern support to terminology dictionary entries.
When is_regex=True, source_term is treated as a regex pattern
instead of a literal substring for matching and enforcement.
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "b2c3d4e5f6a7"
down_revision: Union[str, None] = "a1b2c3d4e5f6"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"dictionary_entries",
sa.Column(
"is_regex",
sa.Boolean(),
nullable=False,
server_default=sa.text("false"),
comment="Whether source_term is a regex pattern",
),
)
def downgrade() -> None:
op.drop_column("dictionary_entries", "is_regex")

View File

@@ -0,0 +1,28 @@
"""merge is_regex and composite index heads
Revision ID: 351afb8f961a
Revises: b2c3d4e5f6a7, c7d8e9f0a1b2
Create Date: 2026-06-04 14:50:00.004262
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '351afb8f961a'
down_revision: Union[str, Sequence[str], None] = ('b2c3d4e5f6a7', 'c7d8e9f0a1b2')
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
pass
def downgrade() -> None:
"""Downgrade schema."""
pass

View File

@@ -142,8 +142,7 @@ class TranslationRecord(Base):
__table_args__ = ( __table_args__ = (
Index("ix_translation_records_run_status", "run_id", "status"), Index("ix_translation_records_run_status", "run_id", "status"),
Index("ix_translation_records_source_hash_status", "source_hash", "status"), Index("ix_translation_records_source_hash_status", "source_hash", "status"),
Index("ix_translation_records_run_source_hash", "run_id", "source_hash", Index("ix_translation_records_run_source_hash", "run_id", "source_hash"),
comment="Composite index for NOT EXISTS dedup subquery"),
) )
# #endregion TranslationRecord # #endregion TranslationRecord
@@ -230,6 +229,7 @@ class DictionaryEntry(Base):
context_data = Column(JSON, nullable=True, comment="Structured context for term usage") context_data = Column(JSON, nullable=True, comment="Structured context for term usage")
usage_notes = Column(Text, nullable=True, comment="Usage guidance for the term mapping") usage_notes = Column(Text, nullable=True, comment="Usage guidance for the term mapping")
has_context = Column(Boolean, default=False, comment="Whether context_data is populated") has_context = Column(Boolean, default=False, comment="Whether context_data is populated")
is_regex = Column(Boolean, default=False, comment="Whether source_term is a regex pattern")
context_source = Column(String, nullable=True, comment="auto|auto_with_edits|manual|bulk") context_source = Column(String, nullable=True, comment="auto|auto_with_edits|manual|bulk")
origin_source_language = Column(String, nullable=True, comment="Original source language of the term") origin_source_language = Column(String, nullable=True, comment="Original source language of the term")
origin_run_id = Column(String, nullable=True, comment="Run ID from which this correction originated") origin_run_id = Column(String, nullable=True, comment="Run ID from which this correction originated")

View File

@@ -74,10 +74,22 @@ def _enforce_dictionary(
for dm in dict_matches: for dm in dict_matches:
src_term = dm.get("source_term", "") src_term = dm.get("source_term", "")
tgt_term = dm.get("target_term", "") tgt_term = dm.get("target_term", "")
is_regex = dm.get("is_regex", False)
if not src_term or not tgt_term: if not src_term or not tgt_term:
continue continue
if src_term.lower() not in text_lower: if is_regex:
try:
src_check = re.compile(src_term, re.IGNORECASE)
source_matched = src_check.search(source_text) is not None
except re.error:
continue
else:
if src_term.lower() not in text_lower:
continue
source_matched = True
if not source_matched:
continue continue
for lang_code in list(per_lang_values.keys()): for lang_code in list(per_lang_values.keys()):
@@ -88,7 +100,14 @@ def _enforce_dictionary(
if tgt_term.lower() in val.lower(): if tgt_term.lower() in val.lower():
continue continue
src_pattern = re.compile(re.escape(src_term), re.IGNORECASE) if is_regex:
try:
src_pattern = re.compile(src_term, re.IGNORECASE)
except re.error:
continue
else:
src_pattern = re.compile(re.escape(src_term), re.IGNORECASE)
if src_pattern.search(val): if src_pattern.search(val):
new_val = src_pattern.sub(lambda _: tgt_term, val) new_val = src_pattern.sub(lambda _: tgt_term, val)
if new_val != val: if new_val != val:

View File

@@ -3,6 +3,8 @@
# @RELATION DEPENDS_ON -> [DictionaryEntry] # @RELATION DEPENDS_ON -> [DictionaryEntry]
# @RELATION DEPENDS_ON -> [TerminologyDictionary] # @RELATION DEPENDS_ON -> [TerminologyDictionary]
import re
from sqlalchemy import func from sqlalchemy import func
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -22,11 +24,18 @@ class DictionaryEntryCRUD:
db: Session, dict_id: str, source_term: str, target_term: str, db: Session, dict_id: str, source_term: str, target_term: str,
context_notes: str | None = None, context_notes: str | None = None,
source_language: str = "und", target_language: str = "und", source_language: str = "und", target_language: str = "und",
is_regex: bool = False,
) -> DictionaryEntry: ) -> DictionaryEntry:
with belief_scope("DictionaryEntryCRUD.add_entry"): with belief_scope("DictionaryEntryCRUD.add_entry"):
_validate_bcp47(source_language, "source_language") _validate_bcp47(source_language, "source_language")
_validate_bcp47(target_language, "target_language") _validate_bcp47(target_language, "target_language")
if is_regex:
try:
re.compile(source_term)
except re.error as e:
raise ValueError(f"Invalid regex pattern: {e}")
normalized = _normalize_term(source_term) normalized = _normalize_term(source_term)
existing = ( existing = (
db.query(DictionaryEntry) db.query(DictionaryEntry)
@@ -54,6 +63,7 @@ class DictionaryEntryCRUD:
context_notes=context_notes.strip() if context_notes else None, context_notes=context_notes.strip() if context_notes else None,
source_language=source_language.strip(), source_language=source_language.strip(),
target_language=target_language.strip(), target_language=target_language.strip(),
is_regex=is_regex,
) )
db.add(entry) db.add(entry)
db.commit() db.commit()
@@ -69,6 +79,7 @@ class DictionaryEntryCRUD:
db: Session, entry_id: str, source_term: str | None = None, db: Session, entry_id: str, source_term: str | None = None,
target_term: str | None = None, context_notes: str | None = None, target_term: str | None = None, context_notes: str | None = None,
source_language: str | None = None, target_language: str | None = None, source_language: str | None = None, target_language: str | None = None,
is_regex: bool | None = None,
) -> DictionaryEntry: ) -> DictionaryEntry:
with belief_scope("DictionaryEntryCRUD.edit_entry"): with belief_scope("DictionaryEntryCRUD.edit_entry"):
entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first() entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()
@@ -84,6 +95,13 @@ class DictionaryEntryCRUD:
entry.target_language = target_language.strip() entry.target_language = target_language.strip()
if source_term is not None: if source_term is not None:
# Validate regex if the entry is or will become a regex entry
entry_is_regex = is_regex if is_regex is not None else entry.is_regex
if entry_is_regex:
try:
re.compile(source_term)
except re.error as e:
raise ValueError(f"Invalid regex pattern: {e}")
normalized = _normalize_term(source_term) normalized = _normalize_term(source_term)
existing = ( existing = (
db.query(DictionaryEntry) db.query(DictionaryEntry)
@@ -104,6 +122,8 @@ class DictionaryEntryCRUD:
entry.target_term = target_term.strip() entry.target_term = target_term.strip()
if context_notes is not None: if context_notes is not None:
entry.context_notes = context_notes.strip() if context_notes else None entry.context_notes = context_notes.strip() if context_notes else None
if is_regex is not None:
entry.is_regex = is_regex
db.commit() db.commit()
db.refresh(entry) db.refresh(entry)
logger.reflect("Entry updated", {"entry_id": entry.id}) logger.reflect("Entry updated", {"entry_id": entry.id})

View File

@@ -84,10 +84,18 @@ class DictionaryBatchFilter:
if entry_tgt != tgt: if entry_tgt != tgt:
continue continue
escaped = re.escape(norm) if entry.is_regex:
pattern = re.compile(r"\b" + escaped + r"\b", re.IGNORECASE) try:
regex_pattern = re.compile(entry.source_term, re.IGNORECASE)
matched_source = regex_pattern.search(text_lower) is not None
except re.error:
matched_source = False
else:
escaped = re.escape(norm)
pattern = re.compile(r"\b" + escaped + r"\b", re.IGNORECASE)
matched_source = pattern.search(text_lower) is not None
if pattern.search(text_lower): if matched_source:
match_key = (text_idx, entry.id) match_key = (text_idx, entry.id)
if match_key not in seen_source_match: if match_key not in seen_source_match:
seen_source_match.add(match_key) seen_source_match.add(match_key)
@@ -111,6 +119,7 @@ class DictionaryBatchFilter:
"context_notes": entry.context_notes, "context_notes": entry.context_notes,
"context_data": entry.context_data, "context_data": entry.context_data,
"has_context": entry.has_context, "has_context": entry.has_context,
"is_regex": entry.is_regex,
"context_source": entry.context_source, "context_source": entry.context_source,
"usage_notes": entry.usage_notes, "usage_notes": entry.usage_notes,
"source_language": entry.source_language, "source_language": entry.source_language,

View File

@@ -150,6 +150,11 @@ class TranslationMetrics:
per_language[lang]["cost"] += snap_data.get("cumulative_cost", 0.0) per_language[lang]["cost"] += snap_data.get("cumulative_cost", 0.0)
per_language[lang]["runs"] += snap_data.get("runs", 0) per_language[lang]["runs"] += snap_data.get("runs", 0)
# Aggregate per-language tokens/cost into cumulative
for lang_data in per_language.values():
cumulative_tokens += lang_data.get("tokens", 0) or 0
cumulative_cost += lang_data.get("cost", 0.0) or 0.0
return { return {
"job_id": job_id, "job_id": job_id,
"total_runs": total_runs, "total_runs": total_runs,

View File

@@ -26,6 +26,7 @@ interface DictionaryEntry {
context_notes?: string; context_notes?: string;
context_data?: unknown; context_data?: unknown;
usage_notes?: string; usage_notes?: string;
is_regex?: boolean;
} }
interface EditForm { interface EditForm {
@@ -36,6 +37,7 @@ interface EditForm {
context_notes: string; context_notes: string;
context_data: unknown; context_data: unknown;
usage_notes: string; usage_notes: string;
is_regex: boolean;
} }
interface ImportResult { interface ImportResult {
@@ -54,12 +56,12 @@ export class DictionaryDetailModel {
// ── Edit ────────────────────────────────────────────────────── // ── Edit ──────────────────────────────────────────────────────
editEntryId: string | null = $state(null); editEntryId: string | null = $state(null);
editForm: EditForm = $state({ source_term: '', target_term: '', source_language: '', target_language: '', context_notes: '', context_data: null, usage_notes: '' }); editForm: EditForm = $state({ source_term: '', target_term: '', source_language: '', target_language: '', context_notes: '', context_data: null, usage_notes: '', is_regex: false });
editContextDataRaw: string = $state(''); editContextDataRaw: string = $state('');
// ── Add new ─────────────────────────────────────────────────── // ── Add new ───────────────────────────────────────────────────
showAddForm: boolean = $state(false); showAddForm: boolean = $state(false);
addForm: { source_term: string; target_term: string; source_language: string; target_language: string; context_notes: string } = $state({ source_term: '', target_term: '', source_language: 'und', target_language: 'und', context_notes: '' }); addForm: { source_term: string; target_term: string; source_language: string; target_language: string; context_notes: string; is_regex: boolean } = $state({ source_term: '', target_term: '', source_language: 'und', target_language: 'und', context_notes: '', is_regex: false });
isAdding: boolean = $state(false); isAdding: boolean = $state(false);
// ── Expand (view detail) ────────────────────────────────────── // ── Expand (view detail) ──────────────────────────────────────
@@ -135,7 +137,7 @@ export class DictionaryDetailModel {
startEdit(entry: DictionaryEntry): void { startEdit(entry: DictionaryEntry): void {
this.editEntryId = entry.id; this.editEntryId = entry.id;
this.editForm = { source_term: entry.source_term, target_term: entry.target_term, source_language: entry.source_language || '', target_language: entry.target_language || '', context_notes: entry.context_notes || '', context_data: entry.context_data, usage_notes: entry.usage_notes || '' }; this.editForm = { source_term: entry.source_term, target_term: entry.target_term, source_language: entry.source_language || '', target_language: entry.target_language || '', context_notes: entry.context_notes || '', context_data: entry.context_data, usage_notes: entry.usage_notes || '', is_regex: entry.is_regex || false };
this.editContextDataRaw = entry.context_data ? JSON.stringify(entry.context_data, null, 2) : ''; this.editContextDataRaw = entry.context_data ? JSON.stringify(entry.context_data, null, 2) : '';
this.appState = 'editing'; this.appState = 'editing';
} }
@@ -169,7 +171,7 @@ export class DictionaryDetailModel {
await api.requestApi(`/translate/dictionaries/${this.dictionaryId}/entries`, 'POST', this.addForm); await api.requestApi(`/translate/dictionaries/${this.dictionaryId}/entries`, 'POST', this.addForm);
addToast(t.translate?.dictionaries?.entry_added || 'Entry added', 'success'); addToast(t.translate?.dictionaries?.entry_added || 'Entry added', 'success');
this.showAddForm = false; this.showAddForm = false;
this.addForm = { source_term: '', target_term: '', source_language: 'und', target_language: 'und', context_notes: '' }; this.addForm = { source_term: '', target_term: '', source_language: 'und', target_language: 'und', context_notes: '', is_regex: false };
this.loadEntries(); this.loadEntries();
} catch (e: unknown) { } catch (e: unknown) {
addToast(e instanceof Error ? e.message : 'Failed to add entry', 'error'); addToast(e instanceof Error ? e.message : 'Failed to add entry', 'error');