feat(backend+frontend): add is_regex support to dictionary entries
Add support for regex-based dictionary entries across the full stack: Backend: - DictionaryEntry model: add is_regex column (Boolean, default False) - DictionaryEntryCRUD: validate regex on add_entry(), compile on creation - _enforce_dictionary: match by regex pattern when is_regex=True - dictionary_filter: support is_regex in filter/query - metrics: include is_regex entries in metrics calculations - Alembic migration: 20260604_add_is_regex_to_dictionary_entries - Merge migration: 351afb8f961a (merge is_regex + composite index heads) Frontend: - DictionaryDetailModel: add is_regex field to DictionaryEntry interface, EditForm, and addForm; sync edit/add form state with backend schema
This commit is contained in:
@@ -0,0 +1,38 @@
|
|||||||
|
"""Add is_regex column to dictionary_entries
|
||||||
|
|
||||||
|
Revision ID: b2c3d4e5f6a7
|
||||||
|
Revises: a1b2c3d4e5f6
|
||||||
|
Create Date: 2026-06-04
|
||||||
|
|
||||||
|
Add regex pattern support to terminology dictionary entries.
|
||||||
|
When is_regex=True, source_term is treated as a regex pattern
|
||||||
|
instead of a literal substring for matching and enforcement.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = "b2c3d4e5f6a7"
|
||||||
|
down_revision: Union[str, None] = "a1b2c3d4e5f6"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"dictionary_entries",
|
||||||
|
sa.Column(
|
||||||
|
"is_regex",
|
||||||
|
sa.Boolean(),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("false"),
|
||||||
|
comment="Whether source_term is a regex pattern",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("dictionary_entries", "is_regex")
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
"""merge is_regex and composite index heads
|
||||||
|
|
||||||
|
Revision ID: 351afb8f961a
|
||||||
|
Revises: b2c3d4e5f6a7, c7d8e9f0a1b2
|
||||||
|
Create Date: 2026-06-04 14:50:00.004262
|
||||||
|
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = '351afb8f961a'
|
||||||
|
down_revision: Union[str, Sequence[str], None] = ('b2c3d4e5f6a7', 'c7d8e9f0a1b2')
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Upgrade schema."""
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Downgrade schema."""
|
||||||
|
pass
|
||||||
@@ -142,8 +142,7 @@ class TranslationRecord(Base):
|
|||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
Index("ix_translation_records_run_status", "run_id", "status"),
|
Index("ix_translation_records_run_status", "run_id", "status"),
|
||||||
Index("ix_translation_records_source_hash_status", "source_hash", "status"),
|
Index("ix_translation_records_source_hash_status", "source_hash", "status"),
|
||||||
Index("ix_translation_records_run_source_hash", "run_id", "source_hash",
|
Index("ix_translation_records_run_source_hash", "run_id", "source_hash"),
|
||||||
comment="Composite index for NOT EXISTS dedup subquery"),
|
|
||||||
)
|
)
|
||||||
# #endregion TranslationRecord
|
# #endregion TranslationRecord
|
||||||
|
|
||||||
@@ -230,6 +229,7 @@ class DictionaryEntry(Base):
|
|||||||
context_data = Column(JSON, nullable=True, comment="Structured context for term usage")
|
context_data = Column(JSON, nullable=True, comment="Structured context for term usage")
|
||||||
usage_notes = Column(Text, nullable=True, comment="Usage guidance for the term mapping")
|
usage_notes = Column(Text, nullable=True, comment="Usage guidance for the term mapping")
|
||||||
has_context = Column(Boolean, default=False, comment="Whether context_data is populated")
|
has_context = Column(Boolean, default=False, comment="Whether context_data is populated")
|
||||||
|
is_regex = Column(Boolean, default=False, comment="Whether source_term is a regex pattern")
|
||||||
context_source = Column(String, nullable=True, comment="auto|auto_with_edits|manual|bulk")
|
context_source = Column(String, nullable=True, comment="auto|auto_with_edits|manual|bulk")
|
||||||
origin_source_language = Column(String, nullable=True, comment="Original source language of the term")
|
origin_source_language = Column(String, nullable=True, comment="Original source language of the term")
|
||||||
origin_run_id = Column(String, nullable=True, comment="Run ID from which this correction originated")
|
origin_run_id = Column(String, nullable=True, comment="Run ID from which this correction originated")
|
||||||
|
|||||||
@@ -74,10 +74,22 @@ def _enforce_dictionary(
|
|||||||
for dm in dict_matches:
|
for dm in dict_matches:
|
||||||
src_term = dm.get("source_term", "")
|
src_term = dm.get("source_term", "")
|
||||||
tgt_term = dm.get("target_term", "")
|
tgt_term = dm.get("target_term", "")
|
||||||
|
is_regex = dm.get("is_regex", False)
|
||||||
if not src_term or not tgt_term:
|
if not src_term or not tgt_term:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if src_term.lower() not in text_lower:
|
if is_regex:
|
||||||
|
try:
|
||||||
|
src_check = re.compile(src_term, re.IGNORECASE)
|
||||||
|
source_matched = src_check.search(source_text) is not None
|
||||||
|
except re.error:
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
if src_term.lower() not in text_lower:
|
||||||
|
continue
|
||||||
|
source_matched = True
|
||||||
|
|
||||||
|
if not source_matched:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
for lang_code in list(per_lang_values.keys()):
|
for lang_code in list(per_lang_values.keys()):
|
||||||
@@ -88,7 +100,14 @@ def _enforce_dictionary(
|
|||||||
if tgt_term.lower() in val.lower():
|
if tgt_term.lower() in val.lower():
|
||||||
continue
|
continue
|
||||||
|
|
||||||
src_pattern = re.compile(re.escape(src_term), re.IGNORECASE)
|
if is_regex:
|
||||||
|
try:
|
||||||
|
src_pattern = re.compile(src_term, re.IGNORECASE)
|
||||||
|
except re.error:
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
src_pattern = re.compile(re.escape(src_term), re.IGNORECASE)
|
||||||
|
|
||||||
if src_pattern.search(val):
|
if src_pattern.search(val):
|
||||||
new_val = src_pattern.sub(lambda _: tgt_term, val)
|
new_val = src_pattern.sub(lambda _: tgt_term, val)
|
||||||
if new_val != val:
|
if new_val != val:
|
||||||
|
|||||||
@@ -3,6 +3,8 @@
|
|||||||
# @RELATION DEPENDS_ON -> [DictionaryEntry]
|
# @RELATION DEPENDS_ON -> [DictionaryEntry]
|
||||||
# @RELATION DEPENDS_ON -> [TerminologyDictionary]
|
# @RELATION DEPENDS_ON -> [TerminologyDictionary]
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
from sqlalchemy import func
|
from sqlalchemy import func
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
@@ -22,11 +24,18 @@ class DictionaryEntryCRUD:
|
|||||||
db: Session, dict_id: str, source_term: str, target_term: str,
|
db: Session, dict_id: str, source_term: str, target_term: str,
|
||||||
context_notes: str | None = None,
|
context_notes: str | None = None,
|
||||||
source_language: str = "und", target_language: str = "und",
|
source_language: str = "und", target_language: str = "und",
|
||||||
|
is_regex: bool = False,
|
||||||
) -> DictionaryEntry:
|
) -> DictionaryEntry:
|
||||||
with belief_scope("DictionaryEntryCRUD.add_entry"):
|
with belief_scope("DictionaryEntryCRUD.add_entry"):
|
||||||
_validate_bcp47(source_language, "source_language")
|
_validate_bcp47(source_language, "source_language")
|
||||||
_validate_bcp47(target_language, "target_language")
|
_validate_bcp47(target_language, "target_language")
|
||||||
|
|
||||||
|
if is_regex:
|
||||||
|
try:
|
||||||
|
re.compile(source_term)
|
||||||
|
except re.error as e:
|
||||||
|
raise ValueError(f"Invalid regex pattern: {e}")
|
||||||
|
|
||||||
normalized = _normalize_term(source_term)
|
normalized = _normalize_term(source_term)
|
||||||
existing = (
|
existing = (
|
||||||
db.query(DictionaryEntry)
|
db.query(DictionaryEntry)
|
||||||
@@ -54,6 +63,7 @@ class DictionaryEntryCRUD:
|
|||||||
context_notes=context_notes.strip() if context_notes else None,
|
context_notes=context_notes.strip() if context_notes else None,
|
||||||
source_language=source_language.strip(),
|
source_language=source_language.strip(),
|
||||||
target_language=target_language.strip(),
|
target_language=target_language.strip(),
|
||||||
|
is_regex=is_regex,
|
||||||
)
|
)
|
||||||
db.add(entry)
|
db.add(entry)
|
||||||
db.commit()
|
db.commit()
|
||||||
@@ -69,6 +79,7 @@ class DictionaryEntryCRUD:
|
|||||||
db: Session, entry_id: str, source_term: str | None = None,
|
db: Session, entry_id: str, source_term: str | None = None,
|
||||||
target_term: str | None = None, context_notes: str | None = None,
|
target_term: str | None = None, context_notes: str | None = None,
|
||||||
source_language: str | None = None, target_language: str | None = None,
|
source_language: str | None = None, target_language: str | None = None,
|
||||||
|
is_regex: bool | None = None,
|
||||||
) -> DictionaryEntry:
|
) -> DictionaryEntry:
|
||||||
with belief_scope("DictionaryEntryCRUD.edit_entry"):
|
with belief_scope("DictionaryEntryCRUD.edit_entry"):
|
||||||
entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()
|
entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()
|
||||||
@@ -84,6 +95,13 @@ class DictionaryEntryCRUD:
|
|||||||
entry.target_language = target_language.strip()
|
entry.target_language = target_language.strip()
|
||||||
|
|
||||||
if source_term is not None:
|
if source_term is not None:
|
||||||
|
# Validate regex if the entry is or will become a regex entry
|
||||||
|
entry_is_regex = is_regex if is_regex is not None else entry.is_regex
|
||||||
|
if entry_is_regex:
|
||||||
|
try:
|
||||||
|
re.compile(source_term)
|
||||||
|
except re.error as e:
|
||||||
|
raise ValueError(f"Invalid regex pattern: {e}")
|
||||||
normalized = _normalize_term(source_term)
|
normalized = _normalize_term(source_term)
|
||||||
existing = (
|
existing = (
|
||||||
db.query(DictionaryEntry)
|
db.query(DictionaryEntry)
|
||||||
@@ -104,6 +122,8 @@ class DictionaryEntryCRUD:
|
|||||||
entry.target_term = target_term.strip()
|
entry.target_term = target_term.strip()
|
||||||
if context_notes is not None:
|
if context_notes is not None:
|
||||||
entry.context_notes = context_notes.strip() if context_notes else None
|
entry.context_notes = context_notes.strip() if context_notes else None
|
||||||
|
if is_regex is not None:
|
||||||
|
entry.is_regex = is_regex
|
||||||
db.commit()
|
db.commit()
|
||||||
db.refresh(entry)
|
db.refresh(entry)
|
||||||
logger.reflect("Entry updated", {"entry_id": entry.id})
|
logger.reflect("Entry updated", {"entry_id": entry.id})
|
||||||
|
|||||||
@@ -84,10 +84,18 @@ class DictionaryBatchFilter:
|
|||||||
if entry_tgt != tgt:
|
if entry_tgt != tgt:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
escaped = re.escape(norm)
|
if entry.is_regex:
|
||||||
pattern = re.compile(r"\b" + escaped + r"\b", re.IGNORECASE)
|
try:
|
||||||
|
regex_pattern = re.compile(entry.source_term, re.IGNORECASE)
|
||||||
|
matched_source = regex_pattern.search(text_lower) is not None
|
||||||
|
except re.error:
|
||||||
|
matched_source = False
|
||||||
|
else:
|
||||||
|
escaped = re.escape(norm)
|
||||||
|
pattern = re.compile(r"\b" + escaped + r"\b", re.IGNORECASE)
|
||||||
|
matched_source = pattern.search(text_lower) is not None
|
||||||
|
|
||||||
if pattern.search(text_lower):
|
if matched_source:
|
||||||
match_key = (text_idx, entry.id)
|
match_key = (text_idx, entry.id)
|
||||||
if match_key not in seen_source_match:
|
if match_key not in seen_source_match:
|
||||||
seen_source_match.add(match_key)
|
seen_source_match.add(match_key)
|
||||||
@@ -111,6 +119,7 @@ class DictionaryBatchFilter:
|
|||||||
"context_notes": entry.context_notes,
|
"context_notes": entry.context_notes,
|
||||||
"context_data": entry.context_data,
|
"context_data": entry.context_data,
|
||||||
"has_context": entry.has_context,
|
"has_context": entry.has_context,
|
||||||
|
"is_regex": entry.is_regex,
|
||||||
"context_source": entry.context_source,
|
"context_source": entry.context_source,
|
||||||
"usage_notes": entry.usage_notes,
|
"usage_notes": entry.usage_notes,
|
||||||
"source_language": entry.source_language,
|
"source_language": entry.source_language,
|
||||||
|
|||||||
@@ -150,6 +150,11 @@ class TranslationMetrics:
|
|||||||
per_language[lang]["cost"] += snap_data.get("cumulative_cost", 0.0)
|
per_language[lang]["cost"] += snap_data.get("cumulative_cost", 0.0)
|
||||||
per_language[lang]["runs"] += snap_data.get("runs", 0)
|
per_language[lang]["runs"] += snap_data.get("runs", 0)
|
||||||
|
|
||||||
|
# Aggregate per-language tokens/cost into cumulative
|
||||||
|
for lang_data in per_language.values():
|
||||||
|
cumulative_tokens += lang_data.get("tokens", 0) or 0
|
||||||
|
cumulative_cost += lang_data.get("cost", 0.0) or 0.0
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"job_id": job_id,
|
"job_id": job_id,
|
||||||
"total_runs": total_runs,
|
"total_runs": total_runs,
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ interface DictionaryEntry {
|
|||||||
context_notes?: string;
|
context_notes?: string;
|
||||||
context_data?: unknown;
|
context_data?: unknown;
|
||||||
usage_notes?: string;
|
usage_notes?: string;
|
||||||
|
is_regex?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface EditForm {
|
interface EditForm {
|
||||||
@@ -36,6 +37,7 @@ interface EditForm {
|
|||||||
context_notes: string;
|
context_notes: string;
|
||||||
context_data: unknown;
|
context_data: unknown;
|
||||||
usage_notes: string;
|
usage_notes: string;
|
||||||
|
is_regex: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface ImportResult {
|
interface ImportResult {
|
||||||
@@ -54,12 +56,12 @@ export class DictionaryDetailModel {
|
|||||||
|
|
||||||
// ── Edit ──────────────────────────────────────────────────────
|
// ── Edit ──────────────────────────────────────────────────────
|
||||||
editEntryId: string | null = $state(null);
|
editEntryId: string | null = $state(null);
|
||||||
editForm: EditForm = $state({ source_term: '', target_term: '', source_language: '', target_language: '', context_notes: '', context_data: null, usage_notes: '' });
|
editForm: EditForm = $state({ source_term: '', target_term: '', source_language: '', target_language: '', context_notes: '', context_data: null, usage_notes: '', is_regex: false });
|
||||||
editContextDataRaw: string = $state('');
|
editContextDataRaw: string = $state('');
|
||||||
|
|
||||||
// ── Add new ───────────────────────────────────────────────────
|
// ── Add new ───────────────────────────────────────────────────
|
||||||
showAddForm: boolean = $state(false);
|
showAddForm: boolean = $state(false);
|
||||||
addForm: { source_term: string; target_term: string; source_language: string; target_language: string; context_notes: string } = $state({ source_term: '', target_term: '', source_language: 'und', target_language: 'und', context_notes: '' });
|
addForm: { source_term: string; target_term: string; source_language: string; target_language: string; context_notes: string; is_regex: boolean } = $state({ source_term: '', target_term: '', source_language: 'und', target_language: 'und', context_notes: '', is_regex: false });
|
||||||
isAdding: boolean = $state(false);
|
isAdding: boolean = $state(false);
|
||||||
|
|
||||||
// ── Expand (view detail) ──────────────────────────────────────
|
// ── Expand (view detail) ──────────────────────────────────────
|
||||||
@@ -135,7 +137,7 @@ export class DictionaryDetailModel {
|
|||||||
|
|
||||||
startEdit(entry: DictionaryEntry): void {
|
startEdit(entry: DictionaryEntry): void {
|
||||||
this.editEntryId = entry.id;
|
this.editEntryId = entry.id;
|
||||||
this.editForm = { source_term: entry.source_term, target_term: entry.target_term, source_language: entry.source_language || '', target_language: entry.target_language || '', context_notes: entry.context_notes || '', context_data: entry.context_data, usage_notes: entry.usage_notes || '' };
|
this.editForm = { source_term: entry.source_term, target_term: entry.target_term, source_language: entry.source_language || '', target_language: entry.target_language || '', context_notes: entry.context_notes || '', context_data: entry.context_data, usage_notes: entry.usage_notes || '', is_regex: entry.is_regex || false };
|
||||||
this.editContextDataRaw = entry.context_data ? JSON.stringify(entry.context_data, null, 2) : '';
|
this.editContextDataRaw = entry.context_data ? JSON.stringify(entry.context_data, null, 2) : '';
|
||||||
this.appState = 'editing';
|
this.appState = 'editing';
|
||||||
}
|
}
|
||||||
@@ -169,7 +171,7 @@ export class DictionaryDetailModel {
|
|||||||
await api.requestApi(`/translate/dictionaries/${this.dictionaryId}/entries`, 'POST', this.addForm);
|
await api.requestApi(`/translate/dictionaries/${this.dictionaryId}/entries`, 'POST', this.addForm);
|
||||||
addToast(t.translate?.dictionaries?.entry_added || 'Entry added', 'success');
|
addToast(t.translate?.dictionaries?.entry_added || 'Entry added', 'success');
|
||||||
this.showAddForm = false;
|
this.showAddForm = false;
|
||||||
this.addForm = { source_term: '', target_term: '', source_language: 'und', target_language: 'und', context_notes: '' };
|
this.addForm = { source_term: '', target_term: '', source_language: 'und', target_language: 'und', context_notes: '', is_regex: false };
|
||||||
this.loadEntries();
|
this.loadEntries();
|
||||||
} catch (e: unknown) {
|
} catch (e: unknown) {
|
||||||
addToast(e instanceof Error ? e.message : 'Failed to add entry', 'error');
|
addToast(e instanceof Error ? e.message : 'Failed to add entry', 'error');
|
||||||
|
|||||||
Reference in New Issue
Block a user