busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(backend+frontend): add is_regex support to dictionary entries

Browse Source 
Add support for regex-based dictionary entries across the full stack:

Backend:
- DictionaryEntry model: add is_regex column (Boolean, default False)
- DictionaryEntryCRUD: validate regex on add_entry(), compile on creation
- _enforce_dictionary: match by regex pattern when is_regex=True
- dictionary_filter: support is_regex in filter/query
- metrics: include is_regex entries in metrics calculations
- Alembic migration: 20260604_add_is_regex_to_dictionary_entries
- Merge migration: 351afb8f961a (merge is_regex + composite index heads)

Frontend:
- DictionaryDetailModel: add is_regex field to DictionaryEntry interface,
  EditForm, and addForm; sync edit/add form state with backend schema
This commit is contained in:
busya
2026-06-04 16:16:02 +03:00
parent b823bc559b
commit a1a855e670
8 changed files with 132 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
backend/alembic/versions/20260604_add_is_regex_to_dictionary_entries.py Normal file
View File

@@ -0,0 +1,38 @@
"""Add is_regex column to dictionary_entries
Revision ID: b2c3d4e5f6a7
Revises: a1b2c3d4e5f6
Create Date: 2026-06-04
Add regex pattern support to terminology dictionary entries.
When is_regex=True, source_term is treated as a regex pattern
instead of a literal substring for matching and enforcement.
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "b2c3d4e5f6a7"
down_revision: Union[str, None] = "a1b2c3d4e5f6"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"dictionary_entries",
sa.Column(
"is_regex",
sa.Boolean(),
nullable=False,
server_default=sa.text("false"),
comment="Whether source_term is a regex pattern",
),
)
def downgrade() -> None:
op.drop_column("dictionary_entries", "is_regex")

28
backend/alembic/versions/351afb8f961a_merge_is_regex_and_composite_index_heads.py Normal file
View File

@@ -0,0 +1,28 @@
"""merge is_regex and composite index heads
Revision ID: 351afb8f961a
Revises: b2c3d4e5f6a7, c7d8e9f0a1b2
Create Date: 2026-06-04 14:50:00.004262
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '351afb8f961a'
down_revision: Union[str, Sequence[str], None] = ('b2c3d4e5f6a7', 'c7d8e9f0a1b2')
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
pass
def downgrade() -> None:
"""Downgrade schema."""
pass

4
backend/src/models/translate.py
View File

@@ -142,8 +142,7 @@ class TranslationRecord(Base):
__table_args__ = (
Index("ix_translation_records_run_status", "run_id", "status"),
Index("ix_translation_records_source_hash_status", "source_hash", "status"),
Index("ix_translation_records_run_source_hash", "run_id", "source_hash",
comment="Composite index for NOT EXISTS dedup subquery"),
Index("ix_translation_records_run_source_hash", "run_id", "source_hash"),
)
# #endregion TranslationRecord
@@ -230,6 +229,7 @@ class DictionaryEntry(Base):
context_data = Column(JSON, nullable=True, comment="Structured context for term usage")
usage_notes = Column(Text, nullable=True, comment="Usage guidance for the term mapping")
has_context = Column(Boolean, default=False, comment="Whether context_data is populated")
is_regex = Column(Boolean, default=False, comment="Whether source_term is a regex pattern")
context_source = Column(String, nullable=True, comment="auto|auto_with_edits|manual|bulk")
origin_source_language = Column(String, nullable=True, comment="Original source language of the term")
origin_run_id = Column(String, nullable=True, comment="Run ID from which this correction originated")

23
backend/src/plugins/translate/_utils.py
View File

@@ -74,10 +74,22 @@ def _enforce_dictionary(
for dm in dict_matches:
src_term = dm.get("source_term", "")
tgt_term = dm.get("target_term", "")
is_regex = dm.get("is_regex", False)
if not src_term or not tgt_term:
continue
if src_term.lower() not in text_lower:
if is_regex:
try:
src_check = re.compile(src_term, re.IGNORECASE)
source_matched = src_check.search(source_text) is not None
except re.error:
continue
else:
if src_term.lower() not in text_lower:
continue
source_matched = True
if not source_matched:
continue
for lang_code in list(per_lang_values.keys()):
@@ -88,7 +100,14 @@ def _enforce_dictionary(
if tgt_term.lower() in val.lower():
continue
src_pattern = re.compile(re.escape(src_term), re.IGNORECASE)
if is_regex:
try:
src_pattern = re.compile(src_term, re.IGNORECASE)
except re.error:
continue
else:
src_pattern = re.compile(re.escape(src_term), re.IGNORECASE)
if src_pattern.search(val):
new_val = src_pattern.sub(lambda _: tgt_term, val)
if new_val != val:

20
backend/src/plugins/translate/dictionary_entries.py
View File

@@ -3,6 +3,8 @@
# @RELATION DEPENDS_ON -> [DictionaryEntry]
# @RELATION DEPENDS_ON -> [TerminologyDictionary]
import re
from sqlalchemy import func
from sqlalchemy.orm import Session
@@ -22,11 +24,18 @@ class DictionaryEntryCRUD:
db: Session, dict_id: str, source_term: str, target_term: str,
context_notes: str | None = None,
source_language: str = "und", target_language: str = "und",
is_regex: bool = False,
) -> DictionaryEntry:
with belief_scope("DictionaryEntryCRUD.add_entry"):
_validate_bcp47(source_language, "source_language")
_validate_bcp47(target_language, "target_language")
if is_regex:
try:
re.compile(source_term)
except re.error as e:
raise ValueError(f"Invalid regex pattern: {e}")
normalized = _normalize_term(source_term)
existing = (
db.query(DictionaryEntry)
@@ -54,6 +63,7 @@ class DictionaryEntryCRUD:
context_notes=context_notes.strip() if context_notes else None,
source_language=source_language.strip(),
target_language=target_language.strip(),
is_regex=is_regex,
)
db.add(entry)
db.commit()
@@ -69,6 +79,7 @@ class DictionaryEntryCRUD:
db: Session, entry_id: str, source_term: str | None = None,
target_term: str | None = None, context_notes: str | None = None,
source_language: str | None = None, target_language: str | None = None,
is_regex: bool | None = None,
) -> DictionaryEntry:
with belief_scope("DictionaryEntryCRUD.edit_entry"):
entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()
@@ -84,6 +95,13 @@ class DictionaryEntryCRUD:
entry.target_language = target_language.strip()
if source_term is not None:
# Validate regex if the entry is or will become a regex entry
entry_is_regex = is_regex if is_regex is not None else entry.is_regex
if entry_is_regex:
try:
re.compile(source_term)
except re.error as e:
raise ValueError(f"Invalid regex pattern: {e}")
normalized = _normalize_term(source_term)
existing = (
db.query(DictionaryEntry)
@@ -104,6 +122,8 @@ class DictionaryEntryCRUD:
entry.target_term = target_term.strip()
if context_notes is not None:
entry.context_notes = context_notes.strip() if context_notes else None
if is_regex is not None:
entry.is_regex = is_regex
db.commit()
db.refresh(entry)
logger.reflect("Entry updated", {"entry_id": entry.id})

15
backend/src/plugins/translate/dictionary_filter.py
View File

@@ -84,10 +84,18 @@ class DictionaryBatchFilter:
if entry_tgt != tgt:
continue
escaped = re.escape(norm)
pattern = re.compile(r"\b" + escaped + r"\b", re.IGNORECASE)
if entry.is_regex:
try:
regex_pattern = re.compile(entry.source_term, re.IGNORECASE)
matched_source = regex_pattern.search(text_lower) is not None
except re.error:
matched_source = False
else:
escaped = re.escape(norm)
pattern = re.compile(r"\b" + escaped + r"\b", re.IGNORECASE)
matched_source = pattern.search(text_lower) is not None
if pattern.search(text_lower):
if matched_source:
match_key = (text_idx, entry.id)
if match_key not in seen_source_match:
seen_source_match.add(match_key)
@@ -111,6 +119,7 @@ class DictionaryBatchFilter:
"context_notes": entry.context_notes,
"context_data": entry.context_data,
"has_context": entry.has_context,
"is_regex": entry.is_regex,
"context_source": entry.context_source,
"usage_notes": entry.usage_notes,
"source_language": entry.source_language,

5
backend/src/plugins/translate/metrics.py
View File

@@ -150,6 +150,11 @@ class TranslationMetrics:
per_language[lang]["cost"] += snap_data.get("cumulative_cost", 0.0)
per_language[lang]["runs"] += snap_data.get("runs", 0)
# Aggregate per-language tokens/cost into cumulative
for lang_data in per_language.values():
cumulative_tokens += lang_data.get("tokens", 0) or 0
cumulative_cost += lang_data.get("cost", 0.0) or 0.0
return {
"job_id": job_id,
"total_runs": total_runs,