ss-tools/backend/src/plugins/translate/_utils.py

# #region TranslationUtils [C:3] [TYPE Module] [SEMANTICS translate, utils, hash, dictionary, cache]
# @defgroup Translate Module group.
# @BRIEF Shared utility functions for the translation plugin — dictionary enforcement,
#         source hashing, cache lookup. Extracted from executor.py to break circular imports.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationRecord]
# @RELATION DEPENDS_ON -> [TranslationLanguage]
# @RATIONALE Extracted from TranslationExecutor to avoid circular imports when sub-services
#            (batch_proc, llm_call) need these helpers. The original monolithic executor.py
#            violated INV_7 at 1974 lines.

import hashlib
import json
import re
from typing import Any
import unicodedata
from urllib.parse import urlsplit, urlunsplit

from sqlalchemy.orm import Session, joinedload

from ...core.logger import logger
from ...models.translate import TranslationRecord


# #region _sanitize_url [C:1] [TYPE Function] [SEMANTICS translate, url, sanitize]
# @BRIEF Strip embedded credentials from URL for safe logging.
# @POST Returns URL with user:pass@ portion removed, preserving host:port.
def _sanitize_url(url: str) -> str:
    """Strip embedded credentials from URL for safe logging."""
    if not url:
        return url
    parsed = urlsplit(url)
    if parsed.username or parsed.password:
        safe_netloc = parsed.hostname
        if parsed.port:
            safe_netloc += f":{parsed.port}"
        parsed = parsed._replace(netloc=safe_netloc)
    return urlunsplit(parsed)
# #endregion _sanitize_url


# #region _normalize_term [TYPE Function]
# @BRIEF Normalize a term for case-insensitive unique constraint lookup.
# @RATIONALE  NFC normalization is applied before lowercasing to ensure consistent
#             comparison of Unicode characters (e.g. precomposed vs decomposed forms).
# @REJECTED  Lowercasing without NFC normalization — would cause duplicate entries
#            for semantically identical Unicode strings in different normalization forms.
def _normalize_term(term: str) -> str:
    """Normalize a term by NFC, lowercasing, and removing extra whitespace."""
    if not term:
        return ""
    term = unicodedata.normalize('NFC', term)
    return " ".join(term.lower().split())
# #endregion _normalize_term


# #region _detect_delimiter [TYPE Function]
# @BRIEF Detect the delimiter used in a CSV/TSV header line.
def _detect_delimiter(header_line: str) -> str:
    """Detect delimiter by counting tabs vs commas in the first line."""
    if not header_line:
        return ","
    tab_count = header_line.count("\t")
    comma_count = header_line.count(",")
    return "\t" if tab_count > comma_count else ","
# #endregion _detect_delimiter


# #region _enforce_dictionary [C:2] [TYPE Function] [SEMANTICS translate, dictionary, post-processing]
# @BRIEF Post-process LLM output: enforce dictionary term replacements.
# @PRE dict_matches is a list of dict entries with source_term/target_term.
# @POST per_lang_values may be mutated to include forced dictionary replacements.
# @SIDE_EFFECT Logs when enforcement is applied.
def _enforce_dictionary(
    source_text: str,
    per_lang_values: dict[str, str],
    dict_matches: list[dict[str, Any]],
    batch_id: str,
    row_id: str,
) -> None:
    """Post-process LLM output: enforce dictionary term replacements.

    For each dictionary entry whose source_term appears in the source_text,
    verify that the target_term is present in each language's translation.
    If missing, replace any occurrence of the source term (which the LLM
    may have left untranslated) with the dictionary target term.
    """
    if not dict_matches or not source_text:
        return

    text_lower = source_text.lower()

    for dm in dict_matches:
        src_term = dm.get("source_term", "")
        tgt_term = dm.get("target_term", "")
        is_regex = dm.get("is_regex", False)
        if not src_term or not tgt_term:
            continue

        if is_regex:
            try:
                src_check = re.compile(src_term, re.IGNORECASE)
                source_matched = src_check.search(source_text) is not None
            except re.error:
                continue
        else:
            if src_term.lower() not in text_lower:
                continue
            source_matched = True

        if not source_matched:
            continue

        for lang_code in list(per_lang_values.keys()):
            val = per_lang_values[lang_code]
            if not val:
                continue

            if tgt_term.lower() in val.lower():
                continue

            if is_regex:
                try:
                    src_pattern = re.compile(src_term, re.IGNORECASE)
                except re.error:
                    continue
            else:
                src_pattern = re.compile(re.escape(src_term), re.IGNORECASE)

            if src_pattern.search(val):
                new_val = src_pattern.sub(lambda _: tgt_term, val)
                if new_val != val:
                    logger.reason("Dictionary enforcement applied", {
                        "batch_id": batch_id,
                        "row_id": row_id,
                        "language_code": lang_code,
                        "source_term": src_term,
                        "target_term": tgt_term,
                        "before": val[:200],
                        "after": new_val[:200],
                    })
                    per_lang_values[lang_code] = new_val
# #endregion _enforce_dictionary


# #region _compute_source_hash [C:2] [TYPE Function] [SEMANTICS translate, hash, cache-key]
# @BRIEF Compute deterministic cache key for a source row.
# SHA256 of (source_text + context_fields + dict_snapshot_hash + config_hash).
def _compute_source_hash(
    source_text: str,
    source_data: dict | None,
    dict_snapshot_hash: str | None,
    config_hash: str | None,
    context_keys: list[str] | None = None,
) -> str:
    """Deterministic cache key for a translation source row.

    Only includes source_text and context-relevant fields from source_data.
    Key/identifier columns are excluded so identical text in different rows hits cache.
    """
    context_data: dict[str, str] = {}
    if source_data and context_keys:
        for key in context_keys:
            val = source_data.get(key)
            if val is not None:
                context_data[key] = str(val)

    payload = json.dumps({
        "text": source_text,
        "ctx": context_data,
        "dict_hash": dict_snapshot_hash or "",
        "config_hash": config_hash or "",
    }, sort_keys=True, ensure_ascii=False)
    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
# #endregion _compute_source_hash


# #region _check_translation_cache [C:2] [TYPE Function] [SEMANTICS translate, cache, lookup]
# @BRIEF Look up a previously successful translation by source_hash.
# Returns per-language dict {lang_code: final_value} or None.
def _check_translation_cache(
    db: Session,
    source_hash: str,
) -> dict[str, str] | None:
    """Check if this source_hash was already translated successfully."""
    cached = (
        db.query(TranslationRecord)
        .options(joinedload(TranslationRecord.languages))
        .filter(
            TranslationRecord.source_hash == source_hash,
            TranslationRecord.status == "SUCCESS",
        )
        .order_by(TranslationRecord.created_at.desc())
        .first()
    )
    if not cached:
        return None

    lang_values: dict[str, str] = {}
    for lang in cached.languages:
        if lang.status == "translated" and lang.final_value:
            lang_values[lang.language_code] = lang.final_value

    return lang_values if lang_values else None
# #endregion _check_translation_cache


# #region _compute_key_hash [C:1] [TYPE Function]
# @BRIEF Compute a stable hash from source_data dict for matching preview edits.
def _compute_key_hash(source_data: dict) -> str:
    """Compute a stable hash from source_data dict for matching preview edits."""
    import hashlib
    stable = json.dumps(source_data, sort_keys=True)
    return hashlib.sha256(stable.encode()).hexdigest()[:16]
# #endregion _compute_key_hash


# #region estimate_row_tokens [C:2] [TYPE Function] [SEMANTICS translate, token, estimation]
# @ingroup Translate
# @BRIEF Estimate token count for a single source row including context fields.
# @PRE source_text is a string.
# @POST Returns estimated token count >= 1.
def estimate_row_tokens(
    source_text: str,
    source_data: dict | None,
    job,
) -> int:
    """Estimate token count for a single source row including context fields.

    Uses CJK-aware heuristics via _token_budget._estimate_tokens_for_text.
    Context fields from job.context_columns are included in the estimate.
    """
    from ._token_budget import _estimate_tokens_for_text

    text_tokens = _estimate_tokens_for_text(source_text or "")

    context_keys = job.context_columns or []
    ctx_text = ""
    if source_data and context_keys:
        ctx_text = " ".join(str(source_data.get(k, "")) for k in context_keys)
    ctx_tokens = _estimate_tokens_for_text(ctx_text)

    return text_tokens + ctx_tokens
# #endregion estimate_row_tokens
# #endregion TranslationUtils