busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(translate): multi-language optimization (Phase 11)

Browse Source 
- Auto-detection of source language per row via LLM (US6)
- Multi-target translation — one LLM call for N languages (US1-US3)
- Language-aware storage: TranslationLanguage, per-language stats
- Multilingual dictionaries with language-pair-aware filtering (US7)
- Inline correction on any run result + submit-to-dictionary (US8)
- Context-aware dictionary: auto-capture row context, usage notes,
  Jaccard similarity, priority flagging in LLM prompts (US8b)
- Configurable preview sample size 1-100, cost warning at >30
- Per-language history & metrics with MetricSnapshot preservation
- 36 files, +5022/-373, all specs GRACE-Poly v2.6 compliant
This commit is contained in:
busya
2026-05-14 17:12:41 +03:00
parent 5e741a4332
commit bb0fbfdafd
36 changed files with 5025 additions and 376 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/api/routes/translate/_correction_routes.py
View File

@@ -45,6 +45,8 @@ async def submit_correction(
origin_run_id=payload.origin_run_id,
origin_row_key=payload.origin_row_key,
origin_user_id=current_user.username,
context_data=payload.context_data,
usage_notes=payload.usage_notes,
)
return result
except ValueError as e:

46
backend/src/api/routes/translate/_dictionary_routes.py
View File

@@ -163,14 +163,16 @@ async def delete_dictionary(
# ============================================================
# #region list_dictionary_entries [TYPE Function]
# @BRIEF List entries for a dictionary.
# @BRIEF List entries for a dictionary, optionally filtered by language pair.
# @PRE: User has translate.dictionary.view permission.
# @POST: Returns paginated list of entries.
# @POST: Returns paginated list of entries with language pair fields.
@router.get("/dictionaries/{dictionary_id}/entries")
async def list_dictionary_entries(
dictionary_id: str,
page: int = Query(1, ge=1),
page_size: int = Query(50, ge=1, le=500),
source_language: str | None = Query(None, description="Filter by BCP-47 source language"),
target_language: str | None = Query(None, description="Filter by BCP-47 target language"),
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.dictionary", "VIEW")),
db: Session = Depends(get_db),
@@ -180,6 +182,17 @@ async def list_dictionary_entries(
logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}")
try:
entries, total = DictionaryManager.list_entries(db, dictionary_id, page=page, page_size=page_size)
# Apply language pair filtering
if source_language or target_language:
filtered = []
for e in entries:
if source_language and e.source_language != source_language and e.source_language != "und":
continue
if target_language and e.target_language != target_language:
continue
filtered.append(e)
entries = filtered
total = len(filtered)
items = [
{
"id": e.id,
@@ -187,13 +200,20 @@ async def list_dictionary_entries(
"source_term": e.source_term,
"source_term_normalized": e.source_term_normalized,
"target_term": e.target_term,
"source_language": e.source_language,
"target_language": e.target_language,
"context_notes": e.context_notes,
"context_data": e.context_data,
"usage_notes": e.usage_notes,
"has_context": e.has_context,
"context_source": e.context_source,
"origin_source_language": e.origin_source_language,
"created_at": e.created_at,
"updated_at": e.updated_at,
}
for e in entries
]
logger.reflect("Entries listed", {"dict_id": dictionary_id, "total": total})
logger.reflect("Entries listed", {"dict_id": dictionary_id, "total": total, "source_language": source_language, "target_language": target_language})
return {"items": items, "total": total, "page": page, "page_size": page_size}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
@@ -221,6 +241,8 @@ async def add_dictionary_entry(
source_term=payload.source_term,
target_term=payload.target_term,
context_notes=payload.context_notes,
source_language=payload.source_language,
target_language=payload.target_language,
)
logger.reflect("Entry added", {"entry_id": entry.id})
return {
@@ -229,7 +251,14 @@ async def add_dictionary_entry(
"source_term": entry.source_term,
"source_term_normalized": entry.source_term_normalized,
"target_term": entry.target_term,
"source_language": entry.source_language,
"target_language": entry.target_language,
"context_notes": entry.context_notes,
"context_data": entry.context_data,
"usage_notes": entry.usage_notes,
"has_context": entry.has_context,
"context_source": entry.context_source,
"origin_source_language": entry.origin_source_language,
"created_at": entry.created_at,
"updated_at": entry.updated_at,
}
@@ -262,6 +291,8 @@ async def edit_dictionary_entry(
source_term=payload.source_term,
target_term=payload.target_term,
context_notes=payload.context_notes,
source_language=payload.source_language,
target_language=payload.target_language,
)
logger.reflect("Entry updated", {"entry_id": entry_id})
return {
@@ -270,7 +301,14 @@ async def edit_dictionary_entry(
"source_term": entry.source_term,
"source_term_normalized": entry.source_term_normalized,
"target_term": entry.target_term,
"source_language": entry.source_language,
"target_language": entry.target_language,
"context_notes": entry.context_notes,
"context_data": entry.context_data,
"usage_notes": entry.usage_notes,
"has_context": entry.has_context,
"context_source": entry.context_source,
"origin_source_language": entry.origin_source_language,
"created_at": entry.created_at,
"updated_at": entry.updated_at,
}
@@ -335,6 +373,8 @@ async def import_dictionary_entries(
delimiter=payload.delimiter,
on_conflict=payload.on_conflict,
preview_only=payload.preview_only,
default_source_language=payload.default_source_language,
default_target_language=payload.default_target_language,
)
logger.reflect("Import completed", {
"dict_id": dictionary_id,

7
backend/src/api/routes/translate/_preview_routes.py
View File

@@ -59,7 +59,7 @@ async def preview_translation(
# #region update_preview_row [TYPE Function]
# @BRIEF Approve, edit, or reject a preview row.
# @BRIEF Approve, edit, or reject a preview row (optionally per language).
# @PRE: User has translate.job.execute permission.
# @POST: Preview row status is updated.
@router.put("/jobs/{job_id}/preview/rows/{row_key}")
@@ -72,8 +72,8 @@ async def update_preview_row(
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Approve, edit, or reject a preview row."""
logger.reason(f"update_preview_row — Job: {job_id}, Row: {row_key}, Action: {payload.action}", extra={"src": "translate_routes"})
"""Approve, edit, or reject a preview row (optionally per language)."""
logger.reason(f"update_preview_row — Job: {job_id}, Row: {row_key}, Action: {payload.action}, Lang: {payload.language_code}", extra={"src": "translate_routes"})
try:
preview_service = TranslationPreview(db, config_manager, current_user.username)
result = preview_service.update_preview_row(
@@ -82,6 +82,7 @@ async def update_preview_row(
action=payload.action,
translation=payload.translation,
feedback=payload.feedback,
language_code=payload.language_code,
)
return result
except ValueError as e:

19
backend/src/api/routes/translate/_run_list_routes.py
View File

@@ -76,6 +76,22 @@ async def list_runs(
items = []
for r in runs:
# Gather per-language statistics from the run's language_stats relationship
lang_stats = r.language_stats or []
target_languages = [ls.language_code for ls in lang_stats]
total_translated = sum((ls.translated_rows or 0) for ls in lang_stats)
language_stats = {
ls.language_code: {
"total_rows": ls.total_rows or 0,
"translated_rows": ls.translated_rows or 0,
"failed_rows": ls.failed_rows or 0,
"skipped_rows": ls.skipped_rows or 0,
"token_count": ls.token_count or 0,
"estimated_cost": ls.estimated_cost or 0.0,
}
for ls in lang_stats
}
items.append({
"id": r.id,
"job_id": r.job_id,
@@ -94,6 +110,9 @@ async def list_runs(
"dict_snapshot_hash": r.dict_snapshot_hash,
"created_by": r.created_by,
"created_at": r.created_at.isoformat() if r.created_at else None,
"target_languages": target_languages,
"total_translated": total_translated,
"language_stats": language_stats,
})
return {"items": items, "total": total, "page": page, "page_size": page_size}

197
backend/src/api/routes/translate/_run_routes.py
View File

@@ -13,6 +13,11 @@ from ....core.logger import logger
from ....dependencies import get_config_manager, get_current_user, has_permission
from ....plugins.translate.orchestrator import TranslationOrchestrator
from ....schemas.auth import User
from ....schemas.translate import (
BulkFindReplaceRequest,
InlineEditRequest,
OverrideLanguageRequest,
)
from ._helpers import _run_to_response
from ._router import router
@@ -341,4 +346,196 @@ async def get_batches(
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# #endregion get_batches
# ============================================================
# Manual Language Override
# ============================================================
# #region override_detected_language [TYPE Function]
# @BRIEF Manually override the detected source language for a specific translation language entry.
# @PRE: User has translate.job.execute permission. Run, record, and language entry exist.
# @POST: TranslationLanguage.source_language_detected is updated; language_overridden is set to True.
# @SIDE_EFFECT: DB write.
@router.put("/runs/{run_id}/records/{record_id}/languages/{language_code}/override-language")
async def override_detected_language(
run_id: str,
record_id: str,
language_code: str,
payload: OverrideLanguageRequest,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EXECUTE")),
db: Session = Depends(get_db),
):
"""Manually override the detected source language for a translation entry."""
logger.reason(f"override_detected_language — Run: {run_id}, Record: {record_id}, Lang: {language_code}, Override: {payload.source_language}", extra={"src": "translate_routes"})
try:
from ....models.translate import TranslationLanguage, TranslationRecord
# Verify the record exists and belongs to the run
record = (
db.query(TranslationRecord)
.filter(
TranslationRecord.id == record_id,
TranslationRecord.run_id == run_id,
)
.first()
)
if not record:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Translation record '{record_id}' not found in run '{run_id}'",
)
# Find the language entry
lang_entry = (
db.query(TranslationLanguage)
.filter(
TranslationLanguage.record_id == record_id,
TranslationLanguage.language_code == language_code,
)
.first()
)
if not lang_entry:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Language entry '{language_code}' not found for record '{record_id}'",
)
# Update the language entry
lang_entry.source_language_detected = payload.source_language
lang_entry.language_overridden = True
lang_entry.needs_review = False # Override clears the review flag
db.commit()
db.refresh(lang_entry)
logger.reason("Language override applied", {
"run_id": run_id,
"record_id": record_id,
"language_code": language_code,
"new_source_language": payload.source_language,
"overridden_by": current_user.username,
})
from ....schemas.translate import TranslationLanguageResponse
return TranslationLanguageResponse.model_validate(lang_entry)
except HTTPException:
raise
except Exception as e:
logger.explore("override_detected_language failed", extra={"src": "translate_routes", "error": str(e)})
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# #endregion override_detected_language
# ============================================================
# Inline Correction
# ============================================================
# #region inline_edit_translation [C:3] [TYPE Function] [SEMANTICS api,translate,correction]
# @BRIEF Apply an inline correction to a translated value on a completed run result.
# @PRE: User has translate.job.execute permission. Run, record, and language entry exist.
# @POST: TranslationLanguage.final_value and user_edit are updated. Optional dictionary submission.
@router.put("/runs/{run_id}/records/{record_id}/languages/{language_code}")
async def inline_edit_translation(
run_id: str,
record_id: str,
language_code: str,
payload: InlineEditRequest,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EXECUTE")),
db: Session = Depends(get_db),
):
"""Apply an inline correction to a translated value."""
logger.reason(
f"inline_edit_translation — Run: {run_id}, Record: {record_id}, "
f"Lang: {language_code}, User: {current_user.username}",
extra={"src": "translate_routes"},
)
try:
from ....plugins.translate.service import InlineCorrectionService
result = InlineCorrectionService.apply_inline_edit(
db=db,
run_id=run_id,
record_id=record_id,
language_code=language_code,
final_value=payload.final_value,
submit_to_dictionary=payload.submit_to_dictionary,
dictionary_id=payload.dictionary_id,
current_user=current_user.username,
context_data_override=payload.context_data_override,
usage_notes=payload.usage_notes,
keep_context=payload.keep_context,
)
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
except Exception as e:
logger.explore("inline_edit_translation failed", extra={"src": "translate_routes", "error": str(e)})
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# #endregion inline_edit_translation
# ============================================================
# Bulk Find-and-Replace
# ============================================================
# #region bulk_find_replace [C:3] [TYPE Function] [SEMANTICS api,translate,bulk,replace]
# @BRIEF Perform bulk find-and-replace on translated values within a run.
# @PRE: User has translate.job.execute permission. Run exists.
# @POST: If preview=false, matching translations are updated. Optional dictionary submission.
@router.post("/runs/{run_id}/bulk-replace")
async def bulk_find_replace(
run_id: str,
payload: BulkFindReplaceRequest,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EXECUTE")),
db: Session = Depends(get_db),
):
"""Perform bulk find-and-replace on translated values within a run."""
logger.reason(
f"bulk_find_replace — Run: {run_id}, Pattern: '{payload.find_pattern}', "
f"Regex: {payload.is_regex}, Lang: {payload.target_language}, "
f"User: {current_user.username}",
extra={"src": "translate_routes"},
)
try:
from ....plugins.translate.service import BulkFindReplaceService
if payload.preview:
# Preview mode: return matches without applying
preview = BulkFindReplaceService.preview(
db=db,
run_id=run_id,
pattern=payload.find_pattern,
is_regex=payload.is_regex,
target_language=payload.target_language,
)
return {
"rows_affected": len(preview),
"corrections_submitted": 0,
"preview": preview,
}
# Apply mode: perform replacements
result = BulkFindReplaceService.apply(
db=db,
run_id=run_id,
pattern=payload.find_pattern,
is_regex=payload.is_regex,
replacement_text=payload.replacement_text,
target_language=payload.target_language,
submit_to_dictionary=payload.submit_to_dictionary,
dictionary_id=payload.dictionary_id,
usage_notes=payload.usage_notes,
current_user=current_user.username,
submit_to_dictionary_with_context=payload.submit_to_dictionary_with_context,
)
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
except Exception as e:
logger.explore("bulk_find_replace failed", extra={"src": "translate_routes", "error": str(e)})
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# #endregion bulk_find_replace
# #endregion TranslateRunRoutesModule