feat(translate): add run preflight and focus execution UX
This commit is contained in:
@@ -0,0 +1,52 @@
|
|||||||
|
# #region Alembic.TranslateActiveRunGuard [C:3] [TYPE Module] [SEMANTICS alembic,migration,translate,concurrency]
|
||||||
|
# @BRIEF Enforces at most one pending/running translation run per job.
|
||||||
|
# @RELATION DEPENDS_ON -> [EXT:SQLAlchemy:Alembic]
|
||||||
|
# @POST Concurrent manual/scheduled triggers cannot create duplicate active runs.
|
||||||
|
"""add unique active translation run guard
|
||||||
|
|
||||||
|
Revision ID: d4e5f6a7b8c9
|
||||||
|
Revises: c3d4e5f6a7b8
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision: str = "d4e5f6a7b8c9"
|
||||||
|
down_revision: str | Sequence[str] | None = "c3d4e5f6a7b8"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Create a partial unique index on supported application databases."""
|
||||||
|
bind = op.get_bind()
|
||||||
|
dialect = bind.dialect.name
|
||||||
|
if dialect == "postgresql":
|
||||||
|
op.create_index(
|
||||||
|
"uq_translation_runs_one_active_per_job",
|
||||||
|
"translation_runs",
|
||||||
|
["job_id"],
|
||||||
|
unique=True,
|
||||||
|
postgresql_where=sa.text("status IN ('PENDING', 'RUNNING')"),
|
||||||
|
)
|
||||||
|
elif dialect == "sqlite":
|
||||||
|
op.create_index(
|
||||||
|
"uq_translation_runs_one_active_per_job",
|
||||||
|
"translation_runs",
|
||||||
|
["job_id"],
|
||||||
|
unique=True,
|
||||||
|
sqlite_where=sa.text("status IN ('PENDING', 'RUNNING')"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Drop the active-run guard."""
|
||||||
|
bind = op.get_bind()
|
||||||
|
if bind.dialect.name in {"postgresql", "sqlite"}:
|
||||||
|
op.drop_index("uq_translation_runs_one_active_per_job", table_name="translation_runs")
|
||||||
|
|
||||||
|
|
||||||
|
# #endregion Alembic.TranslateActiveRunGuard
|
||||||
@@ -10,6 +10,7 @@ from datetime import UTC, datetime
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from fastapi import Depends, HTTPException, Query, status
|
from fastapi import Depends, HTTPException, Query, status
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from ....core.config_manager import ConfigManager
|
from ....core.config_manager import ConfigManager
|
||||||
@@ -19,6 +20,8 @@ from ....core.logger import logger
|
|||||||
from ....dependencies import get_config_manager, get_current_user, has_permission
|
from ....dependencies import get_config_manager, get_current_user, has_permission
|
||||||
from ....plugins.translate.orchestrator import TranslationOrchestrator
|
from ....plugins.translate.orchestrator import TranslationOrchestrator
|
||||||
from ....schemas.auth import User
|
from ....schemas.auth import User
|
||||||
|
from ....schemas.translate import RunPreflightResponse
|
||||||
|
from ....plugins.translate.run_preflight import TranslationRunPreflight
|
||||||
from ._helpers import _run_to_response
|
from ._helpers import _run_to_response
|
||||||
from ._router import router
|
from ._router import router
|
||||||
|
|
||||||
@@ -33,6 +36,7 @@ from ._router import router
|
|||||||
async def run_translation(
|
async def run_translation(
|
||||||
job_id: str,
|
job_id: str,
|
||||||
full_translation: bool = Query(False, description="Translate ALL rows (skip new-key-only filter)"),
|
full_translation: bool = Query(False, description="Translate ALL rows (skip new-key-only filter)"),
|
||||||
|
language_detection: str = Query("auto", pattern="^(auto|skip)$"),
|
||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
_ = Depends(has_permission("translate.job", "EXECUTE")),
|
_ = Depends(has_permission("translate.job", "EXECUTE")),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
@@ -49,7 +53,12 @@ async def run_translation(
|
|||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
orch = TranslationOrchestrator(db, config_manager, current_user.username)
|
orch = TranslationOrchestrator(db, config_manager, current_user.username)
|
||||||
run = orch.start_run(job_id=job_id, is_scheduled=False, full_translation=full_translation)
|
run = orch.start_run(
|
||||||
|
job_id=job_id,
|
||||||
|
is_scheduled=False,
|
||||||
|
full_translation=full_translation,
|
||||||
|
language_detection=language_detection,
|
||||||
|
)
|
||||||
# The request-scoped db session will be closed after this handler returns.
|
# The request-scoped db session will be closed after this handler returns.
|
||||||
# The background task must use its OWN session.
|
# The background task must use its OWN session.
|
||||||
_request_trace_id = get_trace_id()
|
_request_trace_id = get_trace_id()
|
||||||
@@ -124,12 +133,41 @@ async def run_translation(
|
|||||||
return _run_to_response(run)
|
return _run_to_response(run)
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||||
|
except IntegrityError:
|
||||||
|
db.rollback()
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_409_CONFLICT,
|
||||||
|
detail="This translation job already has an active run. Wait for it to finish or cancel it before starting another run.",
|
||||||
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.explore("run_translation failed", extra={"src": "translate_routes", "error": str(e)})
|
logger.explore("run_translation failed", extra={"src": "translate_routes", "error": str(e)})
|
||||||
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Run failed: {e}")
|
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Run failed: {e}")
|
||||||
# #endregion run_translation
|
# #endregion run_translation
|
||||||
|
|
||||||
|
|
||||||
|
# #region calculate_run_preflight [C:4] [TYPE Function] [SEMANTICS translate,api,preflight]
|
||||||
|
# @BRIEF Calculate the actual run scope and Lingua distribution without creating a translation run.
|
||||||
|
# @PRE User can execute the translation job.
|
||||||
|
# @POST Returns scope, token/cost estimate, and aggregate language distribution only.
|
||||||
|
@router.post("/jobs/{job_id}/run-preflight", response_model=RunPreflightResponse)
|
||||||
|
async def calculate_run_preflight(
|
||||||
|
job_id: str,
|
||||||
|
full_translation: bool = Query(False),
|
||||||
|
current_user: User = Depends(get_current_user),
|
||||||
|
_ = Depends(has_permission("translate.job", "EXECUTE")),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
config_manager: ConfigManager = Depends(get_config_manager),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
return await TranslationRunPreflight(db, config_manager).calculate(job_id, full_translation)
|
||||||
|
except ValueError as e:
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||||
|
except Exception as e:
|
||||||
|
logger.explore("calculate_run_preflight failed", extra={"src": "translate_routes", "error": str(e)})
|
||||||
|
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Run preflight failed: {e}")
|
||||||
|
# #endregion calculate_run_preflight
|
||||||
|
|
||||||
|
|
||||||
# #region retry_run [C:4] [TYPE Function]
|
# #region retry_run [C:4] [TYPE Function]
|
||||||
# @ingroup Api
|
# @ingroup Api
|
||||||
# @BRIEF Retry failed batches in a translation run.
|
# @BRIEF Retry failed batches in a translation run.
|
||||||
|
|||||||
@@ -103,6 +103,16 @@ class TranslationRun(Base):
|
|||||||
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
|
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
|
||||||
|
|
||||||
language_stats = relationship("TranslationRunLanguageStats", back_populates="run")
|
language_stats = relationship("TranslationRunLanguageStats", back_populates="run")
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
Index(
|
||||||
|
"uq_translation_runs_one_active_per_job",
|
||||||
|
"job_id",
|
||||||
|
unique=True,
|
||||||
|
postgresql_where=status.in_(("PENDING", "RUNNING")),
|
||||||
|
sqlite_where=status.in_(("PENDING", "RUNNING")),
|
||||||
|
),
|
||||||
|
)
|
||||||
# #endregion TranslationRun
|
# #endregion TranslationRun
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ from sqlalchemy.orm import Session
|
|||||||
|
|
||||||
from ...core.config_manager import ConfigManager
|
from ...core.config_manager import ConfigManager
|
||||||
from ...core.logger import belief_scope, logger
|
from ...core.logger import belief_scope, logger
|
||||||
from ...models.translate import TranslationBatch, TranslationJob, TranslationLanguage, TranslationRecord
|
from ...models.translate import TranslationBatch, TranslationJob, TranslationLanguage, TranslationRecord, TranslationRun
|
||||||
from ...services.llm_provider import LLMProviderService
|
from ...services.llm_provider import LLMProviderService
|
||||||
from ._batch_insert import insert_batch_to_target
|
from ._batch_insert import insert_batch_to_target
|
||||||
from ._lang_detect import batch_detect
|
from ._lang_detect import batch_detect
|
||||||
@@ -68,8 +68,14 @@ class BatchProcessingService:
|
|||||||
tls = job.target_languages or [job.target_dialect or "en"]
|
tls = job.target_languages or [job.target_dialect or "en"]
|
||||||
tls = [str(tls)] if not isinstance(tls, list) else tls
|
tls = [str(tls)] if not isinstance(tls, list) else tls
|
||||||
|
|
||||||
# ★ Run local language detection on all rows (heuristic, no LLM)
|
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
|
||||||
await self._detect_languages(batch_rows, tls)
|
detection_mode = (run.config_snapshot or {}).get("language_detection", "auto") if run else "auto"
|
||||||
|
if detection_mode == "skip":
|
||||||
|
for row in batch_rows:
|
||||||
|
row["_detected_lang"] = "und"
|
||||||
|
else:
|
||||||
|
# ★ Run local language detection on all rows (heuristic, no LLM)
|
||||||
|
await self._detect_languages(batch_rows, tls)
|
||||||
|
|
||||||
# Emit LANGUAGE_DETECTION_COMPLETED aggregate event (no source text in payload)
|
# Emit LANGUAGE_DETECTION_COMPLETED aggregate event (no source text in payload)
|
||||||
lang_counts: dict[str, int] = {}
|
lang_counts: dict[str, int] = {}
|
||||||
@@ -83,7 +89,7 @@ class BatchProcessingService:
|
|||||||
run_id=run_id,
|
run_id=run_id,
|
||||||
event_type="LANGUAGE_DETECTION_COMPLETED",
|
event_type="LANGUAGE_DETECTION_COMPLETED",
|
||||||
payload={
|
payload={
|
||||||
"detector": "lingua",
|
"detector": "lingua" if detection_mode != "skip" else "skipped",
|
||||||
"source_language_distribution": dict(sorted(lang_counts.items())),
|
"source_language_distribution": dict(sorted(lang_counts.items())),
|
||||||
"und_count": und_count,
|
"und_count": und_count,
|
||||||
"und_rate": round(und_count / total, 4) if total > 0 else 0.0,
|
"und_rate": round(und_count / total, 4) if total > 0 else 0.0,
|
||||||
|
|||||||
@@ -73,6 +73,7 @@ class TranslationOrchestrator:
|
|||||||
is_scheduled: bool = False,
|
is_scheduled: bool = False,
|
||||||
trigger_type: str | None = None,
|
trigger_type: str | None = None,
|
||||||
full_translation: bool = False,
|
full_translation: bool = False,
|
||||||
|
language_detection: str = "auto",
|
||||||
) -> TranslationRun:
|
) -> TranslationRun:
|
||||||
with belief_scope("TranslationOrchestrator.start_run"):
|
with belief_scope("TranslationOrchestrator.start_run"):
|
||||||
return self._planner.plan_run(
|
return self._planner.plan_run(
|
||||||
@@ -80,6 +81,7 @@ class TranslationOrchestrator:
|
|||||||
is_scheduled=is_scheduled,
|
is_scheduled=is_scheduled,
|
||||||
trigger_type=trigger_type,
|
trigger_type=trigger_type,
|
||||||
full_translation=full_translation,
|
full_translation=full_translation,
|
||||||
|
language_detection=language_detection,
|
||||||
)
|
)
|
||||||
# endregion start_run
|
# endregion start_run
|
||||||
|
|
||||||
|
|||||||
@@ -46,12 +46,27 @@ class TranslationPlanner:
|
|||||||
is_scheduled: bool = False,
|
is_scheduled: bool = False,
|
||||||
trigger_type: str | None = None,
|
trigger_type: str | None = None,
|
||||||
full_translation: bool = False,
|
full_translation: bool = False,
|
||||||
|
language_detection: str = "auto",
|
||||||
) -> TranslationRun:
|
) -> TranslationRun:
|
||||||
with belief_scope("TranslationPlanner.plan_run"):
|
with belief_scope("TranslationPlanner.plan_run"):
|
||||||
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
|
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
|
||||||
if not job:
|
if not job:
|
||||||
raise ValueError(f"Translation job '{job_id}' not found")
|
raise ValueError(f"Translation job '{job_id}' not found")
|
||||||
|
|
||||||
|
active_run = (
|
||||||
|
self.db.query(TranslationRun)
|
||||||
|
.filter(
|
||||||
|
TranslationRun.job_id == job_id,
|
||||||
|
TranslationRun.status.in_(("PENDING", "RUNNING")),
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if isinstance(active_run, TranslationRun):
|
||||||
|
raise ValueError(
|
||||||
|
f"Translation job '{job_id}' already has an active run '{active_run.id}'. "
|
||||||
|
"Wait for it to finish or cancel it before starting another run."
|
||||||
|
)
|
||||||
|
|
||||||
validate_job_preconditions(job, self.db, is_scheduled=is_scheduled)
|
validate_job_preconditions(job, self.db, is_scheduled=is_scheduled)
|
||||||
|
|
||||||
config_hash_value = compute_config_hash(job)
|
config_hash_value = compute_config_hash(job)
|
||||||
@@ -75,6 +90,7 @@ class TranslationPlanner:
|
|||||||
"upsert_strategy": job.upsert_strategy,
|
"upsert_strategy": job.upsert_strategy,
|
||||||
"dictionary_ids": dict_hash_value,
|
"dictionary_ids": dict_hash_value,
|
||||||
"full_translation": full_translation,
|
"full_translation": full_translation,
|
||||||
|
"language_detection": language_detection,
|
||||||
}
|
}
|
||||||
|
|
||||||
key_hash_input = json.dumps({
|
key_hash_input = json.dumps({
|
||||||
|
|||||||
126
backend/src/plugins/translate/run_preflight.py
Normal file
126
backend/src/plugins/translate/run_preflight.py
Normal file
@@ -0,0 +1,126 @@
|
|||||||
|
# #region TranslationRunPreflight [C:4] [TYPE Class] [SEMANTICS translate,preflight,scope,lingua,cost]
|
||||||
|
# @BRIEF Calculate the real translation scope, token cost, and local language distribution before a run.
|
||||||
|
# @LAYER Domain
|
||||||
|
# @INVARIANT The incremental preflight applies the same new-key filter as execution.
|
||||||
|
# @INVARIANT Source texts are never returned by this service; only aggregate language counts leave the backend.
|
||||||
|
# @RELATION DEPENDS_ON -> [RunSourceFetcher]
|
||||||
|
# @RELATION DEPENDS_ON -> [LanguageDetectService]
|
||||||
|
# @RELATION DEPENDS_ON -> [TranslationPlanner]
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from ...core.config_manager import ConfigManager
|
||||||
|
from ...core.logger import belief_scope
|
||||||
|
from ...models.translate import TranslationJob
|
||||||
|
from ._lang_detect import batch_detect
|
||||||
|
from ._run_service import RunExecutionService
|
||||||
|
from ._run_source import fetch_source_rows
|
||||||
|
from ._utils import estimate_row_tokens
|
||||||
|
from .orchestrator_config import compute_config_hash
|
||||||
|
from .preview_token_estimator import TokenEstimator
|
||||||
|
|
||||||
|
LINGUA_MAX_SECONDS_PER_1000_ROWS = 2.0
|
||||||
|
|
||||||
|
|
||||||
|
class TranslationRunPreflight:
|
||||||
|
"""Calculate a non-persisted, current-state execution estimate for a job."""
|
||||||
|
|
||||||
|
def __init__(self, db: Session, config_manager: ConfigManager) -> None:
|
||||||
|
self.db = db
|
||||||
|
self.config_manager = config_manager
|
||||||
|
|
||||||
|
# #region TranslationRunPreflight.calculate [C:4] [TYPE Function] [SEMANTICS translate,preflight,calculate]
|
||||||
|
# @BRIEF Fetch real source rows, apply run-mode scope, and benchmark Lingua before a translation run.
|
||||||
|
# @PRE Job exists and is runnable enough to fetch source rows.
|
||||||
|
# @POST Returns only aggregate execution evidence; does not create TranslationRun or call an LLM.
|
||||||
|
# @SIDE_EFFECT Reads source data through Superset and performs CPU-bound local detection.
|
||||||
|
async def calculate(self, job_id: str, full_translation: bool) -> dict[str, Any]:
|
||||||
|
with belief_scope("TranslationRunPreflight.calculate"):
|
||||||
|
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
|
||||||
|
if not job:
|
||||||
|
raise ValueError(f"Translation job '{job_id}' not found")
|
||||||
|
|
||||||
|
source_rows = await fetch_source_rows(
|
||||||
|
self.db, self.config_manager, job_id, f"preflight-{uuid.uuid4()}"
|
||||||
|
)
|
||||||
|
total_source_rows = len(source_rows)
|
||||||
|
eligible_rows = source_rows
|
||||||
|
if not full_translation:
|
||||||
|
eligible_rows = RunExecutionService(
|
||||||
|
self.db, self.config_manager
|
||||||
|
)._filter_new_keys(job, "preflight", source_rows)
|
||||||
|
|
||||||
|
target_languages = job.target_languages or [job.target_dialect or "en"]
|
||||||
|
if not isinstance(target_languages, list):
|
||||||
|
target_languages = [str(target_languages)]
|
||||||
|
|
||||||
|
source_tokens = sum(
|
||||||
|
estimate_row_tokens(
|
||||||
|
str(row.get("source_text", "")), row.get("source_data"), job
|
||||||
|
)
|
||||||
|
for row in eligible_rows
|
||||||
|
)
|
||||||
|
output_tokens = TokenEstimator.estimate_output_tokens(
|
||||||
|
len(eligible_rows), len(target_languages)
|
||||||
|
)
|
||||||
|
estimated_tokens = source_tokens + output_tokens
|
||||||
|
|
||||||
|
distribution, duration_ms = await self._detect_distribution(
|
||||||
|
eligible_rows, target_languages
|
||||||
|
)
|
||||||
|
rows_per_second = (
|
||||||
|
round(len(eligible_rows) / (duration_ms / 1000), 2)
|
||||||
|
if duration_ms > 0 else 0.0
|
||||||
|
)
|
||||||
|
seconds_per_1000 = (
|
||||||
|
round((duration_ms / 1000) * 1000 / len(eligible_rows), 3)
|
||||||
|
if eligible_rows else 0.0
|
||||||
|
)
|
||||||
|
# A short run is dominated by detector warm-up/dispatch overhead. The SLA is
|
||||||
|
# explicitly a 1,000-row throughput criterion, so do not discard useful
|
||||||
|
# language evidence for smaller scopes because of extrapolation noise.
|
||||||
|
lingua_accepted = (
|
||||||
|
len(eligible_rows) < 1000
|
||||||
|
or seconds_per_1000 <= LINGUA_MAX_SECONDS_PER_1000_ROWS
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"config_hash": compute_config_hash(job),
|
||||||
|
"full_translation": full_translation,
|
||||||
|
"total_source_rows": total_source_rows,
|
||||||
|
"eligible_rows": len(eligible_rows),
|
||||||
|
"skipped_rows": total_source_rows - len(eligible_rows),
|
||||||
|
"target_language_count": len(target_languages),
|
||||||
|
"estimated_tokens": estimated_tokens,
|
||||||
|
"estimated_cost": TokenEstimator.estimate_cost(estimated_tokens),
|
||||||
|
"language_distribution": distribution if lingua_accepted else None,
|
||||||
|
"lingua_duration_ms": duration_ms,
|
||||||
|
"lingua_rows_per_second": rows_per_second,
|
||||||
|
"lingua_seconds_per_1000": seconds_per_1000,
|
||||||
|
"lingua_accepted": lingua_accepted,
|
||||||
|
"recommended_language_detection": "auto" if lingua_accepted else "skip",
|
||||||
|
}
|
||||||
|
|
||||||
|
# #endregion TranslationRunPreflight.calculate
|
||||||
|
|
||||||
|
async def _detect_distribution(
|
||||||
|
self, rows: list[dict[str, Any]], target_languages: list[str]
|
||||||
|
) -> tuple[dict[str, int], int]:
|
||||||
|
"""Run Lingua off the event loop and return aggregate counts with elapsed time."""
|
||||||
|
texts = [str(row.get("source_text", "")) for row in rows]
|
||||||
|
started = time.perf_counter()
|
||||||
|
detected = await asyncio.to_thread(batch_detect, texts, target_languages)
|
||||||
|
duration_ms = int((time.perf_counter() - started) * 1000)
|
||||||
|
distribution: dict[str, int] = {}
|
||||||
|
for language in detected:
|
||||||
|
distribution[language] = distribution.get(language, 0) + 1
|
||||||
|
return dict(sorted(distribution.items())), duration_ms
|
||||||
|
|
||||||
|
|
||||||
|
# #endregion TranslationRunPreflight
|
||||||
@@ -294,6 +294,26 @@ class PreviewRequest(BaseModel):
|
|||||||
# #endregion PreviewRequest
|
# #endregion PreviewRequest
|
||||||
|
|
||||||
|
|
||||||
|
# #region RunPreflightResponse [C:1] [TYPE Class]
|
||||||
|
# @BRIEF Response for the non-persisted run scope and Lingua performance calculation.
|
||||||
|
class RunPreflightResponse(BaseModel):
|
||||||
|
config_hash: str
|
||||||
|
full_translation: bool
|
||||||
|
total_source_rows: int
|
||||||
|
eligible_rows: int
|
||||||
|
skipped_rows: int
|
||||||
|
target_language_count: int
|
||||||
|
estimated_tokens: int
|
||||||
|
estimated_cost: float
|
||||||
|
language_distribution: dict[str, int] | None = None
|
||||||
|
lingua_duration_ms: int
|
||||||
|
lingua_rows_per_second: float
|
||||||
|
lingua_seconds_per_1000: float
|
||||||
|
lingua_accepted: bool
|
||||||
|
recommended_language_detection: str
|
||||||
|
# #endregion RunPreflightResponse
|
||||||
|
|
||||||
|
|
||||||
# #region PreviewRowUpdate [TYPE Class]
|
# #region PreviewRowUpdate [TYPE Class]
|
||||||
# @defgroup Schemas Module group.
|
# @defgroup Schemas Module group.
|
||||||
# @BRIEF Schema for approving/editing/rejecting a preview row.
|
# @BRIEF Schema for approving/editing/rejecting a preview row.
|
||||||
@@ -723,7 +743,7 @@ class InlineCorrectionSubmit(BaseModel):
|
|||||||
# @BRIEF Schema for requesting target table schema validation.
|
# @BRIEF Schema for requesting target table schema validation.
|
||||||
class TargetSchemaValidationRequest(BaseModel):
|
class TargetSchemaValidationRequest(BaseModel):
|
||||||
environment_id: str = Field(..., description="Superset environment ID")
|
environment_id: str = Field(..., description="Superset environment ID")
|
||||||
target_database_id: str = Field(..., description="Superset database ID for the target DB (SQL Lab)")
|
target_database_id: str | None = Field(None, description="Superset database ID for the target DB (SQL Lab)")
|
||||||
target_schema: str = Field("public", description="Target table schema (default: public)")
|
target_schema: str = Field("public", description="Target table schema (default: public)")
|
||||||
target_table: str = Field(..., description="Target table name")
|
target_table: str = Field(..., description="Target table name")
|
||||||
# Column mapping from job config (everything that affects expected columns)
|
# Column mapping from job config (everything that affects expected columns)
|
||||||
@@ -736,6 +756,15 @@ class TargetSchemaValidationRequest(BaseModel):
|
|||||||
# Direct DB support — when insert_method=direct_db, use connection_id instead of target_database_id
|
# Direct DB support — when insert_method=direct_db, use connection_id instead of target_database_id
|
||||||
insert_method: str | None = Field(None, description="Insert method: 'sqllab' or 'direct_db'")
|
insert_method: str | None = Field(None, description="Insert method: 'sqllab' or 'direct_db'")
|
||||||
connection_id: str | None = Field(None, description="Direct DB connection ID (required when insert_method=direct_db)")
|
connection_id: str | None = Field(None, description="Direct DB connection ID (required when insert_method=direct_db)")
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_backend_target(self):
|
||||||
|
if self.insert_method == "direct_db":
|
||||||
|
if not self.connection_id:
|
||||||
|
raise ValueError("connection_id is required when insert_method=direct_db")
|
||||||
|
elif not self.target_database_id:
|
||||||
|
raise ValueError("target_database_id is required when insert_method=sqllab")
|
||||||
|
return self
|
||||||
# #endregion TargetSchemaValidationRequest
|
# #endregion TargetSchemaValidationRequest
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from unittest.mock import MagicMock, patch
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from src.plugins.translate.orchestrator_planner import TranslationPlanner
|
from src.plugins.translate.orchestrator_planner import TranslationPlanner
|
||||||
|
from src.models.translate import TranslationRun
|
||||||
|
|
||||||
|
|
||||||
class TestTranslationPlanner:
|
class TestTranslationPlanner:
|
||||||
@@ -77,11 +78,46 @@ class TestTranslationPlanner:
|
|||||||
assert result.config_snapshot["target_dialect"] == "clickhouse"
|
assert result.config_snapshot["target_dialect"] == "clickhouse"
|
||||||
assert result.config_snapshot["batch_size"] == 50
|
assert result.config_snapshot["batch_size"] == 50
|
||||||
assert result.config_snapshot["full_translation"] is False
|
assert result.config_snapshot["full_translation"] is False
|
||||||
|
assert result.config_snapshot["language_detection"] == "auto"
|
||||||
db.add.assert_called_once_with(result)
|
db.add.assert_called_once_with(result)
|
||||||
db.flush.assert_called()
|
db.flush.assert_called()
|
||||||
db.commit.assert_called()
|
db.commit.assert_called()
|
||||||
db.refresh.assert_called_once_with(result)
|
db.refresh.assert_called_once_with(result)
|
||||||
|
|
||||||
|
@patch("src.plugins.translate.orchestrator_planner.compute_config_hash", return_value="hash1")
|
||||||
|
@patch("src.plugins.translate.orchestrator_planner.compute_dict_snapshot_hash", return_value="hash2")
|
||||||
|
@patch("src.plugins.translate.orchestrator_planner.validate_job_preconditions")
|
||||||
|
def test_plan_run_snapshots_lingua_fallback(self, _validate, _dict_hash, _config_hash):
|
||||||
|
"""A preflight-selected Lingua fallback is immutable for the created run."""
|
||||||
|
db = MagicMock()
|
||||||
|
job = self._make_job()
|
||||||
|
db.query.return_value.filter.return_value.first.return_value = job
|
||||||
|
|
||||||
|
result = TranslationPlanner(db, MagicMock(), "user").plan_run(
|
||||||
|
"job-1", language_detection="skip"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.config_snapshot["language_detection"] == "skip"
|
||||||
|
|
||||||
|
@patch("src.plugins.translate.orchestrator_planner.validate_job_preconditions")
|
||||||
|
def test_active_run_blocks_duplicate_manual_trigger(self, mock_validate):
|
||||||
|
"""An existing PENDING/RUNNING run blocks a second run for the same job."""
|
||||||
|
db = MagicMock()
|
||||||
|
event_log = MagicMock()
|
||||||
|
job = self._make_job()
|
||||||
|
active = TranslationRun(id="run-active", job_id="job-1", status="RUNNING")
|
||||||
|
job_query = MagicMock()
|
||||||
|
job_query.filter.return_value.first.return_value = job
|
||||||
|
active_query = MagicMock()
|
||||||
|
active_query.filter.return_value.first.return_value = active
|
||||||
|
db.query.side_effect = [job_query, active_query]
|
||||||
|
|
||||||
|
planner = TranslationPlanner(db, event_log, "user")
|
||||||
|
with pytest.raises(ValueError, match="already has an active run"):
|
||||||
|
planner.plan_run("job-1")
|
||||||
|
|
||||||
|
mock_validate.assert_not_called()
|
||||||
|
|
||||||
@patch("src.plugins.translate.orchestrator_planner.compute_config_hash")
|
@patch("src.plugins.translate.orchestrator_planner.compute_config_hash")
|
||||||
@patch("src.plugins.translate.orchestrator_planner.compute_dict_snapshot_hash")
|
@patch("src.plugins.translate.orchestrator_planner.compute_dict_snapshot_hash")
|
||||||
@patch("src.plugins.translate.orchestrator_planner.validate_job_preconditions")
|
@patch("src.plugins.translate.orchestrator_planner.validate_job_preconditions")
|
||||||
|
|||||||
66
backend/tests/plugins/translate/test_run_preflight.py
Normal file
66
backend/tests/plugins/translate/test_run_preflight.py
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
# #region Test.Translate.RunPreflight [C:3] [TYPE Module] [SEMANTICS test,translate,preflight,lingua]
|
||||||
|
# @BRIEF Verify run preflight returns aggregate scope only and applies the Lingua SLA fallback.
|
||||||
|
# @RELATION BINDS_TO -> [TranslationRunPreflight]
|
||||||
|
# @TEST_EDGE: missing_job -> ValueError
|
||||||
|
# @TEST_EDGE: slow_lingua -> distribution omitted and skip recommended
|
||||||
|
# @TEST_EDGE: incremental_scope -> existing rows excluded before cost calculation
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from src.plugins.translate.run_preflight import TranslationRunPreflight
|
||||||
|
|
||||||
|
|
||||||
|
def _job():
|
||||||
|
return SimpleNamespace(
|
||||||
|
id="job-1", source_dialect="postgres", target_dialect="en",
|
||||||
|
source_datasource_id="ds-1", translation_column="text", context_columns=[],
|
||||||
|
target_languages=["en", "ru"], provider_id="provider-1", batch_size=50,
|
||||||
|
upsert_strategy="MERGE", source_key_cols=["id"], target_key_cols=["id"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _service(job=None):
|
||||||
|
db = MagicMock()
|
||||||
|
db.query.return_value.filter.return_value.first.return_value = job
|
||||||
|
return TranslationRunPreflight(db, MagicMock())
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_preflight_returns_aggregate_scope_and_lingua_distribution():
|
||||||
|
service = _service(_job())
|
||||||
|
rows = [
|
||||||
|
{"source_text": "Hello", "source_data": {"id": 1}},
|
||||||
|
{"source_text": "Привет", "source_data": {"id": 2}},
|
||||||
|
]
|
||||||
|
with patch("src.plugins.translate.run_preflight.fetch_source_rows", new=AsyncMock(return_value=rows)), \
|
||||||
|
patch.object(service, "_detect_distribution", new=AsyncMock(return_value=({"en": 1, "ru": 1}, 10))), \
|
||||||
|
patch("src.plugins.translate.run_preflight.compute_config_hash", return_value="cfg-1"):
|
||||||
|
result = await service.calculate("job-1", full_translation=True)
|
||||||
|
|
||||||
|
assert result["total_source_rows"] == 2
|
||||||
|
assert result["eligible_rows"] == 2
|
||||||
|
assert result["language_distribution"] == {"en": 1, "ru": 1}
|
||||||
|
assert result["lingua_accepted"] is True
|
||||||
|
assert result["recommended_language_detection"] == "auto"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_preflight_hides_distribution_when_lingua_misses_sla():
|
||||||
|
service = _service(_job())
|
||||||
|
rows = [{"source_text": "Hello", "source_data": {"id": i}} for i in range(1000)]
|
||||||
|
with patch("src.plugins.translate.run_preflight.fetch_source_rows", new=AsyncMock(return_value=rows)), \
|
||||||
|
patch.object(service, "_detect_distribution", new=AsyncMock(return_value=({"en": 1}, 3_000))):
|
||||||
|
result = await service.calculate("job-1", full_translation=True)
|
||||||
|
|
||||||
|
assert result["language_distribution"] is None
|
||||||
|
assert result["lingua_accepted"] is False
|
||||||
|
assert result["recommended_language_detection"] == "skip"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_preflight_rejects_unknown_job():
|
||||||
|
with pytest.raises(ValueError, match="job-404"):
|
||||||
|
await _service(None).calculate("job-404", full_translation=False)
|
||||||
@@ -8,9 +8,9 @@
|
|||||||
<!-- @RELATION DEPENDS_ON -> [TranslationRunResult] -->
|
<!-- @RELATION DEPENDS_ON -> [TranslationRunResult] -->
|
||||||
<!-- @RELATION DEPENDS_ON -> [TranslateMetricsRoutesModule] -->
|
<!-- @RELATION DEPENDS_ON -> [TranslateMetricsRoutesModule] -->
|
||||||
<!-- @RELATION BINDS_TO -> [translationRunStore] -->
|
<!-- @RELATION BINDS_TO -> [translationRunStore] -->
|
||||||
<!-- @UX_STATE Idle — no run active, run cards visible -->
|
<!-- @UX_STATE Idle — run estimate and launch choices are the only primary content. -->
|
||||||
<!-- @UX_STATE Running — TranslationRunProgress bar visible, cards show active state -->
|
<!-- @UX_STATE Running — TranslationRunProgress is the sole primary content; setup and history are hidden. -->
|
||||||
<!-- @UX_STATE Completed — run history list with expandable details, summary metrics bar -->
|
<!-- @UX_STATE Completed — result remains primary; history and cumulative metrics are collapsed secondary content. -->
|
||||||
<!-- @UX_STATE Error — error banner above run history -->
|
<!-- @UX_STATE Error — error banner above run history -->
|
||||||
<!-- @UX_FEEDBACK Toast on status change / run error / run complete -->
|
<!-- @UX_FEEDBACK Toast on status change / run error / run complete -->
|
||||||
<!-- @UX_FEEDBACK BulkReplace modal button in recent runs header -->
|
<!-- @UX_FEEDBACK BulkReplace modal button in recent runs header -->
|
||||||
@@ -61,6 +61,7 @@
|
|||||||
let metricsLoading = $state(false);
|
let metricsLoading = $state(false);
|
||||||
let metricsError = $state(null);
|
let metricsError = $state(null);
|
||||||
let wasRunActive = false;
|
let wasRunActive = false;
|
||||||
|
let secondaryDetailsOpen = $state(false);
|
||||||
|
|
||||||
// Confirmation dialog state
|
// Confirmation dialog state
|
||||||
let showConfirmDialog = $state(false);
|
let showConfirmDialog = $state(false);
|
||||||
@@ -124,6 +125,7 @@
|
|||||||
});
|
});
|
||||||
|
|
||||||
$effect(() => {
|
$effect(() => {
|
||||||
|
if (isRunActive) secondaryDetailsOpen = false;
|
||||||
if (wasRunActive && !isRunActive) loadJobMetrics();
|
if (wasRunActive && !isRunActive) loadJobMetrics();
|
||||||
wasRunActive = isRunActive;
|
wasRunActive = isRunActive;
|
||||||
});
|
});
|
||||||
@@ -199,12 +201,13 @@
|
|||||||
<!-- Status display + transition -->
|
<!-- Status display + transition -->
|
||||||
<div class="flex items-center gap-3 mb-4 p-3 bg-surface-muted rounded-lg">
|
<div class="flex items-center gap-3 mb-4 p-3 bg-surface-muted rounded-lg">
|
||||||
<span class="text-sm text-text-muted">{_t.translate?.config?.status}:</span>
|
<span class="text-sm text-text-muted">{_t.translate?.config?.status}:</span>
|
||||||
<span class="inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium {statusClass}">
|
<span class="inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium {isRunActive ? 'bg-primary-light text-primary' : statusClass}">
|
||||||
{getJobStatusLabel(status)}
|
{isRunActive ? (_t.translate?.config?.running || 'Running') : getJobStatusLabel(status)}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="space-y-4">
|
<div class="space-y-4">
|
||||||
|
{#if !isRunActive}
|
||||||
<!-- Run evidence: real source scope, local Lingua distribution, and cost before execution -->
|
<!-- Run evidence: real source scope, local Lingua distribution, and cost before execution -->
|
||||||
<div class="border border-border rounded-lg bg-surface-card p-4" aria-busy={preflightLoading}>
|
<div class="border border-border rounded-lg bg-surface-card p-4" aria-busy={preflightLoading}>
|
||||||
<div class="flex flex-wrap items-center justify-between gap-3">
|
<div class="flex flex-wrap items-center justify-between gap-3">
|
||||||
@@ -303,6 +306,7 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
{#if runError}
|
{#if runError}
|
||||||
<div class="bg-destructive-light border border-destructive-light rounded-lg p-3">
|
<div class="bg-destructive-light border border-destructive-light rounded-lg p-3">
|
||||||
@@ -319,6 +323,24 @@
|
|||||||
/>
|
/>
|
||||||
{/if}
|
{/if}
|
||||||
|
|
||||||
|
{#if !isRunActive}
|
||||||
|
<!-- Secondary context: it must not compete with the next launch or an active run. -->
|
||||||
|
<div class="rounded-lg border border-border bg-surface-card">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onclick={() => secondaryDetailsOpen = !secondaryDetailsOpen}
|
||||||
|
aria-expanded={secondaryDetailsOpen}
|
||||||
|
class="flex w-full items-center justify-between gap-3 px-4 py-3 text-left text-sm font-medium text-text hover:bg-surface-muted"
|
||||||
|
>
|
||||||
|
<span>{_t.translate?.run?.secondary_details || 'History and job metrics'}</span>
|
||||||
|
<span class="text-xs font-normal text-text-muted">
|
||||||
|
{secondaryDetailsOpen
|
||||||
|
? (_t.translate?.run?.hide_details || 'Hide details')
|
||||||
|
: (_t.translate?.run?.show_details || 'Show details')}
|
||||||
|
</span>
|
||||||
|
</button>
|
||||||
|
{#if secondaryDetailsOpen}
|
||||||
|
<div class="border-t border-border p-4 space-y-4">
|
||||||
<!-- Summary metrics bar -->
|
<!-- Summary metrics bar -->
|
||||||
{#if jobMetrics}
|
{#if jobMetrics}
|
||||||
<div class="bg-surface-card border border-border rounded-lg p-4">
|
<div class="bg-surface-card border border-border rounded-lg p-4">
|
||||||
@@ -401,7 +423,7 @@
|
|||||||
{/if}
|
{/if}
|
||||||
|
|
||||||
{#if completedRuns.length > 0}
|
{#if completedRuns.length > 0}
|
||||||
<div class="mt-4">
|
<div>
|
||||||
<div class="flex items-center justify-between mb-2">
|
<div class="flex items-center justify-between mb-2">
|
||||||
<h4 class="text-sm font-medium text-text">{_t.translate?.config?.recent_runs}</h4>
|
<h4 class="text-sm font-medium text-text">{_t.translate?.config?.recent_runs}</h4>
|
||||||
<button
|
<button
|
||||||
@@ -489,6 +511,10 @@
|
|||||||
{/if}
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
{/if}
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|||||||
@@ -527,6 +527,7 @@
|
|||||||
"preflight_rows_sec": "rows/s",
|
"preflight_rows_sec": "rows/s",
|
||||||
"preflight_languages": "Source language distribution",
|
"preflight_languages": "Source language distribution",
|
||||||
"preflight_lingua_slow": "Lingua exceeded the performance threshold. The run will continue without local language detection.",
|
"preflight_lingua_slow": "Lingua exceeded the performance threshold. The run will continue without local language detection.",
|
||||||
|
"secondary_details": "History and job metrics",
|
||||||
"loading": "Loading run status...",
|
"loading": "Loading run status...",
|
||||||
"loading_result": "Loading result...",
|
"loading_result": "Loading result...",
|
||||||
"result_title": "Run Result",
|
"result_title": "Run Result",
|
||||||
|
|||||||
@@ -528,6 +528,7 @@
|
|||||||
"preflight_rows_sec": "строк/с",
|
"preflight_rows_sec": "строк/с",
|
||||||
"preflight_languages": "Распределение исходных языков",
|
"preflight_languages": "Распределение исходных языков",
|
||||||
"preflight_lingua_slow": "Lingua превысила порог производительности. Перевод продолжится без локального определения языка.",
|
"preflight_lingua_slow": "Lingua превысила порог производительности. Перевод продолжится без локального определения языка.",
|
||||||
|
"secondary_details": "История и метрики задания",
|
||||||
"loading": "Загрузка статуса запуска...",
|
"loading": "Загрузка статуса запуска...",
|
||||||
"loading_result": "Загрузка результата...",
|
"loading_result": "Загрузка результата...",
|
||||||
"result_title": "Результат запуска",
|
"result_title": "Результат запуска",
|
||||||
|
|||||||
Reference in New Issue
Block a user