From 15d47450a385bced7609cd30c9a52636e8179ec1 Mon Sep 17 00:00:00 2001 From: busya Date: Thu, 4 Jun 2026 13:23:10 +0300 Subject: [PATCH] feat: add deduplicate + metrics to Run tab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend: - deduplicate param in GET /runs/{id}/records — NOT EXISTS subquery with (created_at, id) tiebreaker for one row per source_hash - source_data and source_hash added to records JSON response - fix missing status import in _run_history_routes.py (NameError) - fix tab/space mixup in orchestrator_aggregator.py - remove duplicated @RELATION edges in metrics.py - fix #region/#endregion style and @PURPOSE→@BRIEF in metrics.py Frontend: - RunTabContent: summary metrics bar (fetchJobMetrics) with HelpTooltips - RunTabContent: trigger_type badge, duration, cache rate in run rows - TranslationRunResult: deduplicate=true by default, paginated Load more - TranslationRunResult: per-language token_count and estimated_cost - TranslationRunResult: collapsible batch breakdown with timing - TranslationRunResult: Bulk Replace button in header and records table - TranslationRunResult: source_data key values shown under source text i18n: all new keys in EN + RU (load_more_records, showing_records, sum_*, help_sum_*, trigger_*, batch_*, duration_label, cost_label, cache_rate, load_more_records) --- .../routes/translate/_run_history_routes.py | 133 +++++++++++++++ backend/src/plugins/translate/metrics.py | 7 +- backend/src/plugins/translate/orchestrator.py | 3 +- .../translate/orchestrator_aggregator.py | 3 +- .../plugins/translate/orchestrator_query.py | 41 ++++- frontend/src/lib/api/translate/runs.ts | 3 +- .../components/translate/RunTabContent.svelte | 146 +++++++++++++++- .../translate/TranslationRunResult.svelte | 159 ++++++++++++++++-- .../src/lib/i18n/locales/en/translate.json | 28 ++- .../src/lib/i18n/locales/ru/translate.json | 28 ++- 10 files changed, 519 insertions(+), 32 deletions(-) create mode 100644 backend/src/api/routes/translate/_run_history_routes.py diff --git a/backend/src/api/routes/translate/_run_history_routes.py b/backend/src/api/routes/translate/_run_history_routes.py new file mode 100644 index 00000000..177308e8 --- /dev/null +++ b/backend/src/api/routes/translate/_run_history_routes.py @@ -0,0 +1,133 @@ +# #region TranslateRunHistoryRoutesModule [C:3] [TYPE Module] [SEMANTICS fastapi, translate, api, history, status, records] +# @BRIEF Translation Run history, status, records and batches routes. +# @LAYER API +# @RELATION DEPENDS_ON -> [TranslateRunRoutesModule] +# @RELATION CALLED_BY -> [TranslateRoutes] + +from fastapi import Depends, HTTPException, Query, status as http_status +from sqlalchemy.orm import Session + +from ....core.config_manager import ConfigManager +from ....core.database import get_db +from ....core.logger import logger +from ....dependencies import get_config_manager, get_current_user, has_permission +from ....plugins.translate.orchestrator import TranslationOrchestrator +from ....schemas.auth import User +from ._helpers import _run_to_response +from ._router import router + +from datetime import datetime, UTC + + +# #region get_run_history [C:4] [TYPE Function] +# @BRIEF Get run history for a translation job. +# @PRE User has translate.history.view permission. +# @POST Returns list of runs. +@router.get("/jobs/{job_id}/runs") +def get_run_history( + job_id: str, + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.history", "VIEW")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + logger.reason(f"get_run_history — Job: {job_id}, User: {current_user.username}", extra={"src": "translate_routes"}) + try: + orch = TranslationOrchestrator(db, config_manager, current_user.username) + total, runs = orch.get_run_history(job_id, page=page, page_size=page_size) + return {"items": runs, "total": total, "page": page, "page_size": page_size} + except ValueError as e: + raise HTTPException(status_code=http_status.HTTP_404_NOT_FOUND, detail=str(e)) +# #endregion get_run_history + + +# #region get_run_status [C:4] [TYPE Function] +# @BRIEF Get status and statistics for a translation run. +# @PRE User has translate.history.view permission. +# @POST Returns run details with statistics. +@router.get("/runs/{run_id}") +def get_run_status( + run_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.history", "VIEW")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + logger.reason(f"get_run_status — Run: {run_id}, User: {current_user.username}", extra={"src": "translate_routes"}) + try: + orch = TranslationOrchestrator(db, config_manager, current_user.username) + return orch.get_run_status(run_id) + except ValueError as e: + raise HTTPException(status_code=http_status.HTTP_404_NOT_FOUND, detail=str(e)) +# #endregion get_run_status + + +# #region get_run_records [C:4] [TYPE Function] +# @BRIEF Get paginated records for a translation run. +# @PRE User has translate.history.view permission. +# @POST Returns paginated records. +@router.get("/runs/{run_id}/records") +def get_run_records( + run_id: str, + page: int = Query(1, ge=1), + page_size: int = Query(50, ge=1, le=500), + status: str | None = Query(None), + deduplicate: bool = Query(False, description="Return one row per unique source_hash (cache-like view)"), + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.history", "VIEW")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + logger.reason(f"get_run_records — Run: {run_id}, User: {current_user.username}", extra={"src": "translate_routes"}) + try: + orch = TranslationOrchestrator(db, config_manager, current_user.username) + return orch.get_run_records(run_id, page=page, page_size=page_size, status_filter=status, deduplicate=deduplicate) + except ValueError as e: + raise HTTPException(status_code=http_status.HTTP_404_NOT_FOUND, detail=str(e)) +# #endregion get_run_records + + +# #region get_batches [C:4] [TYPE Function] +# @BRIEF Get batches for a translation run. +# @PRE User has translate.job.view permission. +# @POST Returns list of batches. +@router.get("/runs/{run_id}/batches") +def get_batches( + run_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "VIEW")), + db: Session = Depends(get_db), +): + logger.reason(f"get_batches — Run: {run_id}, User: {current_user.username}", extra={"src": "translate_routes"}) + try: + from ....models.translate import TranslationBatch + batches = ( + db.query(TranslationBatch) + .filter(TranslationBatch.run_id == run_id) + .order_by(TranslationBatch.batch_index.asc()) + .all() + ) + return [ + { + "id": b.id, + "run_id": b.run_id, + "batch_index": b.batch_index, + "status": b.status, + "total_records": b.total_records or 0, + "successful_records": b.successful_records or 0, + "failed_records": b.failed_records or 0, + "started_at": b.started_at.isoformat() if b.started_at else None, + "completed_at": b.completed_at.isoformat() if b.completed_at else None, + "created_at": b.created_at.isoformat() if b.created_at else None, + } + for b in batches + ] + except Exception as e: + logger.explore("get_batches failed", extra={"src": "translate_routes", "error": str(e)}) + raise HTTPException(status_code=http_status.HTTP_400_BAD_REQUEST, detail=str(e)) +# #endregion get_batches + + +# #endregion TranslateRunHistoryRoutesModule diff --git a/backend/src/plugins/translate/metrics.py b/backend/src/plugins/translate/metrics.py index 5a9e5289..35479c11 100644 --- a/backend/src/plugins/translate/metrics.py +++ b/backend/src/plugins/translate/metrics.py @@ -2,9 +2,6 @@ # @BRIEF Aggregate translation metrics from live TranslationEvent + MetricSnapshot for per-job reporting. # @LAYER Domain # @RELATION DEPENDS_ON -> [TranslationSchedule] -# @RELATION DEPENDS_ON -> [TranslationSchedule] -# @RELATION DEPENDS_ON -> [TranslationSchedule] -# @RELATION DEPENDS_ON -> [TranslationSchedule] # @PRE Database session is open. # @POST Metrics are aggregated and returned; no side effects. # @RATIONALE Live events (<90 days) + MetricSnapshot (>=90 days) fusion for complete picture. @@ -32,8 +29,8 @@ class TranslationMetrics: def __init__(self, db: Session): self.db = db - # region get_job_metrics [TYPE Function] - # @PURPOSE Get aggregated metrics for a specific job. + #region get_job_metrics [TYPE Function] + # @BRIEF Get aggregated metrics for a specific job. # @PRE job_id exists. # @POST Returns dict with metrics from events + latest snapshot. def get_job_metrics(self, job_id: str) -> dict[str, Any]: diff --git a/backend/src/plugins/translate/orchestrator.py b/backend/src/plugins/translate/orchestrator.py index 5604bad9..01044573 100644 --- a/backend/src/plugins/translate/orchestrator.py +++ b/backend/src/plugins/translate/orchestrator.py @@ -156,8 +156,9 @@ class TranslationOrchestrator: page: int = 1, page_size: int = 50, status_filter: str | None = None, + deduplicate: bool = False, ) -> dict[str, Any]: - return self._aggregator.get_run_records(run_id, page, page_size, status_filter) + return self._aggregator.get_run_records(run_id, page, page_size, status_filter, deduplicate=deduplicate) # endregion get_run_records # region get_run_history [TYPE Function] diff --git a/backend/src/plugins/translate/orchestrator_aggregator.py b/backend/src/plugins/translate/orchestrator_aggregator.py index 1fc6d359..df8096a3 100644 --- a/backend/src/plugins/translate/orchestrator_aggregator.py +++ b/backend/src/plugins/translate/orchestrator_aggregator.py @@ -101,9 +101,10 @@ class TranslationResultAggregator: page: int = 1, page_size: int = 50, status_filter: str | None = None, + deduplicate: bool = False, ) -> dict[str, Any]: with belief_scope("TranslationResultAggregator.get_run_records"): - return _get_run_records(self.db, run_id, page, page_size, status_filter) + return _get_run_records(self.db, run_id, page, page_size, status_filter, deduplicate=deduplicate) # endregion get_run_records # region get_run_history [TYPE Function] diff --git a/backend/src/plugins/translate/orchestrator_query.py b/backend/src/plugins/translate/orchestrator_query.py index 51176392..a3b5e4f1 100644 --- a/backend/src/plugins/translate/orchestrator_query.py +++ b/backend/src/plugins/translate/orchestrator_query.py @@ -11,15 +11,18 @@ from ...models.translate import TranslationRecord, TranslationRun # #region get_run_records [C:2] [TYPE Function] [SEMANTICS records, query, paginated] -# @BRIEF Get paginated records for a run. +# @BRIEF Get paginated records for a run. Supports deduplicate=true to return unique source_hash rows. def get_run_records( db: Session, run_id: str, page: int = 1, page_size: int = 50, status_filter: str | None = None, + deduplicate: bool = False, ) -> dict[str, Any]: """Get paginated records for a run.""" + import sqlalchemy as sa + query = ( db.query(TranslationRecord) .options(selectinload(TranslationRecord.languages)) @@ -28,6 +31,34 @@ def get_run_records( if status_filter: query = query.filter(TranslationRecord.status == status_filter) + if deduplicate: + # One row per unique source_hash using NOT EXISTS on (source_hash, created_at, id). + # When created_at ties, the larger id wins. No relationship joins inside the + # correlated subquery — avoids row multiplication from languages. + RecAlias = sa.orm.aliased(TranslationRecord) + newer_exists = ( + db.query(RecAlias.id) + .filter( + RecAlias.run_id == run_id, + RecAlias.source_hash == TranslationRecord.source_hash, + RecAlias.source_hash.isnot(None), + RecAlias.source_hash != "", + sa.or_( + RecAlias.created_at > TranslationRecord.created_at, + sa.and_( + RecAlias.created_at == TranslationRecord.created_at, + RecAlias.id > TranslationRecord.id, + ), + ), + ) + ).exists() + + query = query.filter( + TranslationRecord.source_hash.isnot(None), + TranslationRecord.source_hash != "", + ~newer_exists, + ) + total = query.count() offset_val = (page - 1) * page_size records = ( @@ -47,6 +78,8 @@ def get_run_records( "source_object_type": r.source_object_type, "source_object_id": r.source_object_id, "source_object_name": r.source_object_name, + "source_data": r.source_data, + "source_hash": r.source_hash, "status": r.status, "error_message": r.error_message, "created_at": r.created_at.isoformat() if r.created_at else None, @@ -68,12 +101,12 @@ def get_run_records( "page": page, "page_size": page_size, "status_filter": status_filter, + "deduplicate": deduplicate, } # #endregion get_run_records - # #region get_run_history [C:2] [TYPE Function] [SEMANTICS history, query, paginated] -# @BRIEF Get run history for a job with pagination. +# @BRIEF Get run history for a job with pagination. Returns trigger_type, cache_hits for enhanced run list display. def get_run_history( db: Session, job_id: str, @@ -95,6 +128,7 @@ def get_run_history( "id": r.id, "job_id": r.job_id, "status": r.status, + "trigger_type": r.trigger_type, "started_at": r.started_at.isoformat() if r.started_at else None, "completed_at": r.completed_at.isoformat() if r.completed_at else None, "error_message": r.error_message, @@ -102,6 +136,7 @@ def get_run_history( "successful_records": r.successful_records or 0, "failed_records": r.failed_records or 0, "skipped_records": r.skipped_records or 0, + "cache_hits": r.cache_hits or 0, "insert_status": r.insert_status, "superset_execution_id": r.superset_execution_id, "created_by": r.created_by, diff --git a/frontend/src/lib/api/translate/runs.ts b/frontend/src/lib/api/translate/runs.ts index 8ada1665..dd4dc055 100644 --- a/frontend/src/lib/api/translate/runs.ts +++ b/frontend/src/lib/api/translate/runs.ts @@ -84,12 +84,13 @@ export async function fetchRunHistory(jobId: string, options: RunHi // @PRE runId is a non-empty string. // @POST Returns paginated record list with translations. // @RELATION DEPENDS_ON -> [fetchApi] -export async function fetchRunRecords(runId: string, options: RunQueryOptions & { status?: string } = {}): Promise { +export async function fetchRunRecords(runId: string, options: RunQueryOptions & { status?: string; deduplicate?: boolean } = {}): Promise { try { const params = new URLSearchParams(); if (options.page != null) params.append('page', String(options.page)); if (options.page_size != null) params.append('page_size', String(options.page_size)); if (options.status) params.append('status', options.status); + if (options.deduplicate != null) params.append('deduplicate', String(options.deduplicate)); const query = params.toString(); return await api.fetchApi(`/translate/runs/${runId}/records${query ? `?${query}` : ''}`); } catch (error) { diff --git a/frontend/src/lib/components/translate/RunTabContent.svelte b/frontend/src/lib/components/translate/RunTabContent.svelte index bc564a70..035ed628 100644 --- a/frontend/src/lib/components/translate/RunTabContent.svelte +++ b/frontend/src/lib/components/translate/RunTabContent.svelte @@ -1,22 +1,26 @@ - + + tracking via TranslationRunProgress, job metrics summary, and run history with + trigger type, duration, cache rate, and expandable details. --> + - + + +