From 28b28338c55bceb1dca5f3890f264991594e35ae Mon Sep 17 00:00:00 2001 From: busya Date: Mon, 1 Jun 2026 22:38:52 +0300 Subject: [PATCH] =?UTF-8?q?fix(translate):=20=D0=B8=D1=81=D0=BF=D1=80?= =?UTF-8?q?=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=D0=B8=D0=B5=20INSERT=20+=20?= =?UTF-8?q?=D0=B2=D1=8B=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D0=B0=20CSV=20=D0=B4?= =?UTF-8?q?=D0=BB=D1=8F=20=D0=BD=D0=B5=D1=83=D1=81=D0=BF=D0=B5=D1=88=D0=BD?= =?UTF-8?q?=D1=8B=D1=85=20=D0=B7=D0=B0=D0=BF=D1=83=D1=81=D0=BA=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Исправления ошибок:** - superset_executor: убран несуществующий fallback endpoint /api/v1/sql_lab/execute/ (правильный URL /api/v1/sqllab/execute/ подтверждён браузерным запросом Superset) Ошибка 500 (permission denied) больше не маскируется 404 - orchestrator_run_completion: run.status=FAILED при провале insert (раньше всегда ставился COMPLETED). timeout тоже treated as failure - test_orchestrator: assertion обновлён COMPLETED→FAILED **Новая функция — выгрузка CSV:** - GET /api/translate/runs/{run_id}/failed.csv — возвращает CSV с успешно переведёнными, но не вставленными записями (status=SUCCESS) - Колонки: record_id, source_*, target_sql, source_data (JSON), языки, error_message - Метаданные запуска (run_status, insert_status, run_error) в конце файла - Кнопки скачивания в 3 местах фронта: - История запусков (при status=FAILED) - TranslationRunProgress (insert_failed) - TranslationRunResult (failed/partial/insert_failed) - downloadFailedCsv() через fetchApiBlob + Blob URL --- .../api/routes/translate/_run_list_routes.py | 133 ++++++++++++++++++ .../translate/__tests__/test_orchestrator.py | 4 +- .../translate/orchestrator_run_completion.py | 12 +- .../plugins/translate/superset_executor.py | 54 ++++--- frontend/src/lib/api/translate.ts | 2 +- frontend/src/lib/api/translate/corrections.ts | 24 ++++ .../translate/TranslationRunProgress.svelte | 25 +++- .../translate/TranslationRunResult.svelte | 15 +- .../src/routes/translate/history/+page.svelte | 17 +++ 9 files changed, 243 insertions(+), 43 deletions(-) diff --git a/backend/src/api/routes/translate/_run_list_routes.py b/backend/src/api/routes/translate/_run_list_routes.py index e6aa9b46..f240ea9e 100644 --- a/backend/src/api/routes/translate/_run_list_routes.py +++ b/backend/src/api/routes/translate/_run_list_routes.py @@ -222,4 +222,137 @@ async def download_skipped_csv( raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) # #endregion download_skipped_csv + +# ============================================================ +# Download Failed CSV (uninserted records from failed runs) +# ============================================================ + +# #region download_failed_csv [C:4] [TYPE Function] +# @BRIEF Download a CSV of successfully translated but uninserted records from a FAILED run. +# @PRE User has translate.job.view permission. Run must have insert_status=failed or status=FAILED. +# @POST Returns CSV file of records that were translated but not inserted into target DB. +# @RATIONALE When Superset SQL Lab INSERT fails (e.g., permission denied), users need to +# recover the translated data. This CSV lets them manually import or retry. +# @REJECTED Exporting ALL records from a failed run was rejected — only SUCCESS records +# (those with valid translations) are useful for recovery. + +@router.get("/runs/{run_id}/failed.csv") +async def download_failed_csv( + run_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "VIEW")), + db: Session = Depends(get_db), +): + """Download successfully translated but uninserted records as CSV (for failed runs).""" + logger.reason(f"download_failed_csv — Run: {run_id}, User: {current_user.username}", + extra={"src": "translate_routes"}) + try: + import csv + import io + import json + + from fastapi.responses import StreamingResponse + from sqlalchemy.orm import joinedload + + from ....models.translate import TranslationRecord, TranslationRun + + run = db.query(TranslationRun).filter(TranslationRun.id == run_id).first() + if not run: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Run not found") + + # Export only SUCCESS records — these are the ones that were translated + # but could not be inserted into the target database. + records = ( + db.query(TranslationRecord) + .options(joinedload(TranslationRecord.languages)) + .filter( + TranslationRecord.run_id == run_id, + TranslationRecord.status == "SUCCESS", + ) + .order_by(TranslationRecord.created_at) + .all() + ) + + if not records: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="No successful records found for this run", + ) + + # Collect all language codes across records for CSV header + language_codes: list[str] = [] + seen_codes: set[str] = set() + for rec in records: + for lang in (rec.languages or []): + if lang.language_code and lang.language_code not in seen_codes: + language_codes.append(lang.language_code) + seen_codes.add(lang.language_code) + + output = io.StringIO() + writer = csv.writer(output) + + # Header: identity + source + target + per-language + metadata + header = [ + "record_id", "source_object_type", "source_object_id", + "source_object_name", "source_sql", "target_sql", + "source_data", + ] + header.extend(language_codes) + header.append("error_message") + writer.writerow(header) + + for rec in records: + # Build per-language values map + lang_values: dict[str, str] = {} + for lang in (rec.languages or []): + if lang.language_code: + lang_values[lang.language_code] = lang.final_value or lang.translated_value or "" + + # Flatten source_data JSON to string for CSV compatibility + source_data_str = "" + if rec.source_data: + try: + source_data_str = json.dumps(rec.source_data, ensure_ascii=False) + except (TypeError, ValueError): + source_data_str = str(rec.source_data) + + row = [ + rec.id, + rec.source_object_type, + rec.source_object_id, + rec.source_object_name, + rec.source_sql, + rec.target_sql, + source_data_str, + ] + row.extend(lang_values.get(code, "") for code in language_codes) + row.append(rec.error_message or "") + writer.writerow(row) + + # Append run-level failure metadata as a comment row + writer.writerow([]) + writer.writerow(["# Run failure metadata:"]) + writer.writerow(["run_id", run.id]) + writer.writerow(["run_status", run.status]) + writer.writerow(["insert_status", run.insert_status or ""]) + writer.writerow(["run_error", run.error_message or ""]) + writer.writerow(["total_records", run.total_records or 0]) + writer.writerow(["successful_records", run.successful_records or 0]) + + output.seek(0) + return StreamingResponse( + iter([output.getvalue()]), + media_type="text/csv", + headers={ + "Content-Disposition": f"attachment; filename=failed_{run_id[:8]}.csv" + }, + ) + except HTTPException: + raise + except Exception as e: + logger.explore("download_failed_csv failed", + extra={"src": "translate_routes", "error": str(e)}) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# #endregion download_failed_csv + # #endregion TranslateRunListRoutesModule diff --git a/backend/src/plugins/translate/__tests__/test_orchestrator.py b/backend/src/plugins/translate/__tests__/test_orchestrator.py index 7d126f5c..dedb99ed 100644 --- a/backend/src/plugins/translate/__tests__/test_orchestrator.py +++ b/backend/src/plugins/translate/__tests__/test_orchestrator.py @@ -420,7 +420,7 @@ class TestTranslationOrchestrator: # endregion test_execute_run_no_job # region test_execute_run_insert_failure [TYPE Function] - # @BRIEF SQL generation/insert returns failure, run still completes but with error logged. + # @BRIEF SQL generation/insert returns failure — run marked FAILED with error logged. def test_execute_run_insert_failure(self, mock_job: MagicMock) -> None: db = MagicMock() config_manager = MagicMock() @@ -464,7 +464,7 @@ class TestTranslationOrchestrator: patch.object(engine, 'event_log'): result = orch.execute_run(run) - assert result.status == "COMPLETED" + assert result.status == "FAILED" assert result.insert_status == "failed" assert "timeout" in (result.error_message or "") diff --git a/backend/src/plugins/translate/orchestrator_run_completion.py b/backend/src/plugins/translate/orchestrator_run_completion.py index 6f25fe6e..5c52e6af 100644 --- a/backend/src/plugins/translate/orchestrator_run_completion.py +++ b/backend/src/plugins/translate/orchestrator_run_completion.py @@ -106,9 +106,17 @@ def complete_success( run.insert_status = insert_result.get("status") run.superset_execution_id = str(insert_result.get("query_id") or "") run.superset_execution_log = insert_result - if run.status != "FAILED": + # Treat "failed" and "timeout" as terminal failures. + # "skipped" (no records) and "success" are non-failure outcomes. + if insert_result.get("status") in ("failed", "timeout"): + run.status = "FAILED" + if insert_result.get("error_message"): + run.error_message = insert_result["error_message"] + elif not run.error_message: + run.error_message = f"SQL insert phase {insert_result.get('status')}" + elif run.status != "FAILED": run.status = "COMPLETED" - if insert_result.get("error_message"): + if insert_result.get("error_message") and not run.error_message: run.error_message = insert_result["error_message"] run.completed_at = datetime.now(UTC) db.flush() diff --git a/backend/src/plugins/translate/superset_executor.py b/backend/src/plugins/translate/superset_executor.py index 0e13f573..6b18ec73 100644 --- a/backend/src/plugins/translate/superset_executor.py +++ b/backend/src/plugins/translate/superset_executor.py @@ -173,36 +173,30 @@ class SupersetSqlLabExecutor: "client_id": f"trl-{uuid.uuid4().hex[:7]}", # max 11 chars for Superset varchar(11) } - # Try multiple SQL Lab endpoints — some Superset versions use /sqllab/execute/, - # others use /sql_lab/execute/ (with underscore) - candidate_endpoints = ["/api/v1/sqllab/execute/", "/api/v1/sql_lab/execute/"] - response = None - last_error = None - for endpoint in candidate_endpoints: - try: - response = client.network.request( - method="POST", - endpoint=endpoint, - data=json.dumps(payload), - headers={"Content-Type": "application/json"}, - ) - if isinstance(response, dict) and response: - logger.reason("SQL Lab endpoint succeeded", extra={ - "endpoint": endpoint, - }) - break - except Exception as ep_err: - last_error = ep_err - logger.explore("SQL Lab endpoint failed, trying next", extra={ - "error": str(ep_err), - "endpoint": endpoint, - }) - response = None - - if response is None: - error_msg = f"All SQL Lab endpoints failed. Last error: {last_error}" - logger.explore("SQL Lab execute failed", extra={"error": error_msg}) - raise ValueError(f"Superset SQL Lab execute failed: {error_msg}") + # Use the canonical Superset SQL Lab execute endpoint. + # NOTE: Some older docs reference /sql_lab/execute/ (with underscore), + # but this endpoint does NOT exist in current Superset versions. + # The browser UI uses /api/v1/sqllab/execute/ — confirmed by + # fetch() calls in Superset SQL Lab. + endpoint = "/api/v1/sqllab/execute/" + try: + response = client.network.request( + method="POST", + endpoint=endpoint, + data=json.dumps(payload), + headers={"Content-Type": "application/json"}, + ) + logger.reason("SQL Lab endpoint succeeded", extra={ + "endpoint": endpoint, + }) + except Exception as ep_err: + logger.explore("SQL Lab execute failed", extra={ + "error": str(ep_err), + "endpoint": endpoint, + }) + raise ValueError( + f"Superset SQL Lab execute failed: {ep_err}" + ) from ep_err # Parse response — try multiple known Superset response formats result = response if isinstance(response, dict) else {} diff --git a/frontend/src/lib/api/translate.ts b/frontend/src/lib/api/translate.ts index e8db75f2..105b01c6 100644 --- a/frontend/src/lib/api/translate.ts +++ b/frontend/src/lib/api/translate.ts @@ -27,7 +27,7 @@ export { dictionaryApi } from './translate/dictionaries'; export { fetchSchedule, setSchedule, deleteSchedule, enableSchedule, disableSchedule, fetchNextExecutions } from './translate/schedules'; // Corrections & bulk replace -export { submitCorrection, submitBulkCorrections, inlineEditCorrection, submitCorrectionToDict, bulkFindReplace, bulkReplacePreview, downloadSkippedCsv } from './translate/corrections'; +export { submitCorrection, submitBulkCorrections, inlineEditCorrection, submitCorrectionToDict, bulkFindReplace, bulkReplacePreview, downloadSkippedCsv, downloadFailedCsv } from './translate/corrections'; // Target schema validation export { checkTargetTableSchema } from './translate/target-schema'; diff --git a/frontend/src/lib/api/translate/corrections.ts b/frontend/src/lib/api/translate/corrections.ts index 8810f236..151ca79c 100644 --- a/frontend/src/lib/api/translate/corrections.ts +++ b/frontend/src/lib/api/translate/corrections.ts @@ -134,4 +134,28 @@ export async function downloadSkippedCsv(runId: string): Promise { } } // #endregion downloadSkippedCsv + +// #region downloadFailedCsv [C:2] [TYPE Function] [SEMANTICS translate, corrections, csv, download] +// @BRIEF Download a CSV of successfully translated but uninserted records for a FAILED run. +// @PRE runId is a non-empty string. The run must have status=FAILED or insert_status=failed. +// @POST Triggers browser download of a CSV file. Returns nothing on success. +// @SIDE_EFFECT Creates a Blob URL and programmatically clicks a download anchor. +// @RATIONALE When INSERT into target DB fails, users need to recover translated data for manual import. +// @RELATION DEPENDS_ON -> [ApiModule.fetchApiBlob] +export async function downloadFailedCsv(runId: string): Promise { + try { + const blob = await api.fetchApiBlob(`/translate/runs/${runId}/failed.csv`); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `failed-${runId.substring(0, 8)}.csv`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to download failed CSV'); + } +} +// #endregion downloadFailedCsv // #endregion TranslateCorrectionsApi diff --git a/frontend/src/lib/components/translate/TranslationRunProgress.svelte b/frontend/src/lib/components/translate/TranslationRunProgress.svelte index dc1457c2..b9b2e34f 100644 --- a/frontend/src/lib/components/translate/TranslationRunProgress.svelte +++ b/frontend/src/lib/components/translate/TranslationRunProgress.svelte @@ -16,7 +16,7 @@ import { fromStore } from 'svelte/store'; import { t } from '$lib/i18n'; import { addToast } from '$lib/toasts'; - import { cancelRun } from '$lib/api/translate.js'; + import { cancelRun, downloadFailedCsv } from '$lib/api/translate.js'; import { translationRunStore } from '$lib/stores/translationRun.js'; /** @type {{ runId: string, onRetry?: () => void, onRetryInsert?: () => void, onComplete?: (statusData: any) => void }} */ @@ -203,12 +203,23 @@ ⚠ {$t.translate?.run?.insert_failed} - +
+ + +

{$t.translate?.run?.completed} ({successfulRecords}) — {$t.translate?.run?.insert_failed} diff --git a/frontend/src/lib/components/translate/TranslationRunResult.svelte b/frontend/src/lib/components/translate/TranslationRunResult.svelte index 42a60e25..86448b8b 100644 --- a/frontend/src/lib/components/translate/TranslationRunResult.svelte +++ b/frontend/src/lib/components/translate/TranslationRunResult.svelte @@ -16,7 +16,8 @@ fetchRunRecords, retryFailedBatches, retryInsert, - fetchRunBatches + fetchRunBatches, + downloadFailedCsv } from '$lib/api/translate.js'; import CorrectionCell from './CorrectionCell.svelte'; import TermCorrectionPopup from './TermCorrectionPopup.svelte'; @@ -178,6 +179,18 @@

{$t.translate?.run?.result_title}

+ + {#if (uxState === 'insert_failed' || uxState === 'failed' || uxState === 'partial') && (status.successful_records || 0) > 0} + + {/if} {#if uxState === 'failed' || uxState === 'partial'} + {/if}