fix(translate): исправление INSERT + выгрузка CSV для неуспешных запусков
**Исправления ошибок:**
- superset_executor: убран несуществующий fallback endpoint /api/v1/sql_lab/execute/
(правильный URL /api/v1/sqllab/execute/ подтверждён браузерным запросом Superset)
Ошибка 500 (permission denied) больше не маскируется 404
- orchestrator_run_completion: run.status=FAILED при провале insert
(раньше всегда ставился COMPLETED). timeout тоже treated as failure
- test_orchestrator: assertion обновлён COMPLETED→FAILED
**Новая функция — выгрузка CSV:**
- GET /api/translate/runs/{run_id}/failed.csv — возвращает CSV с успешно
переведёнными, но не вставленными записями (status=SUCCESS)
- Колонки: record_id, source_*, target_sql, source_data (JSON), языки, error_message
- Метаданные запуска (run_status, insert_status, run_error) в конце файла
- Кнопки скачивания в 3 местах фронта:
- История запусков (при status=FAILED)
- TranslationRunProgress (insert_failed)
- TranslationRunResult (failed/partial/insert_failed)
- downloadFailedCsv() через fetchApiBlob + Blob URL
This commit is contained in:
@@ -222,4 +222,137 @@ async def download_skipped_csv(
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||
# #endregion download_skipped_csv
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Download Failed CSV (uninserted records from failed runs)
|
||||
# ============================================================
|
||||
|
||||
# #region download_failed_csv [C:4] [TYPE Function]
|
||||
# @BRIEF Download a CSV of successfully translated but uninserted records from a FAILED run.
|
||||
# @PRE User has translate.job.view permission. Run must have insert_status=failed or status=FAILED.
|
||||
# @POST Returns CSV file of records that were translated but not inserted into target DB.
|
||||
# @RATIONALE When Superset SQL Lab INSERT fails (e.g., permission denied), users need to
|
||||
# recover the translated data. This CSV lets them manually import or retry.
|
||||
# @REJECTED Exporting ALL records from a failed run was rejected — only SUCCESS records
|
||||
# (those with valid translations) are useful for recovery.
|
||||
|
||||
@router.get("/runs/{run_id}/failed.csv")
|
||||
async def download_failed_csv(
|
||||
run_id: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
_ = Depends(has_permission("translate.job", "VIEW")),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Download successfully translated but uninserted records as CSV (for failed runs)."""
|
||||
logger.reason(f"download_failed_csv — Run: {run_id}, User: {current_user.username}",
|
||||
extra={"src": "translate_routes"})
|
||||
try:
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy.orm import joinedload
|
||||
|
||||
from ....models.translate import TranslationRecord, TranslationRun
|
||||
|
||||
run = db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
|
||||
if not run:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Run not found")
|
||||
|
||||
# Export only SUCCESS records — these are the ones that were translated
|
||||
# but could not be inserted into the target database.
|
||||
records = (
|
||||
db.query(TranslationRecord)
|
||||
.options(joinedload(TranslationRecord.languages))
|
||||
.filter(
|
||||
TranslationRecord.run_id == run_id,
|
||||
TranslationRecord.status == "SUCCESS",
|
||||
)
|
||||
.order_by(TranslationRecord.created_at)
|
||||
.all()
|
||||
)
|
||||
|
||||
if not records:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="No successful records found for this run",
|
||||
)
|
||||
|
||||
# Collect all language codes across records for CSV header
|
||||
language_codes: list[str] = []
|
||||
seen_codes: set[str] = set()
|
||||
for rec in records:
|
||||
for lang in (rec.languages or []):
|
||||
if lang.language_code and lang.language_code not in seen_codes:
|
||||
language_codes.append(lang.language_code)
|
||||
seen_codes.add(lang.language_code)
|
||||
|
||||
output = io.StringIO()
|
||||
writer = csv.writer(output)
|
||||
|
||||
# Header: identity + source + target + per-language + metadata
|
||||
header = [
|
||||
"record_id", "source_object_type", "source_object_id",
|
||||
"source_object_name", "source_sql", "target_sql",
|
||||
"source_data",
|
||||
]
|
||||
header.extend(language_codes)
|
||||
header.append("error_message")
|
||||
writer.writerow(header)
|
||||
|
||||
for rec in records:
|
||||
# Build per-language values map
|
||||
lang_values: dict[str, str] = {}
|
||||
for lang in (rec.languages or []):
|
||||
if lang.language_code:
|
||||
lang_values[lang.language_code] = lang.final_value or lang.translated_value or ""
|
||||
|
||||
# Flatten source_data JSON to string for CSV compatibility
|
||||
source_data_str = ""
|
||||
if rec.source_data:
|
||||
try:
|
||||
source_data_str = json.dumps(rec.source_data, ensure_ascii=False)
|
||||
except (TypeError, ValueError):
|
||||
source_data_str = str(rec.source_data)
|
||||
|
||||
row = [
|
||||
rec.id,
|
||||
rec.source_object_type,
|
||||
rec.source_object_id,
|
||||
rec.source_object_name,
|
||||
rec.source_sql,
|
||||
rec.target_sql,
|
||||
source_data_str,
|
||||
]
|
||||
row.extend(lang_values.get(code, "") for code in language_codes)
|
||||
row.append(rec.error_message or "")
|
||||
writer.writerow(row)
|
||||
|
||||
# Append run-level failure metadata as a comment row
|
||||
writer.writerow([])
|
||||
writer.writerow(["# Run failure metadata:"])
|
||||
writer.writerow(["run_id", run.id])
|
||||
writer.writerow(["run_status", run.status])
|
||||
writer.writerow(["insert_status", run.insert_status or ""])
|
||||
writer.writerow(["run_error", run.error_message or ""])
|
||||
writer.writerow(["total_records", run.total_records or 0])
|
||||
writer.writerow(["successful_records", run.successful_records or 0])
|
||||
|
||||
output.seek(0)
|
||||
return StreamingResponse(
|
||||
iter([output.getvalue()]),
|
||||
media_type="text/csv",
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename=failed_{run_id[:8]}.csv"
|
||||
},
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.explore("download_failed_csv failed",
|
||||
extra={"src": "translate_routes", "error": str(e)})
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||
# #endregion download_failed_csv
|
||||
|
||||
# #endregion TranslateRunListRoutesModule
|
||||
|
||||
@@ -420,7 +420,7 @@ class TestTranslationOrchestrator:
|
||||
# endregion test_execute_run_no_job
|
||||
|
||||
# region test_execute_run_insert_failure [TYPE Function]
|
||||
# @BRIEF SQL generation/insert returns failure, run still completes but with error logged.
|
||||
# @BRIEF SQL generation/insert returns failure — run marked FAILED with error logged.
|
||||
def test_execute_run_insert_failure(self, mock_job: MagicMock) -> None:
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
@@ -464,7 +464,7 @@ class TestTranslationOrchestrator:
|
||||
patch.object(engine, 'event_log'):
|
||||
result = orch.execute_run(run)
|
||||
|
||||
assert result.status == "COMPLETED"
|
||||
assert result.status == "FAILED"
|
||||
assert result.insert_status == "failed"
|
||||
assert "timeout" in (result.error_message or "")
|
||||
|
||||
|
||||
@@ -106,9 +106,17 @@ def complete_success(
|
||||
run.insert_status = insert_result.get("status")
|
||||
run.superset_execution_id = str(insert_result.get("query_id") or "")
|
||||
run.superset_execution_log = insert_result
|
||||
if run.status != "FAILED":
|
||||
# Treat "failed" and "timeout" as terminal failures.
|
||||
# "skipped" (no records) and "success" are non-failure outcomes.
|
||||
if insert_result.get("status") in ("failed", "timeout"):
|
||||
run.status = "FAILED"
|
||||
if insert_result.get("error_message"):
|
||||
run.error_message = insert_result["error_message"]
|
||||
elif not run.error_message:
|
||||
run.error_message = f"SQL insert phase {insert_result.get('status')}"
|
||||
elif run.status != "FAILED":
|
||||
run.status = "COMPLETED"
|
||||
if insert_result.get("error_message"):
|
||||
if insert_result.get("error_message") and not run.error_message:
|
||||
run.error_message = insert_result["error_message"]
|
||||
run.completed_at = datetime.now(UTC)
|
||||
db.flush()
|
||||
|
||||
@@ -173,36 +173,30 @@ class SupersetSqlLabExecutor:
|
||||
"client_id": f"trl-{uuid.uuid4().hex[:7]}", # max 11 chars for Superset varchar(11)
|
||||
}
|
||||
|
||||
# Try multiple SQL Lab endpoints — some Superset versions use /sqllab/execute/,
|
||||
# others use /sql_lab/execute/ (with underscore)
|
||||
candidate_endpoints = ["/api/v1/sqllab/execute/", "/api/v1/sql_lab/execute/"]
|
||||
response = None
|
||||
last_error = None
|
||||
for endpoint in candidate_endpoints:
|
||||
try:
|
||||
response = client.network.request(
|
||||
method="POST",
|
||||
endpoint=endpoint,
|
||||
data=json.dumps(payload),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
if isinstance(response, dict) and response:
|
||||
logger.reason("SQL Lab endpoint succeeded", extra={
|
||||
"endpoint": endpoint,
|
||||
})
|
||||
break
|
||||
except Exception as ep_err:
|
||||
last_error = ep_err
|
||||
logger.explore("SQL Lab endpoint failed, trying next", extra={
|
||||
"error": str(ep_err),
|
||||
"endpoint": endpoint,
|
||||
})
|
||||
response = None
|
||||
|
||||
if response is None:
|
||||
error_msg = f"All SQL Lab endpoints failed. Last error: {last_error}"
|
||||
logger.explore("SQL Lab execute failed", extra={"error": error_msg})
|
||||
raise ValueError(f"Superset SQL Lab execute failed: {error_msg}")
|
||||
# Use the canonical Superset SQL Lab execute endpoint.
|
||||
# NOTE: Some older docs reference /sql_lab/execute/ (with underscore),
|
||||
# but this endpoint does NOT exist in current Superset versions.
|
||||
# The browser UI uses /api/v1/sqllab/execute/ — confirmed by
|
||||
# fetch() calls in Superset SQL Lab.
|
||||
endpoint = "/api/v1/sqllab/execute/"
|
||||
try:
|
||||
response = client.network.request(
|
||||
method="POST",
|
||||
endpoint=endpoint,
|
||||
data=json.dumps(payload),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
logger.reason("SQL Lab endpoint succeeded", extra={
|
||||
"endpoint": endpoint,
|
||||
})
|
||||
except Exception as ep_err:
|
||||
logger.explore("SQL Lab execute failed", extra={
|
||||
"error": str(ep_err),
|
||||
"endpoint": endpoint,
|
||||
})
|
||||
raise ValueError(
|
||||
f"Superset SQL Lab execute failed: {ep_err}"
|
||||
) from ep_err
|
||||
|
||||
# Parse response — try multiple known Superset response formats
|
||||
result = response if isinstance(response, dict) else {}
|
||||
|
||||
@@ -27,7 +27,7 @@ export { dictionaryApi } from './translate/dictionaries';
|
||||
export { fetchSchedule, setSchedule, deleteSchedule, enableSchedule, disableSchedule, fetchNextExecutions } from './translate/schedules';
|
||||
|
||||
// Corrections & bulk replace
|
||||
export { submitCorrection, submitBulkCorrections, inlineEditCorrection, submitCorrectionToDict, bulkFindReplace, bulkReplacePreview, downloadSkippedCsv } from './translate/corrections';
|
||||
export { submitCorrection, submitBulkCorrections, inlineEditCorrection, submitCorrectionToDict, bulkFindReplace, bulkReplacePreview, downloadSkippedCsv, downloadFailedCsv } from './translate/corrections';
|
||||
|
||||
// Target schema validation
|
||||
export { checkTargetTableSchema } from './translate/target-schema';
|
||||
|
||||
@@ -134,4 +134,28 @@ export async function downloadSkippedCsv(runId: string): Promise<void> {
|
||||
}
|
||||
}
|
||||
// #endregion downloadSkippedCsv
|
||||
|
||||
// #region downloadFailedCsv [C:2] [TYPE Function] [SEMANTICS translate, corrections, csv, download]
|
||||
// @BRIEF Download a CSV of successfully translated but uninserted records for a FAILED run.
|
||||
// @PRE runId is a non-empty string. The run must have status=FAILED or insert_status=failed.
|
||||
// @POST Triggers browser download of a CSV file. Returns nothing on success.
|
||||
// @SIDE_EFFECT Creates a Blob URL and programmatically clicks a download anchor.
|
||||
// @RATIONALE When INSERT into target DB fails, users need to recover translated data for manual import.
|
||||
// @RELATION DEPENDS_ON -> [ApiModule.fetchApiBlob]
|
||||
export async function downloadFailedCsv(runId: string): Promise<void> {
|
||||
try {
|
||||
const blob = await api.fetchApiBlob(`/translate/runs/${runId}/failed.csv`);
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = `failed-${runId.substring(0, 8)}.csv`;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(url);
|
||||
} catch (error) {
|
||||
throw normalizeTranslateError(error, 'Failed to download failed CSV');
|
||||
}
|
||||
}
|
||||
// #endregion downloadFailedCsv
|
||||
// #endregion TranslateCorrectionsApi
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
import { fromStore } from 'svelte/store';
|
||||
import { t } from '$lib/i18n';
|
||||
import { addToast } from '$lib/toasts';
|
||||
import { cancelRun } from '$lib/api/translate.js';
|
||||
import { cancelRun, downloadFailedCsv } from '$lib/api/translate.js';
|
||||
import { translationRunStore } from '$lib/stores/translationRun.js';
|
||||
|
||||
/** @type {{ runId: string, onRetry?: () => void, onRetryInsert?: () => void, onComplete?: (statusData: any) => void }} */
|
||||
@@ -203,12 +203,23 @@
|
||||
<span class="text-orange-600 text-lg">⚠</span>
|
||||
<span class="text-sm font-semibold text-orange-800">{$t.translate?.run?.insert_failed}</span>
|
||||
</div>
|
||||
<button
|
||||
onclick={onRetryInsert}
|
||||
class="px-3 py-1 text-xs bg-orange-100 text-orange-700 rounded hover:bg-orange-200 transition-colors"
|
||||
>
|
||||
{$t.translate?.run?.retry_insert}
|
||||
</button>
|
||||
<div class="flex items-center gap-2">
|
||||
<button
|
||||
onclick={async () => {
|
||||
try { await downloadFailedCsv(runId); }
|
||||
catch (e) { addToast(e?.message || 'Download failed', 'error'); }
|
||||
}}
|
||||
class="px-3 py-1 text-xs bg-blue-100 text-blue-700 rounded hover:bg-blue-200 transition-colors"
|
||||
>
|
||||
{$t.translate?.run?.download_csv || 'Download CSV'}
|
||||
</button>
|
||||
<button
|
||||
onclick={onRetryInsert}
|
||||
class="px-3 py-1 text-xs bg-orange-100 text-orange-700 rounded hover:bg-orange-200 transition-colors"
|
||||
>
|
||||
{$t.translate?.run?.retry_insert}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<p class="text-xs text-gray-600 mt-1">
|
||||
{$t.translate?.run?.completed} ({successfulRecords}) — {$t.translate?.run?.insert_failed}
|
||||
|
||||
@@ -16,7 +16,8 @@
|
||||
fetchRunRecords,
|
||||
retryFailedBatches,
|
||||
retryInsert,
|
||||
fetchRunBatches
|
||||
fetchRunBatches,
|
||||
downloadFailedCsv
|
||||
} from '$lib/api/translate.js';
|
||||
import CorrectionCell from './CorrectionCell.svelte';
|
||||
import TermCorrectionPopup from './TermCorrectionPopup.svelte';
|
||||
@@ -178,6 +179,18 @@
|
||||
<div class="flex items-center justify-between">
|
||||
<h3 class="text-lg font-semibold text-gray-900">{$t.translate?.run?.result_title}</h3>
|
||||
<div class="flex items-center gap-2">
|
||||
<!-- Download CSV for failed/insert_failed runs with successful records -->
|
||||
{#if (uxState === 'insert_failed' || uxState === 'failed' || uxState === 'partial') && (status.successful_records || 0) > 0}
|
||||
<button
|
||||
onclick={async () => {
|
||||
try { await downloadFailedCsv(runId); }
|
||||
catch (e) { addToast(e?.message || 'Download failed', 'error'); }
|
||||
}}
|
||||
class="px-3 py-1.5 text-xs bg-blue-100 text-blue-700 rounded hover:bg-blue-200 transition-colors"
|
||||
>
|
||||
{$t.translate?.run?.download_csv || 'CSV'}
|
||||
</button>
|
||||
{/if}
|
||||
<!-- Retry buttons -->
|
||||
{#if uxState === 'failed' || uxState === 'partial'}
|
||||
<button
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
fetchAllMetrics,
|
||||
fetchJobs,
|
||||
downloadSkippedCsv,
|
||||
downloadFailedCsv,
|
||||
cancelRun,
|
||||
retryFailedBatches
|
||||
} from '$lib/api/translate.js';
|
||||
@@ -161,6 +162,14 @@
|
||||
}
|
||||
}
|
||||
|
||||
async function handleDownloadFailed(runId) {
|
||||
try {
|
||||
await downloadFailedCsv(runId);
|
||||
} catch (err) {
|
||||
addToast(err?.message || _('translate.history.download_failed'), 'error');
|
||||
}
|
||||
}
|
||||
|
||||
function totalMetric() {
|
||||
if (!metrics.length) return null;
|
||||
const agg = {
|
||||
@@ -322,6 +331,14 @@
|
||||
{$t.translate?.history?.skipped_csv}
|
||||
</button>
|
||||
{/if}
|
||||
{#if run.status === 'FAILED' && (run.successful_records || 0) > 0}
|
||||
<button
|
||||
onclick={() => handleDownloadFailed(run.id)}
|
||||
class="text-xs text-primary hover:text-primary-hover"
|
||||
>
|
||||
{$t.translate?.history?.failed_csv || 'Download CSV'}
|
||||
</button>
|
||||
{/if}
|
||||
<svg class="w-4 h-4 text-gray-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 5l7 7-7 7" />
|
||||
</svg>
|
||||
|
||||
Reference in New Issue
Block a user