perf: fix translate deadlock, speed, trace_id, UI bugs — fullstack patch
## Backend (7 production files + 6 test files) ### P0-2: LLM output truncation cascade fix - _token_budget.py: OUTPUT_PER_ROW_PER_LANG 120→200, OUTPUT_SAFETY_FACTOR 0.70→0.55 - Prevents finish_reason=length → split → retry cascade (3 calls → 1 call per batch) - P2-8: added qwen-flash/qwen-plus/qwen-max/qwen-coder to PROVIDER_DEFAULTS ### P1-4/P1-5: EncryptionManager singleton - encryption.py: get_encryption_manager() process-wide singleton - llm_provider.py: use singleton instead of new EncryptionManager() per batch - Eliminates ~90 redundant Fernet key validations per translation run ### P1-6: Cache-hit log aggregation - _batch_proc.py: one log per batch (batch_rows + cache_hits) instead of per-row - 1076 log lines → ~30 per run ### P1-7: Timezone-aware datetime fix - scheduler.py: _ensure_aware() helper for naive DB datetime → UTC-aware - Fixes TypeError in scheduled translation concurrency check ### P2-9: Connection test timeout - connection_service.py: asyncio.wait_for(15s) on all dialect tests - Prevents 2-minute UI hangs from DNS/TCP stalls ### Trace ID propagation - middleware/trace.py: inject x-trace-id response header via ASGI send wrapper ### Test fixes & integration tests - test_scheduler.py: AsyncMock for execute_run, mock get_async_job_runner - test_sql_insert_service.py: AsyncMock for execute_sql - test_token_budget.py: batch_size 50→45 for new OUTPUT_PER_ROW_PER_LANG=200 - test_encryption.py: +2 singleton tests - test_scheduler_ensure_aware.py: +4 (naive→aware, passthrough, None, subtraction) - test_batch_classify_persist.py: +2 cache-hit aggregation tests - test_connection_service_edge.py: +2 timeout tests - test_trace_middleware.py: +4 x-trace-id header tests - test_token_budget.py: +4 qwen-flash/O200 tests ## Frontend (7 production files + 5 test files) ### Trace ID propagation - api.ts: _captureTraceId() reads x-trace-id → setTraceId() in fetchApi/requestApi/postApi/deleteApi ### Duplicate datasource columns fetch - ConfigTabForm.svelte: guard availableColumns.length === 0 before fetch ### Admin pages Svelte 5 runes fix - admin/users/+page.svelte: plain let → () for all template-bound vars - admin/roles/+page.svelte: same fix - Both pages were stuck on «Загрузка...» due to mixed reactivity models ### Validation popover positioning - +page.svelte: pass trigger HTMLElement instead of event - DashboardHubModel.svelte.ts: toggleValidationPopover(HTMLElement), closeValidationPopover() - Added X close button + click-outside overlay + i18n ### Test fixes & integration tests - api.test.ts: mock setTraceId/getTraceId, +3 _captureTraceId tests - provider_config.integration.test.ts: handleDelete→promptDeleteProvider - DatasetPreview.test.ts: dashboards/ → ROUTES.dashboards - test_config_tab_form.svelte.js: +2 columns fetch guard tests (NEW) - admin-users.test.ts: +3 loading→table tests (NEW) - admin-roles.test.ts: +2 loading→table tests (NEW) ## Semantic curation - Removed @COMPLEXITY N from 6 route files + metrics.py (duplicate of [C:N]) - Added [C:N] to 2 orphan child contracts in metrics.py - Added [C:N] + @BRIEF to 4 frontend anchors - Fixed #region → # #region consistency in validation_tasks.py ## Verification - Backend: 608 pytest passed (0 failures) - Frontend: 2472 vitest passed (128 files, 0 failures) - Frontend build: ✓ built in 18s - Browser: dashboards, admin/users, admin/roles, validation popover — all green
This commit is contained in:
@@ -400,6 +400,28 @@ class TestExecuteInsertSql:
|
||||
await _execute_insert_sql(executor, "INSERT ...", "batch-1", 5)
|
||||
executor.execute_and_poll.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_db_success(self):
|
||||
"""DbExecutor path: successful execution."""
|
||||
from src.core.db_executor import DbExecutor, DbExecutionResult
|
||||
executor = MagicMock(spec=DbExecutor)
|
||||
executor.execute_sql = AsyncMock(return_value=DbExecutionResult(
|
||||
success=True, rows_affected=5, execution_time_ms=12.0
|
||||
))
|
||||
await _execute_insert_sql(executor, "INSERT ...", "batch-1", 5, connection_id="conn-1")
|
||||
executor.execute_sql.assert_called_once_with("conn-1", "INSERT ...")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_db_failure(self):
|
||||
"""DbExecutor path: failed execution with error."""
|
||||
from src.core.db_executor import DbExecutor, DbExecutionResult
|
||||
executor = MagicMock(spec=DbExecutor)
|
||||
executor.execute_sql = AsyncMock(return_value=DbExecutionResult(
|
||||
success=False, error="Connection refused"
|
||||
))
|
||||
await _execute_insert_sql(executor, "INSERT ...", "batch-1", 5, connection_id="conn-1")
|
||||
executor.execute_sql.assert_called_once_with("conn-1", "INSERT ...")
|
||||
|
||||
|
||||
class TestResolveInsertBackend:
|
||||
"""_resolve_insert_backend — resolve backend dialect and executor."""
|
||||
@@ -421,13 +443,14 @@ class TestResolveInsertBackend:
|
||||
mock_exec.resolve_database_id = AsyncMock()
|
||||
mock_exec_cls.return_value = mock_exec
|
||||
|
||||
dialect, executor = await _resolve_insert_backend(config_manager, job, "batch-1")
|
||||
dialect, executor, connection_id = await _resolve_insert_backend(config_manager, job, "batch-1")
|
||||
assert dialect == "postgresql"
|
||||
assert executor is not None
|
||||
assert connection_id is None # sqllab path has no connection_id
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_returns_none(self):
|
||||
"""Exception in resolve returns (None, None)."""
|
||||
"""Exception in resolve returns (None, None, None)."""
|
||||
config_manager = MagicMock()
|
||||
job = MagicMock(spec=TranslationJob)
|
||||
job.environment_id = None
|
||||
@@ -435,9 +458,10 @@ class TestResolveInsertBackend:
|
||||
|
||||
with patch("src.plugins.translate._batch_insert.SupersetSqlLabExecutor",
|
||||
side_effect=Exception("Failed")):
|
||||
dialect, executor = await _resolve_insert_backend(config_manager, job, "batch-1")
|
||||
dialect, executor, connection_id = await _resolve_insert_backend(config_manager, job, "batch-1")
|
||||
assert dialect is None
|
||||
assert executor is None
|
||||
assert connection_id is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_dialect(self):
|
||||
@@ -456,8 +480,59 @@ class TestResolveInsertBackend:
|
||||
mock_exec.resolve_database_id = AsyncMock()
|
||||
mock_exec_cls.return_value = mock_exec
|
||||
|
||||
dialect, executor = await _resolve_insert_backend(config_manager, job, "batch-1")
|
||||
dialect, _, connection_id = await _resolve_insert_backend(config_manager, job, "batch-1")
|
||||
assert dialect == "mysql"
|
||||
assert connection_id is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_db_path(self):
|
||||
"""direct_db path returns DbExecutor with connection_id."""
|
||||
config_manager = MagicMock()
|
||||
job = MagicMock(spec=TranslationJob)
|
||||
job.insert_method = "direct_db"
|
||||
job.connection_id = "clickhouse-conn-1"
|
||||
|
||||
with patch("src.plugins.translate._batch_insert.ConnectionService") as mock_cs_cls:
|
||||
mock_cs = MagicMock()
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.dialect = "clickhouse"
|
||||
mock_conn.name = "Test ClickHouse"
|
||||
mock_cs.get_connection.return_value = mock_conn
|
||||
mock_cs_cls.return_value = mock_cs
|
||||
|
||||
dialect, executor, connection_id = await _resolve_insert_backend(config_manager, job, "batch-1")
|
||||
assert dialect == "clickhouse"
|
||||
assert connection_id == "clickhouse-conn-1"
|
||||
# Should return DbExecutor, not SupersetSqlLabExecutor
|
||||
from src.core.db_executor import DbExecutor
|
||||
assert isinstance(executor, DbExecutor)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_db_missing_conn_raises(self):
|
||||
"""When direct_db connection not found, raises ValueError (no fallback)."""
|
||||
config_manager = MagicMock()
|
||||
job = MagicMock(spec=TranslationJob)
|
||||
job.insert_method = "direct_db"
|
||||
job.connection_id = "missing-conn"
|
||||
|
||||
with patch("src.plugins.translate._batch_insert.ConnectionService") as mock_cs_cls:
|
||||
mock_cs = MagicMock()
|
||||
mock_cs.get_connection.return_value = None
|
||||
mock_cs_cls.return_value = mock_cs
|
||||
|
||||
with pytest.raises(ValueError, match="connection_id='missing-conn' not found"):
|
||||
await _resolve_insert_backend(config_manager, job, "batch-1")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_db_no_connection_id_raises(self):
|
||||
"""When direct_db is set but connection_id is None, raises ValueError (no fallback)."""
|
||||
config_manager = MagicMock()
|
||||
job = MagicMock(spec=TranslationJob)
|
||||
job.insert_method = "direct_db"
|
||||
job.connection_id = None
|
||||
|
||||
with pytest.raises(ValueError, match="connection_id is missing"):
|
||||
await _resolve_insert_backend(config_manager, job, "batch-1")
|
||||
|
||||
|
||||
class TestInsertBatchToTarget:
|
||||
@@ -499,7 +574,7 @@ class TestInsertBatchToTarget:
|
||||
|
||||
with patch("src.plugins.translate._batch_insert._fetch_batch_records", return_value=[rec]):
|
||||
with patch("src.plugins.translate._batch_insert._resolve_insert_backend",
|
||||
return_value=("postgresql", AsyncMock())):
|
||||
return_value=("postgresql", AsyncMock(), None)):
|
||||
with patch("src.plugins.translate._batch_insert.SQLGenerator.generate_batch",
|
||||
return_value=[("INSERT INTO ...", 1)]):
|
||||
with patch("src.plugins.translate._batch_insert._execute_insert_sql",
|
||||
@@ -530,7 +605,7 @@ class TestInsertBatchToTarget:
|
||||
with patch("src.plugins.translate._batch_insert._build_insert_rows",
|
||||
return_value=[{"text": "bonjour"}]):
|
||||
with patch("src.plugins.translate._batch_insert._resolve_insert_backend",
|
||||
return_value=(None, None)):
|
||||
return_value=(None, None, None)):
|
||||
result = await insert_batch_to_target(
|
||||
db_session, config_manager, job, "batch-1", "run-1"
|
||||
)
|
||||
|
||||
@@ -163,7 +163,7 @@ class TestInsertBatchToTargetEdgeCases:
|
||||
with patch("src.plugins.translate._batch_insert._build_insert_rows",
|
||||
return_value=[{"text": "bonjour"}]):
|
||||
with patch("src.plugins.translate._batch_insert._resolve_insert_backend",
|
||||
return_value=("postgresql", AsyncMock())):
|
||||
return_value=("postgresql", AsyncMock(), None)):
|
||||
with patch("src.plugins.translate._batch_insert.SQLGenerator.generate_batch",
|
||||
return_value=[("INSERT INTO ...", 1)]):
|
||||
with patch("src.plugins.translate._batch_insert._execute_insert_sql",
|
||||
@@ -208,7 +208,7 @@ class TestInsertBatchToTargetEdgeCases:
|
||||
with patch("src.plugins.translate._batch_insert._build_insert_rows",
|
||||
return_value=[{"text": "bonjour", "is_original": 1}]):
|
||||
with patch("src.plugins.translate._batch_insert._resolve_insert_backend",
|
||||
return_value=("postgresql", mock_executor)):
|
||||
return_value=("postgresql", mock_executor, None)):
|
||||
with patch("src.plugins.translate._batch_insert.SQLGenerator.generate_batch",
|
||||
return_value=[(None, 0), ("INSERT INTO target ...", 1)]):
|
||||
result = await insert_batch_to_target(
|
||||
|
||||
Reference in New Issue
Block a user