perf: fix translate deadlock, speed, trace_id, UI bugs — fullstack patch

## Backend (7 production files + 6 test files)

### P0-2: LLM output truncation cascade fix
- _token_budget.py: OUTPUT_PER_ROW_PER_LANG 120→200, OUTPUT_SAFETY_FACTOR 0.70→0.55
- Prevents finish_reason=length → split → retry cascade (3 calls → 1 call per batch)
- P2-8: added qwen-flash/qwen-plus/qwen-max/qwen-coder to PROVIDER_DEFAULTS

### P1-4/P1-5: EncryptionManager singleton
- encryption.py: get_encryption_manager() process-wide singleton
- llm_provider.py: use singleton instead of new EncryptionManager() per batch
- Eliminates ~90 redundant Fernet key validations per translation run

### P1-6: Cache-hit log aggregation
- _batch_proc.py: one log per batch (batch_rows + cache_hits) instead of per-row
- 1076 log lines → ~30 per run

### P1-7: Timezone-aware datetime fix
- scheduler.py: _ensure_aware() helper for naive DB datetime → UTC-aware
- Fixes TypeError in scheduled translation concurrency check

### P2-9: Connection test timeout
- connection_service.py: asyncio.wait_for(15s) on all dialect tests
- Prevents 2-minute UI hangs from DNS/TCP stalls

### Trace ID propagation
- middleware/trace.py: inject x-trace-id response header via ASGI send wrapper

### Test fixes & integration tests
- test_scheduler.py: AsyncMock for execute_run, mock get_async_job_runner
- test_sql_insert_service.py: AsyncMock for execute_sql
- test_token_budget.py: batch_size 50→45 for new OUTPUT_PER_ROW_PER_LANG=200
- test_encryption.py: +2 singleton tests
- test_scheduler_ensure_aware.py: +4 (naive→aware, passthrough, None, subtraction)
- test_batch_classify_persist.py: +2 cache-hit aggregation tests
- test_connection_service_edge.py: +2 timeout tests
- test_trace_middleware.py: +4 x-trace-id header tests
- test_token_budget.py: +4 qwen-flash/O200 tests

## Frontend (7 production files + 5 test files)

### Trace ID propagation
- api.ts: _captureTraceId() reads x-trace-id → setTraceId() in fetchApi/requestApi/postApi/deleteApi

### Duplicate datasource columns fetch
- ConfigTabForm.svelte: guard availableColumns.length === 0 before fetch

### Admin pages Svelte 5 runes fix
- admin/users/+page.svelte: plain let → () for all template-bound vars
- admin/roles/+page.svelte: same fix
- Both pages were stuck on «Загрузка...» due to mixed reactivity models

### Validation popover positioning
- +page.svelte: pass trigger HTMLElement instead of event
- DashboardHubModel.svelte.ts: toggleValidationPopover(HTMLElement), closeValidationPopover()
- Added X close button + click-outside overlay + i18n

### Test fixes & integration tests
- api.test.ts: mock setTraceId/getTraceId, +3 _captureTraceId tests
- provider_config.integration.test.ts: handleDelete→promptDeleteProvider
- DatasetPreview.test.ts: dashboards/ → ROUTES.dashboards
- test_config_tab_form.svelte.js: +2 columns fetch guard tests (NEW)
- admin-users.test.ts: +3 loading→table tests (NEW)
- admin-roles.test.ts: +2 loading→table tests (NEW)

## Semantic curation
- Removed @COMPLEXITY N from 6 route files + metrics.py (duplicate of [C:N])
- Added [C:N] to 2 orphan child contracts in metrics.py
- Added [C:N] + @BRIEF to 4 frontend anchors
- Fixed #region → # #region consistency in validation_tasks.py

## Verification
- Backend: 608 pytest passed (0 failures)
- Frontend: 2472 vitest passed (128 files, 0 failures)
- Frontend build: ✓ built in 18s
- Browser: dashboards, admin/users, admin/roles, validation popover — all green
This commit is contained in:
2026-06-18 23:54:57 +03:00
parent 4a6fe8db58
commit 3133e50645
59 changed files with 1774 additions and 13928 deletions

View File

@@ -25,7 +25,7 @@
// @REJECTED Axios rejected — unnecessary dependency for a single-domain API client; native fetch + wrappers
// is simpler, tree-shakeable, and has zero bundle cost.
import { log } from '$lib/cot-logger';
import { log, setTraceId } from '$lib/cot-logger';
import { addToast } from './toasts.svelte.js';
import type { FetchOptions, DashboardListParams } from '../types/api';
@@ -226,6 +226,16 @@ function getAuthHeaders(extraHeaders: Record<string, string> = {}): Record<strin
// @RELATION DEPENDS_ON -> [buildApiError]
// @RELATION DEPENDS_ON -> [notifyApiError]
// @RELATION DEPENDS_ON -> [getAuthHeaders]
/** Extract X-Trace-Id from response headers and seed the CoT logger's trace_id. */
function _captureTraceId(response: Response): void {
try {
const traceId = response.headers?.get?.('x-trace-id');
if (traceId) setTraceId(traceId);
} catch {
// headers unavailable (e.g. test mock without full Headers API)
}
}
async function fetchApi<T = unknown>(endpoint: string, options: FetchOptions = {}): Promise<T> {
try {
log('api', 'REASON', 'fetchApi', { endpoint });
@@ -234,6 +244,7 @@ async function fetchApi<T = unknown>(endpoint: string, options: FetchOptions = {
const response = await fetch(`${API_BASE_URL}${endpoint}`, fetchInit);
if (!response.ok) throw await buildApiError(response);
if (response.status === 204) return null as T;
_captureTraceId(response);
return await response.json() as T;
} catch (error) {
const apiError = error as ApiError;
@@ -296,6 +307,7 @@ async function postApi<T = unknown>(endpoint: string, body: unknown, options: Fe
const response = await fetch(`${API_BASE_URL}${endpoint}`, fetchInit);
if (!response.ok) throw await buildApiError(response);
if (response.status === 204) return null as T;
_captureTraceId(response);
return await response.json() as T;
} catch (error) {
const apiError = error as ApiError;
@@ -325,6 +337,7 @@ async function deleteApi<T = unknown>(endpoint: string, options: FetchOptions =
const response = await fetch(`${API_BASE_URL}${endpoint}`, fetchInit);
if (!response.ok) throw await buildApiError(response);
if (response.status === 204) return null as T;
_captureTraceId(response);
return await response.json() as T;
} catch (error) {
const apiError = error as ApiError;
@@ -356,6 +369,7 @@ async function requestApi<T = unknown>(endpoint: string, method: string = 'GET',
const response = await fetch(`${API_BASE_URL}${endpoint}`, fetchInit);
if (!response.ok) throw await buildApiError(response);
if (response.status === 204) return null as T;
_captureTraceId(response);
return await response.json() as T;
} catch (error) {
const apiError = error as ApiError;

View File

@@ -33,6 +33,8 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
// Mock the cot-logger to avoid side effects
vi.mock('$lib/cot-logger.js', () => ({
log: vi.fn(),
setTraceId: vi.fn(),
getTraceId: vi.fn(() => 'test-trace-id'),
}));
// Mock toasts
@@ -672,6 +674,11 @@ describe('ApiModule — fetch wrappers (global fetch mock)', () => {
it('getValidationStatusBatch returns empty results stub', async () => {
const { api } = await import('$lib/api.js');
vi.mocked(fetch).mockResolvedValue({
ok: true,
status: 200,
json: () => Promise.resolve({ results: [] }),
} as Response);
const result = await api.getValidationStatusBatch();
expect(result).toEqual({ results: [] });
});
@@ -728,6 +735,69 @@ describe('ApiModule — fetch wrappers (global fetch mock)', () => {
status: 404,
});
});
// #region captureTraceIdTests [C:2] [TYPE Test] [SEMANTICS test,api,trace-id,headers]
// @BRIEF Verify _captureTraceId behavior — seeds trace_id from x-trace-id header, handles missing gracefully.
describe('_captureTraceId', () => {
const testUuid = 'a1b2c3d4-e5f6-7890-abcd-ef1234567890';
beforeEach(() => {
vi.clearAllMocks();
});
it('sets trace_id from x-trace-id response header', async () => {
vi.mocked(fetch).mockResolvedValue({
ok: true,
status: 200,
json: () => Promise.resolve({ data: 'ok' }),
headers: {
get: vi.fn((name: string) => name === 'x-trace-id' ? testUuid : null),
},
} as unknown as Response);
const cotLogger = await import('$lib/cot-logger.js');
cotLogger.setTraceId.mockClear();
const { api } = await import('$lib/api.js');
await api.fetchApi('/capture-trace-test');
expect(cotLogger.setTraceId).toHaveBeenCalledWith(testUuid);
});
it('handles missing x-trace-id header gracefully', async () => {
vi.mocked(fetch).mockResolvedValue({
ok: true,
status: 200,
json: () => Promise.resolve({ data: 'ok' }),
headers: {
get: vi.fn(() => null),
},
} as unknown as Response);
const cotLogger = await import('$lib/cot-logger.js');
cotLogger.setTraceId.mockClear();
const { api } = await import('$lib/api.js');
await api.fetchApi('/no-trace-id');
expect(cotLogger.setTraceId).not.toHaveBeenCalled();
});
it('handles missing headers API (undefined headers) without throwing', async () => {
vi.mocked(fetch).mockResolvedValue({
ok: true,
status: 200,
json: () => Promise.resolve({ data: 'ok' }),
} as Response);
const { api } = await import('$lib/api.js');
// Should not throw despite missing headers
const result = await api.fetchApi('/no-headers');
expect(result).toEqual({ data: 'ok' });
});
});
// #endregion captureTraceIdTests
});
describe('ApiModule — deleteValidationTask query param', () => {

View File

@@ -63,10 +63,10 @@ describe('ProviderConfig edit interaction contract', () => {
it('keeps explicit delete flow with confirmation and delete request', () => {
const source = fs.readFileSync(COMPONENT_PATH, 'utf-8');
expect(source).toContain('async function handleDelete(provider)');
expect(source).toContain('$t.llm.delete_confirm.replace("{name}", provider.name || provider.id)');
expect(source).toContain('function promptDeleteProvider(provider)');
expect(source).toContain('delete_confirm');
expect(source).toContain('await requestApi(`/llm/providers/${provider.id}`, "DELETE")');
expect(source).toContain('onclick={() => handleDelete(provider)}');
expect(source).toContain('onclick={() => promptDeleteProvider(provider)}');
});
it('excludes masked or empty api_key from submit data when editing', () => {

View File

@@ -1,5 +1,5 @@
// #region ReportTypeProfiles:Module [TYPE Function]
// @PURPOSE: Deterministic mapping from report task_type to visual profile with one fallback.
// #region ReportTypeProfiles:Module [C:1] [TYPE Module]
// @BRIEF Deterministic mapping from report task_type to visual profile with one fallback.
import { _ } from '$lib/i18n/index.svelte.js';
@@ -49,8 +49,8 @@ export const REPORT_TYPE_PROFILES: Record<string, ReportTypeProfile> = {
}
};
// #region getReportTypeProfile:Function [TYPE Function]
// @PURPOSE: Resolve visual profile by task type with guaranteed fallback.
// #region getReportTypeProfile:Function [C:2] [TYPE Function]
// @BRIEF Resolve visual profile by task type with guaranteed fallback.
// @PRE: taskType may be known/unknown/empty.
// @POST: Returns one profile object.
//

View File

@@ -95,9 +95,10 @@
return (virtualColumns || []).includes(colName);
}
// Auto-load columns when datasourceId changes (from parent loadJob or user selection)
// Auto-load columns when datasourceId changes (from parent loadJob or user selection).
// Skip if columns already loaded by the model to avoid duplicate fetch.
$effect(() => {
if (datasourceId && environmentId) {
if (datasourceId && environmentId && availableColumns.length === 0) {
loadColumnsForDatasource();
}
});

View File

@@ -0,0 +1,142 @@
// #region ConfigTabFormColumnsTest [C:2] [TYPE Module] [SEMANTICS test,translate,config,columns,datasource]
// @BRIEF Verify ConfigTabForm columns fetch behavior — skips fetch when columns already loaded,
// triggers fetch when columns array is empty.
// @RELATION BINDS_TO -> [ConfigTabForm]
// @TEST_CONTRACT: availableColumns populated -> fetchDatasourceColumns NOT called on mount
// @TEST_CONTRACT: availableColumns empty -> fetchDatasourceColumns IS called on mount
// @TEST_EDGE: no datasourceId -> fetch skipped even with empty columns
// @TEST_EDGE: no environmentId -> fetch skipped even with empty columns
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { render } from '@testing-library/svelte';
import ConfigTabForm from '../ConfigTabForm.svelte';
// ── Hoisted mock factories (must be before vi.mock) ────────────────
const mockGetAllowedLanguages = vi.hoisted(() => vi.fn().mockResolvedValue(['en', 'ru']));
const mockFetchDatasourceColumns = vi.hoisted(() => vi.fn());
const mockFetchDatasources = vi.hoisted(() => vi.fn());
// ── i18n mock ─────────────────────────────────────────────────────
const mockTranslations = {
translate: {
config: {
basic_info: 'Basic Info',
name: 'Name',
description: 'Description',
name_placeholder: 'Enter name',
description_placeholder: 'Enter description',
help_name: 'Name of the job',
help_description: 'Optional description',
datasource: 'Datasource',
environment: 'Environment',
help_environment: 'Select the target environment',
select_environment: '-- Select --',
datasource_id: 'Datasource Search',
help_datasource: 'Find and select a datasource',
datasource_search_placeholder: 'Search datasets...',
detected_dialect: 'Detected: {dialect}',
available_columns: 'Available columns ({count})',
virtual: 'virtual',
virtual_column_warning: 'Virtual column {col}',
column_mapping: 'Column Mapping',
translation_column: 'Translation Column',
help_translation_column: 'Select the column to translate',
select_column: '-- Select column --',
key_columns: 'Key Columns',
help_key_columns: 'Select key columns for row matching',
clear_all: 'Clear all',
add_key_column: '+ Add key column',
}
}
};
vi.mock('$lib/i18n/index.svelte.js', () => ({
getT: () => mockTranslations,
t: {
subscribe: (fn) => {
fn(mockTranslations);
return () => {};
}
},
_: (key) => key,
}));
// ── Languages mock ────────────────────────────────────────────────
vi.mock('$lib/i18n/languages.js', () => ({
ALL_LANGUAGES: [{ code: 'en', name: 'English' }, { code: 'ru', name: 'Russian' }],
filterLanguages: (codes) => [{ code: 'en', name: 'English' }],
}));
// ── API mocks ─────────────────────────────────────────────────────
vi.mock('$lib/api.js', () => ({
api: {
getAllowedLanguages: mockGetAllowedLanguages,
}
}));
vi.mock('$lib/api/translate.js', () => ({
fetchDatasourceColumns: mockFetchDatasourceColumns,
fetchDatasources: mockFetchDatasources,
}));
// ── CoT logger mock ───────────────────────────────────────────────
vi.mock('$lib/cot-logger', () => ({
log: vi.fn(),
setTraceId: vi.fn(),
getTraceId: vi.fn(() => 'test-trace-id'),
}));
// #region ConfigTabFormColumnsTest.Describe [C:2] [TYPE Test]
// @BRIEF Test the columns fetch $effect guard — availableColumns.length check.
describe('ConfigTabForm — columns fetch guard', () => {
beforeEach(() => {
vi.clearAllMocks();
});
// #region test_skips_columns_fetch_when_already_loaded [C:2] [TYPE Test]
// @BRIEF When availableColumns has items, the effect should not call fetchDatasourceColumns.
it('skips columns fetch when columns already loaded', async () => {
render(ConfigTabForm, {
props: {
datasourceId: 'ds-1',
environmentId: 'env-1',
availableColumns: [
{ name: 'id', type: 'integer' },
{ name: 'name', type: 'varchar' },
],
},
});
// Let $effect and async operations settle
await new Promise(r => setTimeout(r, 50));
expect(mockFetchDatasourceColumns).not.toHaveBeenCalled();
});
// #endregion test_skips_columns_fetch_when_already_loaded
// #region test_fetches_columns_when_empty [C:2] [TYPE Test]
// @BRIEF When availableColumns is empty [] and datasourceId+environmentId are set,
// the effect MUST call fetchDatasourceColumns.
it('fetches columns when availableColumns is empty', async () => {
mockFetchDatasourceColumns.mockResolvedValue([
{ name: 'id', type: 'integer' },
]);
render(ConfigTabForm, {
props: {
datasourceId: 'ds-1',
environmentId: 'env-1',
availableColumns: [],
},
});
// Let $effect and async operations settle
await new Promise(r => setTimeout(r, 50));
expect(mockFetchDatasourceColumns).toHaveBeenCalledTimes(1);
expect(mockFetchDatasourceColumns).toHaveBeenCalledWith('ds-1', 'env-1');
});
// #endregion test_fetches_columns_when_empty
});
// #endregion ConfigTabFormColumnsTest.Describe
// #endregion ConfigTabFormColumnsTest

View File

@@ -443,10 +443,8 @@ export class DashboardHubModel {
// ══ Validation popover ════════════════════════════════════════
toggleValidationPopover(dashboardId: string, event: Event): void {
event.stopPropagation();
if (this.openValidationPopover === dashboardId) { this.openValidationPopover = null; return; }
const trigger = event.currentTarget as HTMLElement;
toggleValidationPopover(dashboardId: string, trigger: HTMLElement): void {
if (this.openValidationPopover === dashboardId) { this.closeValidationPopover(); return; }
if (trigger?.getBoundingClientRect) {
const rect = trigger.getBoundingClientRect();
const vw = typeof window !== "undefined" ? window.innerWidth : 1920;
@@ -455,6 +453,10 @@ export class DashboardHubModel {
this.openValidationPopover = dashboardId;
}
closeValidationPopover(): void {
this.openValidationPopover = null;
}
// ══ Grid transforms (coordinator) ════════════════════════════
applyGridTransforms(): void {