perf: fix translate deadlock, speed, trace_id, UI bugs — fullstack patch
## Backend (7 production files + 6 test files) ### P0-2: LLM output truncation cascade fix - _token_budget.py: OUTPUT_PER_ROW_PER_LANG 120→200, OUTPUT_SAFETY_FACTOR 0.70→0.55 - Prevents finish_reason=length → split → retry cascade (3 calls → 1 call per batch) - P2-8: added qwen-flash/qwen-plus/qwen-max/qwen-coder to PROVIDER_DEFAULTS ### P1-4/P1-5: EncryptionManager singleton - encryption.py: get_encryption_manager() process-wide singleton - llm_provider.py: use singleton instead of new EncryptionManager() per batch - Eliminates ~90 redundant Fernet key validations per translation run ### P1-6: Cache-hit log aggregation - _batch_proc.py: one log per batch (batch_rows + cache_hits) instead of per-row - 1076 log lines → ~30 per run ### P1-7: Timezone-aware datetime fix - scheduler.py: _ensure_aware() helper for naive DB datetime → UTC-aware - Fixes TypeError in scheduled translation concurrency check ### P2-9: Connection test timeout - connection_service.py: asyncio.wait_for(15s) on all dialect tests - Prevents 2-minute UI hangs from DNS/TCP stalls ### Trace ID propagation - middleware/trace.py: inject x-trace-id response header via ASGI send wrapper ### Test fixes & integration tests - test_scheduler.py: AsyncMock for execute_run, mock get_async_job_runner - test_sql_insert_service.py: AsyncMock for execute_sql - test_token_budget.py: batch_size 50→45 for new OUTPUT_PER_ROW_PER_LANG=200 - test_encryption.py: +2 singleton tests - test_scheduler_ensure_aware.py: +4 (naive→aware, passthrough, None, subtraction) - test_batch_classify_persist.py: +2 cache-hit aggregation tests - test_connection_service_edge.py: +2 timeout tests - test_trace_middleware.py: +4 x-trace-id header tests - test_token_budget.py: +4 qwen-flash/O200 tests ## Frontend (7 production files + 5 test files) ### Trace ID propagation - api.ts: _captureTraceId() reads x-trace-id → setTraceId() in fetchApi/requestApi/postApi/deleteApi ### Duplicate datasource columns fetch - ConfigTabForm.svelte: guard availableColumns.length === 0 before fetch ### Admin pages Svelte 5 runes fix - admin/users/+page.svelte: plain let → () for all template-bound vars - admin/roles/+page.svelte: same fix - Both pages were stuck on «Загрузка...» due to mixed reactivity models ### Validation popover positioning - +page.svelte: pass trigger HTMLElement instead of event - DashboardHubModel.svelte.ts: toggleValidationPopover(HTMLElement), closeValidationPopover() - Added X close button + click-outside overlay + i18n ### Test fixes & integration tests - api.test.ts: mock setTraceId/getTraceId, +3 _captureTraceId tests - provider_config.integration.test.ts: handleDelete→promptDeleteProvider - DatasetPreview.test.ts: dashboards/ → ROUTES.dashboards - test_config_tab_form.svelte.js: +2 columns fetch guard tests (NEW) - admin-users.test.ts: +3 loading→table tests (NEW) - admin-roles.test.ts: +2 loading→table tests (NEW) ## Semantic curation - Removed @COMPLEXITY N from 6 route files + metrics.py (duplicate of [C:N]) - Added [C:N] to 2 orphan child contracts in metrics.py - Added [C:N] + @BRIEF to 4 frontend anchors - Fixed #region → # #region consistency in validation_tasks.py ## Verification - Backend: 608 pytest passed (0 failures) - Frontend: 2472 vitest passed (128 files, 0 failures) - Frontend build: ✓ built in 18s - Browser: dashboards, admin/users, admin/roles, validation popover — all green
This commit is contained in:
@@ -25,7 +25,7 @@
|
||||
// @REJECTED Axios rejected — unnecessary dependency for a single-domain API client; native fetch + wrappers
|
||||
// is simpler, tree-shakeable, and has zero bundle cost.
|
||||
|
||||
import { log } from '$lib/cot-logger';
|
||||
import { log, setTraceId } from '$lib/cot-logger';
|
||||
import { addToast } from './toasts.svelte.js';
|
||||
import type { FetchOptions, DashboardListParams } from '../types/api';
|
||||
|
||||
@@ -226,6 +226,16 @@ function getAuthHeaders(extraHeaders: Record<string, string> = {}): Record<strin
|
||||
// @RELATION DEPENDS_ON -> [buildApiError]
|
||||
// @RELATION DEPENDS_ON -> [notifyApiError]
|
||||
// @RELATION DEPENDS_ON -> [getAuthHeaders]
|
||||
/** Extract X-Trace-Id from response headers and seed the CoT logger's trace_id. */
|
||||
function _captureTraceId(response: Response): void {
|
||||
try {
|
||||
const traceId = response.headers?.get?.('x-trace-id');
|
||||
if (traceId) setTraceId(traceId);
|
||||
} catch {
|
||||
// headers unavailable (e.g. test mock without full Headers API)
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchApi<T = unknown>(endpoint: string, options: FetchOptions = {}): Promise<T> {
|
||||
try {
|
||||
log('api', 'REASON', 'fetchApi', { endpoint });
|
||||
@@ -234,6 +244,7 @@ async function fetchApi<T = unknown>(endpoint: string, options: FetchOptions = {
|
||||
const response = await fetch(`${API_BASE_URL}${endpoint}`, fetchInit);
|
||||
if (!response.ok) throw await buildApiError(response);
|
||||
if (response.status === 204) return null as T;
|
||||
_captureTraceId(response);
|
||||
return await response.json() as T;
|
||||
} catch (error) {
|
||||
const apiError = error as ApiError;
|
||||
@@ -296,6 +307,7 @@ async function postApi<T = unknown>(endpoint: string, body: unknown, options: Fe
|
||||
const response = await fetch(`${API_BASE_URL}${endpoint}`, fetchInit);
|
||||
if (!response.ok) throw await buildApiError(response);
|
||||
if (response.status === 204) return null as T;
|
||||
_captureTraceId(response);
|
||||
return await response.json() as T;
|
||||
} catch (error) {
|
||||
const apiError = error as ApiError;
|
||||
@@ -325,6 +337,7 @@ async function deleteApi<T = unknown>(endpoint: string, options: FetchOptions =
|
||||
const response = await fetch(`${API_BASE_URL}${endpoint}`, fetchInit);
|
||||
if (!response.ok) throw await buildApiError(response);
|
||||
if (response.status === 204) return null as T;
|
||||
_captureTraceId(response);
|
||||
return await response.json() as T;
|
||||
} catch (error) {
|
||||
const apiError = error as ApiError;
|
||||
@@ -356,6 +369,7 @@ async function requestApi<T = unknown>(endpoint: string, method: string = 'GET',
|
||||
const response = await fetch(`${API_BASE_URL}${endpoint}`, fetchInit);
|
||||
if (!response.ok) throw await buildApiError(response);
|
||||
if (response.status === 204) return null as T;
|
||||
_captureTraceId(response);
|
||||
return await response.json() as T;
|
||||
} catch (error) {
|
||||
const apiError = error as ApiError;
|
||||
|
||||
@@ -33,6 +33,8 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
// Mock the cot-logger to avoid side effects
|
||||
vi.mock('$lib/cot-logger.js', () => ({
|
||||
log: vi.fn(),
|
||||
setTraceId: vi.fn(),
|
||||
getTraceId: vi.fn(() => 'test-trace-id'),
|
||||
}));
|
||||
|
||||
// Mock toasts
|
||||
@@ -672,6 +674,11 @@ describe('ApiModule — fetch wrappers (global fetch mock)', () => {
|
||||
|
||||
it('getValidationStatusBatch returns empty results stub', async () => {
|
||||
const { api } = await import('$lib/api.js');
|
||||
vi.mocked(fetch).mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: () => Promise.resolve({ results: [] }),
|
||||
} as Response);
|
||||
const result = await api.getValidationStatusBatch();
|
||||
expect(result).toEqual({ results: [] });
|
||||
});
|
||||
@@ -728,6 +735,69 @@ describe('ApiModule — fetch wrappers (global fetch mock)', () => {
|
||||
status: 404,
|
||||
});
|
||||
});
|
||||
|
||||
// #region captureTraceIdTests [C:2] [TYPE Test] [SEMANTICS test,api,trace-id,headers]
|
||||
// @BRIEF Verify _captureTraceId behavior — seeds trace_id from x-trace-id header, handles missing gracefully.
|
||||
describe('_captureTraceId', () => {
|
||||
const testUuid = 'a1b2c3d4-e5f6-7890-abcd-ef1234567890';
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('sets trace_id from x-trace-id response header', async () => {
|
||||
vi.mocked(fetch).mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: () => Promise.resolve({ data: 'ok' }),
|
||||
headers: {
|
||||
get: vi.fn((name: string) => name === 'x-trace-id' ? testUuid : null),
|
||||
},
|
||||
} as unknown as Response);
|
||||
|
||||
const cotLogger = await import('$lib/cot-logger.js');
|
||||
cotLogger.setTraceId.mockClear();
|
||||
|
||||
const { api } = await import('$lib/api.js');
|
||||
await api.fetchApi('/capture-trace-test');
|
||||
|
||||
expect(cotLogger.setTraceId).toHaveBeenCalledWith(testUuid);
|
||||
});
|
||||
|
||||
it('handles missing x-trace-id header gracefully', async () => {
|
||||
vi.mocked(fetch).mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: () => Promise.resolve({ data: 'ok' }),
|
||||
headers: {
|
||||
get: vi.fn(() => null),
|
||||
},
|
||||
} as unknown as Response);
|
||||
|
||||
const cotLogger = await import('$lib/cot-logger.js');
|
||||
cotLogger.setTraceId.mockClear();
|
||||
|
||||
const { api } = await import('$lib/api.js');
|
||||
await api.fetchApi('/no-trace-id');
|
||||
|
||||
expect(cotLogger.setTraceId).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('handles missing headers API (undefined headers) without throwing', async () => {
|
||||
vi.mocked(fetch).mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: () => Promise.resolve({ data: 'ok' }),
|
||||
} as Response);
|
||||
|
||||
const { api } = await import('$lib/api.js');
|
||||
|
||||
// Should not throw despite missing headers
|
||||
const result = await api.fetchApi('/no-headers');
|
||||
expect(result).toEqual({ data: 'ok' });
|
||||
});
|
||||
});
|
||||
// #endregion captureTraceIdTests
|
||||
});
|
||||
|
||||
describe('ApiModule — deleteValidationTask query param', () => {
|
||||
|
||||
@@ -63,10 +63,10 @@ describe('ProviderConfig edit interaction contract', () => {
|
||||
it('keeps explicit delete flow with confirmation and delete request', () => {
|
||||
const source = fs.readFileSync(COMPONENT_PATH, 'utf-8');
|
||||
|
||||
expect(source).toContain('async function handleDelete(provider)');
|
||||
expect(source).toContain('$t.llm.delete_confirm.replace("{name}", provider.name || provider.id)');
|
||||
expect(source).toContain('function promptDeleteProvider(provider)');
|
||||
expect(source).toContain('delete_confirm');
|
||||
expect(source).toContain('await requestApi(`/llm/providers/${provider.id}`, "DELETE")');
|
||||
expect(source).toContain('onclick={() => handleDelete(provider)}');
|
||||
expect(source).toContain('onclick={() => promptDeleteProvider(provider)}');
|
||||
});
|
||||
|
||||
it('excludes masked or empty api_key from submit data when editing', () => {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// #region ReportTypeProfiles:Module [TYPE Function]
|
||||
// @PURPOSE: Deterministic mapping from report task_type to visual profile with one fallback.
|
||||
// #region ReportTypeProfiles:Module [C:1] [TYPE Module]
|
||||
// @BRIEF Deterministic mapping from report task_type to visual profile with one fallback.
|
||||
|
||||
import { _ } from '$lib/i18n/index.svelte.js';
|
||||
|
||||
@@ -49,8 +49,8 @@ export const REPORT_TYPE_PROFILES: Record<string, ReportTypeProfile> = {
|
||||
}
|
||||
};
|
||||
|
||||
// #region getReportTypeProfile:Function [TYPE Function]
|
||||
// @PURPOSE: Resolve visual profile by task type with guaranteed fallback.
|
||||
// #region getReportTypeProfile:Function [C:2] [TYPE Function]
|
||||
// @BRIEF Resolve visual profile by task type with guaranteed fallback.
|
||||
// @PRE: taskType may be known/unknown/empty.
|
||||
// @POST: Returns one profile object.
|
||||
//
|
||||
|
||||
@@ -95,9 +95,10 @@
|
||||
return (virtualColumns || []).includes(colName);
|
||||
}
|
||||
|
||||
// Auto-load columns when datasourceId changes (from parent loadJob or user selection)
|
||||
// Auto-load columns when datasourceId changes (from parent loadJob or user selection).
|
||||
// Skip if columns already loaded by the model to avoid duplicate fetch.
|
||||
$effect(() => {
|
||||
if (datasourceId && environmentId) {
|
||||
if (datasourceId && environmentId && availableColumns.length === 0) {
|
||||
loadColumnsForDatasource();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
// #region ConfigTabFormColumnsTest [C:2] [TYPE Module] [SEMANTICS test,translate,config,columns,datasource]
|
||||
// @BRIEF Verify ConfigTabForm columns fetch behavior — skips fetch when columns already loaded,
|
||||
// triggers fetch when columns array is empty.
|
||||
// @RELATION BINDS_TO -> [ConfigTabForm]
|
||||
// @TEST_CONTRACT: availableColumns populated -> fetchDatasourceColumns NOT called on mount
|
||||
// @TEST_CONTRACT: availableColumns empty -> fetchDatasourceColumns IS called on mount
|
||||
// @TEST_EDGE: no datasourceId -> fetch skipped even with empty columns
|
||||
// @TEST_EDGE: no environmentId -> fetch skipped even with empty columns
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { render } from '@testing-library/svelte';
|
||||
import ConfigTabForm from '../ConfigTabForm.svelte';
|
||||
|
||||
// ── Hoisted mock factories (must be before vi.mock) ────────────────
|
||||
const mockGetAllowedLanguages = vi.hoisted(() => vi.fn().mockResolvedValue(['en', 'ru']));
|
||||
const mockFetchDatasourceColumns = vi.hoisted(() => vi.fn());
|
||||
const mockFetchDatasources = vi.hoisted(() => vi.fn());
|
||||
|
||||
// ── i18n mock ─────────────────────────────────────────────────────
|
||||
const mockTranslations = {
|
||||
translate: {
|
||||
config: {
|
||||
basic_info: 'Basic Info',
|
||||
name: 'Name',
|
||||
description: 'Description',
|
||||
name_placeholder: 'Enter name',
|
||||
description_placeholder: 'Enter description',
|
||||
help_name: 'Name of the job',
|
||||
help_description: 'Optional description',
|
||||
datasource: 'Datasource',
|
||||
environment: 'Environment',
|
||||
help_environment: 'Select the target environment',
|
||||
select_environment: '-- Select --',
|
||||
datasource_id: 'Datasource Search',
|
||||
help_datasource: 'Find and select a datasource',
|
||||
datasource_search_placeholder: 'Search datasets...',
|
||||
detected_dialect: 'Detected: {dialect}',
|
||||
available_columns: 'Available columns ({count})',
|
||||
virtual: 'virtual',
|
||||
virtual_column_warning: 'Virtual column {col}',
|
||||
column_mapping: 'Column Mapping',
|
||||
translation_column: 'Translation Column',
|
||||
help_translation_column: 'Select the column to translate',
|
||||
select_column: '-- Select column --',
|
||||
key_columns: 'Key Columns',
|
||||
help_key_columns: 'Select key columns for row matching',
|
||||
clear_all: 'Clear all',
|
||||
add_key_column: '+ Add key column',
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
vi.mock('$lib/i18n/index.svelte.js', () => ({
|
||||
getT: () => mockTranslations,
|
||||
t: {
|
||||
subscribe: (fn) => {
|
||||
fn(mockTranslations);
|
||||
return () => {};
|
||||
}
|
||||
},
|
||||
_: (key) => key,
|
||||
}));
|
||||
|
||||
// ── Languages mock ────────────────────────────────────────────────
|
||||
vi.mock('$lib/i18n/languages.js', () => ({
|
||||
ALL_LANGUAGES: [{ code: 'en', name: 'English' }, { code: 'ru', name: 'Russian' }],
|
||||
filterLanguages: (codes) => [{ code: 'en', name: 'English' }],
|
||||
}));
|
||||
|
||||
// ── API mocks ─────────────────────────────────────────────────────
|
||||
vi.mock('$lib/api.js', () => ({
|
||||
api: {
|
||||
getAllowedLanguages: mockGetAllowedLanguages,
|
||||
}
|
||||
}));
|
||||
|
||||
vi.mock('$lib/api/translate.js', () => ({
|
||||
fetchDatasourceColumns: mockFetchDatasourceColumns,
|
||||
fetchDatasources: mockFetchDatasources,
|
||||
}));
|
||||
|
||||
// ── CoT logger mock ───────────────────────────────────────────────
|
||||
vi.mock('$lib/cot-logger', () => ({
|
||||
log: vi.fn(),
|
||||
setTraceId: vi.fn(),
|
||||
getTraceId: vi.fn(() => 'test-trace-id'),
|
||||
}));
|
||||
|
||||
// #region ConfigTabFormColumnsTest.Describe [C:2] [TYPE Test]
|
||||
// @BRIEF Test the columns fetch $effect guard — availableColumns.length check.
|
||||
describe('ConfigTabForm — columns fetch guard', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
// #region test_skips_columns_fetch_when_already_loaded [C:2] [TYPE Test]
|
||||
// @BRIEF When availableColumns has items, the effect should not call fetchDatasourceColumns.
|
||||
it('skips columns fetch when columns already loaded', async () => {
|
||||
render(ConfigTabForm, {
|
||||
props: {
|
||||
datasourceId: 'ds-1',
|
||||
environmentId: 'env-1',
|
||||
availableColumns: [
|
||||
{ name: 'id', type: 'integer' },
|
||||
{ name: 'name', type: 'varchar' },
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
// Let $effect and async operations settle
|
||||
await new Promise(r => setTimeout(r, 50));
|
||||
|
||||
expect(mockFetchDatasourceColumns).not.toHaveBeenCalled();
|
||||
});
|
||||
// #endregion test_skips_columns_fetch_when_already_loaded
|
||||
|
||||
// #region test_fetches_columns_when_empty [C:2] [TYPE Test]
|
||||
// @BRIEF When availableColumns is empty [] and datasourceId+environmentId are set,
|
||||
// the effect MUST call fetchDatasourceColumns.
|
||||
it('fetches columns when availableColumns is empty', async () => {
|
||||
mockFetchDatasourceColumns.mockResolvedValue([
|
||||
{ name: 'id', type: 'integer' },
|
||||
]);
|
||||
|
||||
render(ConfigTabForm, {
|
||||
props: {
|
||||
datasourceId: 'ds-1',
|
||||
environmentId: 'env-1',
|
||||
availableColumns: [],
|
||||
},
|
||||
});
|
||||
|
||||
// Let $effect and async operations settle
|
||||
await new Promise(r => setTimeout(r, 50));
|
||||
|
||||
expect(mockFetchDatasourceColumns).toHaveBeenCalledTimes(1);
|
||||
expect(mockFetchDatasourceColumns).toHaveBeenCalledWith('ds-1', 'env-1');
|
||||
});
|
||||
// #endregion test_fetches_columns_when_empty
|
||||
});
|
||||
// #endregion ConfigTabFormColumnsTest.Describe
|
||||
// #endregion ConfigTabFormColumnsTest
|
||||
@@ -443,10 +443,8 @@ export class DashboardHubModel {
|
||||
|
||||
// ══ Validation popover ════════════════════════════════════════
|
||||
|
||||
toggleValidationPopover(dashboardId: string, event: Event): void {
|
||||
event.stopPropagation();
|
||||
if (this.openValidationPopover === dashboardId) { this.openValidationPopover = null; return; }
|
||||
const trigger = event.currentTarget as HTMLElement;
|
||||
toggleValidationPopover(dashboardId: string, trigger: HTMLElement): void {
|
||||
if (this.openValidationPopover === dashboardId) { this.closeValidationPopover(); return; }
|
||||
if (trigger?.getBoundingClientRect) {
|
||||
const rect = trigger.getBoundingClientRect();
|
||||
const vw = typeof window !== "undefined" ? window.innerWidth : 1920;
|
||||
@@ -455,6 +453,10 @@ export class DashboardHubModel {
|
||||
this.openValidationPopover = dashboardId;
|
||||
}
|
||||
|
||||
closeValidationPopover(): void {
|
||||
this.openValidationPopover = null;
|
||||
}
|
||||
|
||||
// ══ Grid transforms (coordinator) ════════════════════════════
|
||||
|
||||
applyGridTransforms(): void {
|
||||
|
||||
Reference in New Issue
Block a user