From 084f782065efe9782d464218e61b52742a3d896b Mon Sep 17 00:00:00 2001 From: busya Date: Wed, 20 May 2026 17:15:31 +0300 Subject: [PATCH] lang detect --- .axiom/axiom_config.yaml | 49 ++- backend/alembic/env.py | 3 +- ...5b4a_add_is_multimodal_to_llm_providers.py | 88 ++++ backend/requirements.txt | 2 +- backend/src/api/routes/assistant/_dispatch.py | 4 +- backend/src/api/routes/llm.py | 3 + backend/src/api/routes/tasks.py | 6 +- .../src/api/routes/translate/_job_routes.py | 5 +- backend/src/models/llm.py | 1 + backend/src/plugins/llm_analysis/models.py | 1 + backend/src/plugins/llm_analysis/plugin.py | 4 +- .../translate/__tests__/test_lang_detect.py | 10 +- .../translate/__tests__/test_target_schema.py | 378 ++++++++---------- .../translate/__tests__/test_text_cleaner.py | 186 +++++++++ backend/src/plugins/translate/_lang_detect.py | 54 ++- backend/src/plugins/translate/_llm_call.py | 41 +- .../src/plugins/translate/_text_cleaner.py | 63 +++ .../translate/service_target_schema.py | 376 +++++++++-------- .../__tests__/test_llm_plugin_persistence.py | 1 + .../services/__tests__/test_llm_provider.py | 122 ++++++ backend/src/services/llm_prompt_templates.py | 5 +- backend/src/services/llm_provider.py | 2 + .../src/components/llm/ProviderConfig.svelte | 37 +- .../translate/TargetSchemaHint.svelte | 151 ++++--- frontend/src/lib/i18n/locales/en/llm.json | 3 +- frontend/src/lib/i18n/locales/ru/llm.json | 3 +- .../routes/admin/settings/llm/+page.svelte | 16 +- .../src/routes/settings/settings-utils.js | 19 +- 28 files changed, 1103 insertions(+), 530 deletions(-) create mode 100644 backend/alembic/versions/9f8e7d6c5b4a_add_is_multimodal_to_llm_providers.py create mode 100644 backend/src/plugins/translate/__tests__/test_text_cleaner.py create mode 100644 backend/src/plugins/translate/_text_cleaner.py diff --git a/.axiom/axiom_config.yaml b/.axiom/axiom_config.yaml index 1df8322e..a717a0b3 100644 --- a/.axiom/axiom_config.yaml +++ b/.axiom/axiom_config.yaml @@ -39,29 +39,28 @@ indexing: # @BRIEF Обязательные и запрещённые тэги по уровням сложности C1-C5. # @RELATION BINDS_TO -> [Std.Semantics.Core] # @INVARIANT Каждый тэг в required/forbidden списках обязан иметь определение в TagSchema. -# @RATIONALE BRIEF/PURPOSE/RATIONALE/REJECTED — универсально опциональны (C1+). -# EXAMPLE, ERROR/RAISES, DEPRECATED, REPLACED_BY — тоже опциональны. -# @REJECTED Свободная валидация без строгих forbidden-списков отвергнута — порождает Slop. +# @RATIONALE INVARIANT, RATIONALE, REJECTED, BRIEF/PURPOSE, EXAMPLE, ERROR/RAISES — универсально опциональны (C1+). +# @PRE/@POST — обязательны на C4, но C4 с SIDE_EFFECT-only валиден если предусловия тривиальны. +# @DATA_CONTRACT — рекомендуется на C5, но не обязателен (не все C5 контракты имеют DTO-маппинг). +# @RELATION — обязателен с C3, разрешён на C2 для документирования одиночных зависимостей. +# @REJECTED Строгий запрет INVARIANT ниже C5 — заставлял раздувать контракты до C5 ради одного инварианта. +# Строгий запрет RELATION на C2 — мешал документировать простые зависимости. complexity_rules: '1': required: [] forbidden: - - RELATION - PRE - POST - SIDE_EFFECT - DATA_CONTRACT - - INVARIANT '2': required: - PURPOSE forbidden: - - RELATION - PRE - POST - SIDE_EFFECT - DATA_CONTRACT - - INVARIANT '3': required: - PURPOSE @@ -71,7 +70,6 @@ complexity_rules: - POST - SIDE_EFFECT - DATA_CONTRACT - - INVARIANT '4': required: - PURPOSE @@ -81,7 +79,6 @@ complexity_rules: - SIDE_EFFECT forbidden: - DATA_CONTRACT - - INVARIANT '5': required: - PURPOSE @@ -89,7 +86,6 @@ complexity_rules: - PRE - POST - SIDE_EFFECT - - DATA_CONTRACT - INVARIANT forbidden: [] # #endregion ComplexityRules @@ -124,7 +120,6 @@ contract_type_overrides: - POST - SIDE_EFFECT - DATA_CONTRACT - - INVARIANT '4': required: - PURPOSE @@ -135,7 +130,6 @@ contract_type_overrides: - SIDE_EFFECT forbidden: - DATA_CONTRACT - - INVARIANT '5': required: - PURPOSE @@ -144,7 +138,6 @@ contract_type_overrides: - PRE - POST - SIDE_EFFECT - - DATA_CONTRACT - INVARIANT forbidden: [] Tombstone: @@ -173,7 +166,6 @@ contract_type_overrides: - POST - SIDE_EFFECT - DATA_CONTRACT - - INVARIANT '4': required: - PURPOSE @@ -183,7 +175,6 @@ contract_type_overrides: - SIDE_EFFECT forbidden: - DATA_CONTRACT - - INVARIANT '5': required: - PURPOSE @@ -191,7 +182,6 @@ contract_type_overrides: - PRE - POST - SIDE_EFFECT - - DATA_CONTRACT - INVARIANT forbidden: [] # #endregion ContractTypeOverrides @@ -294,6 +284,15 @@ tags: protected: false orthogonal: false decision_memory: false + THROWS: + type: string + multiline: true + alias_for: ERROR + description: 'Алиас для ERROR.' + contract_types: [] + protected: false + orthogonal: false + decision_memory: false DEPRECATED: type: string multiline: true @@ -331,9 +330,11 @@ tags: type: array multiline: false separator: ',' - description: 'Семантические маркеры. Ортогональный.' + description: 'Семантические маркеры для поиска. Ортогональный.' contract_types: - Module + - Function + - Class - Block - Skill - Agent @@ -361,6 +362,10 @@ tags: - Tombstone - ADR - Module + - Function + - Class + - Component + - Block protected: false orthogonal: true decision_memory: false @@ -457,7 +462,7 @@ tags: PRE: type: string multiline: true - description: 'Предусловия. Обязателен с C4.' + description: 'Предусловия. Обязателен с C4. Если предусловия тривиальны (напр. "DB доступна"), допустимо опустить при наличии @SIDE_EFFECT.' contract_types: - Module - Function @@ -470,7 +475,7 @@ tags: POST: type: string multiline: true - description: 'Гарантии результата. Обязателен с C4.' + description: 'Гарантии результата. Обязателен с C4. Если постусловия очевидны из @PURPOSE, допустимо опустить при наличии @SIDE_EFFECT.' contract_types: - Module - Function @@ -528,7 +533,7 @@ tags: DATA_CONTRACT: type: string multiline: false - description: 'DTO-маппинг. Обязателен с C5.' + description: 'DTO-маппинг (Input→Output). Рекомендуется с C5, не обязателен. Не все C5 контракты имеют DTO.' contract_types: - Module - Function @@ -541,7 +546,7 @@ tags: INVARIANT: type: string multiline: true - description: 'Инвариант. Обязателен с C5.' + description: 'Инвариант — условие, истинное всегда. Универсально опциональный (C1+), обязателен на C5. Документируй неуничтожимые гарантии на любом уровне.' contract_types: - Module - Function @@ -556,7 +561,7 @@ tags: LAYER: type: string multiline: false - description: 'Слой: Core, Domain, API, UI, Service, Infrastructure, Plugin, Tests. Универсально опциональный.' + description: 'Слой архитектуры: Core, Domain, API, UI, Service, Infrastructure, Plugin, Tests. Универсально опциональный.' enum: [Core, Domain, API, UI, Service, Infrastructure, Plugin, Tests] contract_types: - Module diff --git a/backend/alembic/env.py b/backend/alembic/env.py index 097f7fd0..eddd2772 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -31,7 +31,6 @@ from src.models.mapping import Base # Import ALL model modules so their tables are registered in Base.metadata from src.models import ( # noqa: F401 auth, - config, connection, dashboard, dataset_review_pkg, @@ -44,6 +43,8 @@ from src.models import ( # noqa: F401 task, translate, ) +# Import config models with explicit alias to avoid name collision with alembic config +import src.models.config as models_config # noqa: F401 target_metadata = Base.metadata diff --git a/backend/alembic/versions/9f8e7d6c5b4a_add_is_multimodal_to_llm_providers.py b/backend/alembic/versions/9f8e7d6c5b4a_add_is_multimodal_to_llm_providers.py new file mode 100644 index 00000000..092f42b7 --- /dev/null +++ b/backend/alembic/versions/9f8e7d6c5b4a_add_is_multimodal_to_llm_providers.py @@ -0,0 +1,88 @@ +"""Add is_multimodal column to llm_providers with heuristic backfill + +Revision ID: 9f8e7d6c5b4a +Revises: c4a3a2f74bfe +Create Date: 2026-05-20 14:30:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "9f8e7d6c5b4a" +down_revision: Union[str, Sequence[str], None] = "b1c2d3e4f5a6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _is_multimodal_heuristic(model_name: str) -> bool: + """Backfill heuristic: mirrors the logic from services/llm_prompt_templates.py + at the time of migration creation. Returns True for known multimodal/vision models. + """ + token = (model_name or "").strip().lower() + if not token: + return False + text_only_markers = ( + "text-only", + "embedding", + "rerank", + "whisper", + "tts", + "transcribe", + ) + if any(marker in token for marker in text_only_markers): + return False + multimodal_markers = ( + "gpt-4o", + "gpt-4.1", + "vision", + "vl", + "gemini", + "claude-3", + "claude-sonnet-4", + "omni", + "multimodal", + "pixtral", + "llava", + "internvl", + "qwen-vl", + "qwen2-vl", + ) + return any(marker in token for marker in multimodal_markers) + + +def upgrade() -> None: + """Add is_multimodal column and backfill using heuristic.""" + # Step 1: Add column as nullable first + op.add_column( + "llm_providers", + sa.Column("is_multimodal", sa.Boolean(), nullable=True), + ) + + # Step 2: Backfill existing rows using heuristic on default_model + connection = op.get_bind() + providers = connection.execute( + sa.text("SELECT id, default_model FROM llm_providers") + ).fetchall() + + for row in providers: + multimodal = _is_multimodal_heuristic(row.default_model) + connection.execute( + sa.text( + "UPDATE llm_providers SET is_multimodal = :multimodal WHERE id = :id" + ), + {"multimodal": multimodal, "id": row.id}, + ) + + # Step 3: Make column non-nullable with default False + # Use text("false") for PostgreSQL compatibility (not "0") + op.alter_column("llm_providers", "is_multimodal", nullable=False, server_default=sa.text("false")) + + +def downgrade() -> None: + """Remove is_multimodal column.""" + op.drop_column("llm_providers", "is_multimodal") diff --git a/backend/requirements.txt b/backend/requirements.txt index b23cc3b0..6ad85ef6 100755 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -56,4 +56,4 @@ playwright tenacity Pillow ruff>=0.11.0 -lingua-language-detector==2.2.0 +lingua-language-detector==2.1.1 diff --git a/backend/src/api/routes/assistant/_dispatch.py b/backend/src/api/routes/assistant/_dispatch.py index c0dea52b..165e6034 100644 --- a/backend/src/api/routes/assistant/_dispatch.py +++ b/backend/src/api/routes/assistant/_dispatch.py @@ -19,7 +19,6 @@ from src.core.logger import belief_scope, logger from src.core.task_manager import TaskManager from src.schemas.auth import User from src.services.git_service import GitService -from src.services.llm_prompt_templates import is_multimodal_model from src.services.llm_provider import LLMProviderService from ._dataset_review_dispatch import _dispatch_dataset_review_intent @@ -281,8 +280,7 @@ async def _dispatch_intent(intent: dict[str, Any], current_user: User, task_mana if not dashboard_id or not env_id or (not provider_id): raise HTTPException(status_code=422, detail='Missing dashboard_id/environment/provider. Укажите ID/slug дашборда или окружение.') provider = LLMProviderService(db).get_provider(provider_id) - provider_model = provider.default_model if provider else '' - if not is_multimodal_model(provider_model, provider.provider_type if provider else None): + if not provider or not bool(provider.is_multimodal): raise HTTPException(status_code=422, detail='Selected provider model is not multimodal for dashboard validation. Выберите мультимодальную модель (например, gpt-4o).') task = await task_manager.create_task(plugin_id='llm_dashboard_validation', params={'dashboard_id': str(dashboard_id), 'environment_id': env_id, 'provider_id': provider_id}, user_id=current_user.id) logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') diff --git a/backend/src/api/routes/llm.py b/backend/src/api/routes/llm.py index 80b6cb10..962b461c 100644 --- a/backend/src/api/routes/llm.py +++ b/backend/src/api/routes/llm.py @@ -83,6 +83,7 @@ async def get_providers( api_key=mask_api_key(service.get_decrypted_api_key(p.id)) if p.api_key else "", default_model=p.default_model, is_active=p.is_active, + is_multimodal=bool(p.is_multimodal) if p.is_multimodal is not None else False, ) for p in providers ] @@ -268,6 +269,7 @@ async def create_provider( api_key=mask_api_key(config.api_key), default_model=provider.default_model, is_active=provider.is_active, + is_multimodal=bool(provider.is_multimodal) if provider.is_multimodal is not None else False, ) @@ -303,6 +305,7 @@ async def update_provider( api_key=mask_api_key(service.get_decrypted_api_key(provider.id)) if provider.api_key else "", default_model=provider.default_model, is_active=provider.is_active, + is_multimodal=bool(provider.is_multimodal) if provider.is_multimodal is not None else False, ) diff --git a/backend/src/api/routes/tasks.py b/backend/src/api/routes/tasks.py index 81184a3a..2a56224b 100755 --- a/backend/src/api/routes/tasks.py +++ b/backend/src/api/routes/tasks.py @@ -21,7 +21,6 @@ from ...dependencies import ( has_permission, ) from ...services.llm_prompt_templates import ( - is_multimodal_model, normalize_llm_settings, resolve_bound_provider_id, ) @@ -104,10 +103,7 @@ async def create_task( raise ValueError(f"LLM Provider {provider_id} not found") if ( request.plugin_id == "llm_dashboard_validation" - and not is_multimodal_model( - db_provider.default_model, - db_provider.provider_type, - ) + and not bool(db_provider.is_multimodal) ): raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, diff --git a/backend/src/api/routes/translate/_job_routes.py b/backend/src/api/routes/translate/_job_routes.py index 4428646f..907b592b 100644 --- a/backend/src/api/routes/translate/_job_routes.py +++ b/backend/src/api/routes/translate/_job_routes.py @@ -250,11 +250,8 @@ async def get_datasource_columns( # #endregion get_datasource_columns -# #region check_target_schema [C:3] [TYPE Function] +# #region check_target_schema [C:3] [TYPE Function] [SEMANTICS api,translate,validate] # @BRIEF Проверяет схему целевой таблицы: какие колонки ожидаются, какие есть, каких не хватает. -# @PRE User has translate.job.view permission. -# @POST Возвращает diff expected vs actual columns. -# @SIDE_EFFECT Делает SQL-запрос к information_schema.columns через Superset SQL Lab. # @RELATION DEPENDS_ON -> [validate_target_table_schema] # @RELATION DEPENDS_ON -> [TargetSchemaValidationRequest] # @RELATION DEPENDS_ON -> [TargetSchemaValidationResponse] diff --git a/backend/src/models/llm.py b/backend/src/models/llm.py index 6ddc84b1..8e1f088d 100644 --- a/backend/src/models/llm.py +++ b/backend/src/models/llm.py @@ -46,6 +46,7 @@ class LLMProvider(Base): api_key = Column(String, nullable=False) # Should be encrypted default_model = Column(String, nullable=False) is_active = Column(Boolean, default=True) + is_multimodal = Column(Boolean, default=False) created_at = Column(DateTime, default=datetime.utcnow) # #endregion LLMProvider diff --git a/backend/src/plugins/llm_analysis/models.py b/backend/src/plugins/llm_analysis/models.py index fb6c9f2f..27c0d852 100644 --- a/backend/src/plugins/llm_analysis/models.py +++ b/backend/src/plugins/llm_analysis/models.py @@ -28,6 +28,7 @@ class LLMProviderConfig(BaseModel): api_key: str | None = None default_model: str is_active: bool = True + is_multimodal: bool = False # #endregion LLMProviderConfig # #region ValidationStatus [TYPE Class] diff --git a/backend/src/plugins/llm_analysis/plugin.py b/backend/src/plugins/llm_analysis/plugin.py index 8f4ce6cd..7f10cf10 100644 --- a/backend/src/plugins/llm_analysis/plugin.py +++ b/backend/src/plugins/llm_analysis/plugin.py @@ -22,7 +22,6 @@ from ...core.task_manager.context import TaskContext from ...models.llm import ValidationPolicy, ValidationRecord from ...services.llm_prompt_templates import ( DEFAULT_LLM_PROMPTS, - is_multimodal_model, normalize_llm_settings, render_prompt, ) @@ -140,7 +139,8 @@ class DashboardValidationPlugin(PluginBase): llm_log.debug(f" Base URL: {db_provider.base_url}") llm_log.debug(f" Default Model: {db_provider.default_model}") llm_log.debug(f" Is Active: {db_provider.is_active}") - if not is_multimodal_model(db_provider.default_model, db_provider.provider_type): + llm_log.debug(f" Is Multimodal: {db_provider.is_multimodal}") + if not bool(db_provider.is_multimodal): raise ValueError( "Dashboard validation requires a multimodal model (image input support)." ) diff --git a/backend/src/plugins/translate/__tests__/test_lang_detect.py b/backend/src/plugins/translate/__tests__/test_lang_detect.py index 60340ab3..da24aa14 100644 --- a/backend/src/plugins/translate/__tests__/test_lang_detect.py +++ b/backend/src/plugins/translate/__tests__/test_lang_detect.py @@ -1,14 +1,6 @@ # #region LanguageDetectServiceTests [C:3] [TYPE Module] [SEMANTICS test, language, detection, heuristic] # @BRIEF Unit tests for LanguageDetectService — local language detection via lingua. -# @LAYER Test -# @RELATION BINDS_TO -> [LanguageDetectService:Module] -# @TEST_CONTRACT LanguageDetectService -> detect_language, build_detector, get_detector, batch_detect -# @TEST_EDGE empty_text -> returns "und" -# @TEST_EDGE numeric_text -> returns "und" -# @TEST_EDGE short_text -> correctly detects language -# @TEST_EDGE same_language_pre_filter -> batch_detect returns target language for matching rows -# @TEST_EDGE detector_cache -> get_detector returns same instance for same language set -# @TEST_EDGE cross_language -> English text with Russian detector returns "und" +# @RELATION BINDS_TO -> [LanguageDetectService] import pytest diff --git a/backend/src/plugins/translate/__tests__/test_target_schema.py b/backend/src/plugins/translate/__tests__/test_target_schema.py index 9cff6557..e6a8d5ff 100644 --- a/backend/src/plugins/translate/__tests__/test_target_schema.py +++ b/backend/src/plugins/translate/__tests__/test_target_schema.py @@ -1,20 +1,12 @@ # region TargetSchemaValidationTests [TYPE Module] # @SEMANTICS: test, translate, target-schema, validation -# @PURPOSE: Tests for target table schema validation: _build_expected_columns, _parse_column_results, validate_target_table_schema. +# @PURPOSE: Tests for target table schema validation. # @LAYER: Test # @RELATION: BINDS_TO -> [TargetSchemaValidation] # @TEST_CONTRACT: _build_expected_columns -> list[TargetSchemaColumnInfo] -# @TEST_CONTRACT: _parse_column_results -> (list[dict], bool) +# @TEST_CONTRACT: _extract_columns_from_rows -> list[dict] +# @TEST_CONTRACT: _parse_sqllab_result -> (list[dict], bool) # @TEST_CONTRACT: validate_target_table_schema -> TargetSchemaValidationResponse -# @TEST_EDGE: missing_target_key_cols -> Only context + is_original returned -# @TEST_EDGE: deduplication -> Duplicate column names are removed -# @TEST_EDGE: empty_result -> Returns ([], False) -# @TEST_EDGE: superset_sync_format -> Standard Superset response format parsed -# @TEST_EDGE: superset_async_format -> Async polling response format parsed -# @TEST_EDGE: get_query_results_format -> get_query_results response format parsed -# @TEST_EDGE: executor_failure -> SQL Lab query fails, error returned -# @TEST_EDGE: sql_injection_table_name -> Invalid table name rejected with error -# @TEST_EDGE: executor_exception -> Exception during execution returns error response from unittest.mock import MagicMock, patch @@ -26,13 +18,13 @@ from src.schemas.translate import ( ) from src.plugins.translate.service_target_schema import ( _build_expected_columns, - _parse_column_results, + _extract_columns_from_rows, + _parse_sqllab_result, validate_target_table_schema, ) # region build_request [TYPE Function] -# @PURPOSE: Helper to build a TargetSchemaValidationRequest with defaults. @pytest.fixture def build_request(): def _build(**overrides): @@ -58,21 +50,14 @@ def build_request(): # region TestBuildExpectedColumns [TYPE Class] -# @PURPOSE: Tests for _build_expected_columns logic. class TestBuildExpectedColumns: - # region test_basic_case [TYPE Function] - # @PURPOSE: target_key_cols + target_column produces id, translated_text, context, is_original. def test_basic_case(self, build_request) -> None: req = build_request() result = _build_expected_columns(req) names = [c.name for c in result] assert names == ["id", "translated_text", "context", "is_original"] - # endregion test_basic_case - - # region test_all_fields [TYPE Function] - # @PURPOSE: All optional column fields produce 7 columns. def test_all_fields(self, build_request) -> None: req = build_request( target_language_column="lang", @@ -90,20 +75,12 @@ class TestBuildExpectedColumns: assert "context" in names assert "is_original" in names - # endregion test_all_fields - - # region test_empty_target_key_cols [TYPE Function] - # @PURPOSE: Empty target_key_cols produces just translated_text, context, is_original. def test_empty_target_key_cols(self, build_request) -> None: req = build_request(target_key_cols=[]) result = _build_expected_columns(req) names = [c.name for c in result] assert names == ["translated_text", "context", "is_original"] - # endregion test_empty_target_key_cols - - # region test_translation_column_fallback [TYPE Function] - # @PURPOSE: When target_column is None, translation_column is used as fallback. def test_translation_column_fallback(self, build_request) -> None: req = build_request(target_column=None, translation_column="source_col") result = _build_expected_columns(req) @@ -111,10 +88,6 @@ class TestBuildExpectedColumns: assert "source_col" in names assert names[1] == "source_col" - # endregion test_translation_column_fallback - - # region test_dedup_same_names [TYPE Function] - # @PURPOSE: Duplicate column names are removed preserving order. def test_dedup_same_names(self, build_request) -> None: req = build_request( target_key_cols=["id", "id"], @@ -126,10 +99,6 @@ class TestBuildExpectedColumns: assert names.count("id") == 1 assert names.count("name") == 1 - # endregion test_dedup_same_names - - # region test_empty_request [TYPE Function] - # @PURPOSE: No key cols and no target/translation column produces just context + is_original. def test_empty_request(self, build_request) -> None: req = build_request( target_key_cols=[], @@ -140,10 +109,6 @@ class TestBuildExpectedColumns: names = [c.name for c in result] assert names == ["context", "is_original"] - # endregion test_empty_request - - # region test_column_order_preserved [TYPE Function] - # @PURPOSE: Column order is preserved: key_cols, target, lang, source, src_lang, context, is_original. def test_column_order_preserved(self, build_request) -> None: req = build_request( target_key_cols=["pk1", "pk2"], @@ -159,27 +124,19 @@ class TestBuildExpectedColumns: "src_col", "src_lang_col", "context", "is_original", ] assert names == expected_order - - # endregion test_column_order_preserved # endregion TestBuildExpectedColumns -# region TestParseColumnResults [TYPE Class] -# @PURPOSE: Tests for _parse_column_results response parsing. -class TestParseColumnResults: +# region TestExtractColumnsFromRows [TYPE Class] +class TestExtractColumnsFromRows: - # region test_superset_sync_format [TYPE Function] - # @PURPOSE: Standard Superset sync mode (columns + data directly in response). - def test_superset_sync_format(self) -> None: - result = { - "columns": [ - {"name": "id", "type": "integer", "is_nullable": "YES"}, - {"name": "name", "type": "varchar", "is_nullable": "NO"}, - ], - "data": [{"id": 1, "name": "test"}], - } - cols, exists = _parse_column_results({"raw_response": result}) - assert exists is True + def test_postgres_information_schema(self) -> None: + """PostgreSQL information_schema формат: column_name, data_type, is_nullable.""" + rows = [ + {"column_name": "id", "data_type": "integer", "is_nullable": "YES"}, + {"column_name": "name", "data_type": "varchar", "is_nullable": "NO"}, + ] + cols = _extract_columns_from_rows(rows) assert len(cols) == 2 assert cols[0]["name"] == "id" assert cols[0]["type"] == "integer" @@ -188,166 +145,152 @@ class TestParseColumnResults: assert cols[1]["type"] == "varchar" assert cols[1]["is_nullable"] is False - # endregion test_superset_sync_format + def test_clickhouse_describe_table(self) -> None: + """ClickHouse DESCRIBE TABLE формат: name, type.""" + rows = [ + {"name": "report_date", "type": "Date"}, + {"name": "comment_text", "type": "String"}, + {"name": "is_original", "type": "UInt8"}, + ] + cols = _extract_columns_from_rows(rows) + assert len(cols) == 3 + assert cols[0]["name"] == "report_date" + assert cols[0]["type"] == "Date" + assert cols[2]["name"] == "is_original" + assert cols[2]["type"] == "UInt8" - # region test_async_polling_format [TYPE Function] - # @PURPOSE: Async polling format (result -> columns + data). - def test_async_polling_format(self) -> None: + def test_mixed_keys(self) -> None: + """Разные ключи в одном наборе — приоритет column_name > name > Field.""" + rows = [ + {"Field": "id", "Type": "int"}, + {"name": "name", "type": "text"}, + {"column_name": "value", "data_type": "float"}, + ] + cols = _extract_columns_from_rows(rows) + assert len(cols) == 3 + # column_name имеет приоритет, потом name, потом Field + assert cols[0]["name"] == "id" + assert cols[1]["name"] == "name" + assert cols[2]["name"] == "value" + + def test_empty_rows(self) -> None: + cols = _extract_columns_from_rows([]) + assert cols == [] + + def test_invalid_row_type(self) -> None: + cols = _extract_columns_from_rows([123, "str", None]) + assert cols == [] + + def test_row_without_name_keys(self) -> None: + cols = _extract_columns_from_rows([{"foo": "bar"}]) + assert cols == [] +# endregion TestExtractColumnsFromRows + + +# region TestParseSqlLabResult [TYPE Class] +class TestParseSqlLabResult: + + def test_sync_format_with_data(self) -> None: + """Формат 1: прямой ответ с data.""" result = { - "result": { - "columns": [ - {"name": "col1", "type": "text"}, - ], + "raw_response": { + "columns": [{"name": "col1"}], "data": [{"col1": "val1"}], } } - cols, exists = _parse_column_results({"raw_response": result}) + rows, exists = _parse_sqllab_result(result) assert exists is True - assert len(cols) == 1 - assert cols[0]["name"] == "col1" + assert len(rows) == 1 + assert rows[0]["col1"] == "val1" - # endregion test_async_polling_format - - # region test_get_query_results_format [TYPE Function] - # @PURPOSE: get_query_results format (results -> columns + data). - def test_get_query_results_format(self) -> None: + def test_async_format(self) -> None: + """Формат 2: вложенный result.""" result = { - "results": { - "columns": [ - {"name": "a", "type": "int"}, - ], - "data": [{"a": 1}], - } - } - cols, exists = _parse_column_results({"raw_response": result}) - assert exists is True - assert len(cols) == 1 - assert cols[0]["name"] == "a" - - # endregion test_get_query_results_format - - # region test_deeply_nested_get_query_results [TYPE Function] - # @PURPOSE: get_query_results with nested result dict (results -> result -> columns + data). - def test_deeply_nested_get_query_results(self) -> None: - result = { - "results": { + "raw_response": { "result": { - "columns": [ - {"name": "x", "type": "float"}, - ], - "data": [{"x": 1.0}], + "columns": [{"name": "col1"}], + "data": [{"col1": "val1"}], } } } - cols, exists = _parse_column_results({"raw_response": result}) + rows, exists = _parse_sqllab_result(result) assert exists is True - assert len(cols) == 1 - assert cols[0]["name"] == "x" + assert len(rows) == 1 - # endregion test_deeply_nested_get_query_results + def test_get_query_results_format(self) -> None: + """Формат 3: results → data.""" + result = { + "raw_response": { + "results": { + "columns": [{"name": "col1"}], + "data": [{"col1": "val1"}], + } + } + } + rows, exists = _parse_sqllab_result(result) + assert exists is True + assert len(rows) == 1 - # region test_empty_result [TYPE Function] - # @PURPOSE: Empty result dict returns ([], False). - def test_empty_result(self) -> None: - cols, exists = _parse_column_results({"raw_response": {}}) - assert cols == [] + def test_empty_data(self) -> None: + result = {"raw_response": {"columns": [{"name": "c"}], "data": []}} + rows, exists = _parse_sqllab_result(result) assert exists is False + assert rows == [] - # endregion test_empty_result + def test_no_data_key(self) -> None: + result = {"raw_response": {"columns": [{"name": "c"}]}} + rows, exists = _parse_sqllab_result(result) + assert exists is False + assert rows == [] - # region test_none_result [TYPE Function] - # @PURPOSE: None result returns ([], False) via early guard. def test_none_result(self) -> None: - cols, exists = _parse_column_results(None) - assert cols == [] + rows, exists = _parse_sqllab_result(None) + assert rows == [] assert exists is False - # endregion test_none_result - - # region test_raw_response_none [TYPE Function] - # @PURPOSE: raw_response=None causes TypeError — known production gap (not guarded). def test_raw_response_none(self) -> None: - with pytest.raises(TypeError): - _parse_column_results({"raw_response": None}) - - # endregion test_raw_response_none - - # region test_result_without_columns [TYPE Function] - # @PURPOSE: Result without columns key returns ([], False). - def test_result_without_columns(self) -> None: - result = {"status": "success", "data": []} - cols, exists = _parse_column_results({"raw_response": result}) - assert cols == [] + rows, exists = _parse_sqllab_result({"raw_response": None}) + assert rows == [] assert exists is False - # endregion test_result_without_columns - - # region test_table_exists_with_data [TYPE Function] - # @PURPOSE: Non-empty data means table exists. - def test_table_exists_with_data(self) -> None: + def test_list_based_data(self) -> None: + """Строки в виде списков вместо словарей (старые версии Superset).""" result = { - "columns": [{"name": "c"}], - "data": [{"c": "v"}], + "raw_response": { + "columns": [{"name": "id"}, {"name": "name"}], + "data": [[1, "foo"], [2, "bar"]], + } } - _, exists = _parse_column_results({"raw_response": result}) + rows, exists = _parse_sqllab_result(result) assert exists is True - - # endregion test_table_exists_with_data - - # region test_table_not_exists_empty_data [TYPE Function] - # @PURPOSE: Empty data means table does not exist. - def test_table_not_exists_empty_data(self) -> None: - result = { - "columns": [{"name": "c"}], - "data": [], - } - _, exists = _parse_column_results({"raw_response": result}) - assert exists is False - - # endregion test_table_not_exists_empty_data - - # region test_alternate_column_key_names [TYPE Function] - # @PURPOSE: Columns with alternate key names (column_name, Column) parsed correctly. - def test_alternate_column_key_names(self) -> None: - result = { - "columns": [ - {"column_name": "id", "data_type": "int"}, - {"Column": "name", "Type": "text"}, - ], - "data": [], - } - cols, _ = _parse_column_results({"raw_response": result}) - assert len(cols) == 2 - assert cols[0]["name"] == "id" - assert cols[1]["name"] == "name" - - # endregion test_alternate_column_key_names -# endregion TestParseColumnResults + assert len(rows) == 2 + assert rows[0]["id"] == 1 + assert rows[0]["name"] == "foo" +# endregion TestParseSqlLabResult # region TestValidateTargetTableSchema [TYPE Class] -# @PURPOSE: Tests for validate_target_table_schema with mocked Superset executor. class TestValidateTargetTableSchema: - # region test_all_columns_present [TYPE Function] - # @PURPOSE: All expected columns present in target table. @patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor") def test_all_columns_present(self, mock_executor, build_request) -> None: + """Все ожидаемые колонки присутствуют в таблице.""" req = build_request() config_manager = MagicMock() instance = mock_executor.return_value instance.resolve_database_id.return_value = None + instance.get_database_backend.return_value = "postgresql" instance.execute_and_poll.return_value = { "status": "success", "raw_response": { - "columns": [ - {"name": "id", "type": "integer", "is_nullable": "YES"}, - {"name": "translated_text", "type": "text", "is_nullable": "YES"}, - {"name": "context", "type": "text", "is_nullable": "YES"}, - {"name": "is_original", "type": "boolean", "is_nullable": "YES"}, + "columns": [{"name": "column_name"}, {"name": "data_type"}], + "data": [ + {"column_name": "id", "data_type": "integer"}, + {"column_name": "translated_text", "data_type": "text"}, + {"column_name": "context", "data_type": "text"}, + {"column_name": "is_original", "data_type": "boolean"}, ], - "data": [{"id": 1}], }, } @@ -359,25 +302,23 @@ class TestValidateTargetTableSchema: assert len(response.expected_columns) == 4 assert len(response.actual_columns) == 4 - # endregion test_all_columns_present - - # region test_missing_columns [TYPE Function] - # @PURPOSE: Target table is missing some expected columns. @patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor") def test_missing_columns(self, mock_executor, build_request) -> None: + """В таблице не хватает некоторых ожидаемых колонок.""" req = build_request() config_manager = MagicMock() instance = mock_executor.return_value instance.resolve_database_id.return_value = None + instance.get_database_backend.return_value = "postgresql" instance.execute_and_poll.return_value = { "status": "success", "raw_response": { - "columns": [ - {"name": "id", "type": "integer", "is_nullable": "YES"}, - {"name": "context", "type": "text", "is_nullable": "YES"}, + "columns": [{"name": "column_name"}], + "data": [ + {"column_name": "id"}, + {"column_name": "context"}, ], - "data": [{"id": 1}], }, } @@ -389,17 +330,54 @@ class TestValidateTargetTableSchema: assert "is_original" in missing_names assert "id" not in missing_names - # endregion test_missing_columns + @patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor") + def test_clickhouse_describe_table(self, mock_executor, build_request) -> None: + """ClickHouse использует DESCRIBE TABLE — другой формат ответа.""" + req = build_request( + target_key_cols=["report_date", "document_number"], + target_column="comment_text", + target_language_column="lang_code", + target_source_column="source_text", + target_source_language_column="src_lang", + ) + config_manager = MagicMock() + + instance = mock_executor.return_value + instance.resolve_database_id.return_value = None + instance.get_database_backend.return_value = "clickhouse" + instance.execute_and_poll.return_value = { + "status": "success", + "raw_response": { + "columns": [{"name": "name"}, {"name": "type"}], + "data": [ + {"name": "report_date", "type": "Date"}, + {"name": "document_number", "type": "String"}, + {"name": "lang_code", "type": "String"}, + {"name": "source_text", "type": "String"}, + {"name": "src_lang", "type": "String"}, + {"name": "comment_text", "type": "String"}, + {"name": "context", "type": "String"}, + {"name": "is_original", "type": "UInt8"}, + ], + }, + } + + response = validate_target_table_schema(req, config_manager) + assert response.table_exists is True + assert response.error is None + assert response.all_present is True + assert len(response.missing_columns) == 0 + assert len(response.actual_columns) == 8 - # region test_superset_failed_status [TYPE Function] - # @PURPOSE: Superset returns failed status. @patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor") def test_superset_failed_status(self, mock_executor, build_request) -> None: + """Superset возвращает failed статус.""" req = build_request() config_manager = MagicMock() instance = mock_executor.return_value instance.resolve_database_id.return_value = None + instance.get_database_backend.return_value = "postgresql" instance.execute_and_poll.return_value = { "status": "failed", "error_message": "Query timeout", @@ -411,17 +389,15 @@ class TestValidateTargetTableSchema: assert "Query timeout" in response.error assert response.all_present is False - # endregion test_superset_failed_status - - # region test_executor_exception [TYPE Function] - # @PURPOSE: Executor throws an exception. @patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor") def test_executor_exception(self, mock_executor, build_request) -> None: + """Executor бросает исключение.""" req = build_request() config_manager = MagicMock() instance = mock_executor.return_value instance.resolve_database_id.return_value = None + instance.get_database_backend.return_value = "postgresql" instance.execute_and_poll.side_effect = ConnectionError("Superset API unavailable") response = validate_target_table_schema(req, config_manager) @@ -430,11 +406,8 @@ class TestValidateTargetTableSchema: assert "Superset API unavailable" in response.error assert response.all_present is False - # endregion test_executor_exception - - # region test_sql_injection_table_name [TYPE Function] - # @PURPOSE: Invalid table name with space is rejected with error. def test_sql_injection_table_name(self, build_request) -> None: + """Невалидное имя таблицы с пробелом — ошибка.""" req = build_request(target_table="my table; DROP TABLE students; --") config_manager = MagicMock() @@ -444,11 +417,8 @@ class TestValidateTargetTableSchema: assert "Invalid target table name" in response.error assert response.all_present is False - # endregion test_sql_injection_table_name - - # region test_sql_injection_schema_name [TYPE Function] - # @PURPOSE: Invalid schema name with DROP is rejected with error. def test_sql_injection_schema_name(self, build_request) -> None: + """Невалидное имя схемы с DROP — ошибка.""" req = build_request(target_schema="public; DROP TABLE students; --") config_manager = MagicMock() @@ -458,23 +428,19 @@ class TestValidateTargetTableSchema: assert "Invalid target schema name" in response.error assert response.all_present is False - # endregion test_sql_injection_schema_name - - # region test_table_not_exists [TYPE Function] - # @PURPOSE: Target table does not exist (empty data from information_schema). @patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor") def test_table_not_exists(self, mock_executor, build_request) -> None: + """Таблица не существует — пустой data.""" req = build_request() config_manager = MagicMock() instance = mock_executor.return_value instance.resolve_database_id.return_value = None + instance.get_database_backend.return_value = "postgresql" instance.execute_and_poll.return_value = { "status": "success", "raw_response": { - "columns": [ - {"name": "id", "type": "integer"}, - ], + "columns": [{"name": "column_name"}], "data": [], }, } @@ -483,8 +449,6 @@ class TestValidateTargetTableSchema: assert response.table_exists is False assert response.error is None assert response.all_present is False - assert len(response.missing_columns) == 3 # translated_text, context, is_original - - # endregion test_table_not_exists + assert len(response.missing_columns) == 4 # id, translated_text, context, is_original # endregion TestValidateTargetTableSchema # endregion TargetSchemaValidationTests diff --git a/backend/src/plugins/translate/__tests__/test_text_cleaner.py b/backend/src/plugins/translate/__tests__/test_text_cleaner.py new file mode 100644 index 00000000..073e2812 --- /dev/null +++ b/backend/src/plugins/translate/__tests__/test_text_cleaner.py @@ -0,0 +1,186 @@ +# #region TestTextCleaner [C:3] [TYPE Module] [SEMANTICS test, text, cleaner, whitespace, truncation] +# @BRIEF Verify text cleaning contracts — normalize_whitespace, truncate_text, clean_text. +# @RELATION BINDS_TO -> [_text_cleaner:Module] +# @TEST_EDGE: empty_string — empty/whitespace-only input returns "" +# @TEST_EDGE: whitespace_variants — multiple spaces, newlines, tabs all collapse to single space +# @TEST_EDGE: truncation_boundary — text shorter than, equal to, and longer than max_length +# @TEST_EDGE: clean_combined — clean_text correctly chains normalize + truncate +# @TEST_EDGE: zero_max_length — max_length=0 forces truncation on any non-empty text + +from src.plugins.translate._text_cleaner import normalize_whitespace, truncate_text, clean_text + + +# region TestNormalizeWhitespace [TYPE Class] +# @BRIEF Test suite for normalize_whitespace. +class TestNormalizeWhitespace: + + # region test_empty_string [TYPE Function] + # @BRIEF Empty string returns empty string. + def test_empty_string(self): + assert normalize_whitespace("") == "" + # endregion test_empty_string + + # region test_whitespace_only [TYPE Function] + # @BRIEF String with only whitespace returns empty string. + def test_whitespace_only(self): + assert normalize_whitespace(" ") == "" + assert normalize_whitespace("\t") == "" + assert normalize_whitespace("\n") == "" + # endregion test_whitespace_only + + # region test_multiple_spaces [TYPE Function] + # @BRIEF Multiple spaces between words are collapsed to single space. + def test_multiple_spaces(self): + assert normalize_whitespace("hello world") == "hello world" + # endregion test_multiple_spaces + + # region test_multiple_newlines [TYPE Function] + # @BRIEF Multiple newlines are collapsed to single space. + def test_multiple_newlines(self): + assert normalize_whitespace("hello\n\n\nworld") == "hello world" + # endregion test_multiple_newlines + + # region test_mixed_whitespace [TYPE Function] + # @BRIEF Tabs, newlines, and spaces all collapse to single space. + def test_mixed_whitespace(self): + assert normalize_whitespace("hello\t \n world") == "hello world" + # endregion test_mixed_whitespace + + # region test_leading_trailing_whitespace [TYPE Function] + # @BRIEF Leading and trailing whitespace is trimmed. + def test_leading_trailing_whitespace(self): + assert normalize_whitespace(" hello world ") == "hello world" + assert normalize_whitespace("\nhello world\n") == "hello world" + # endregion test_leading_trailing_whitespace + + # region test_already_clean [TYPE Function] + # @BRIEF Already clean text is returned unchanged. + def test_already_clean(self): + assert normalize_whitespace("hello world") == "hello world" + # endregion test_already_clean + + # region test_single_word [TYPE Function] + # @BRIEF Single word with surrounding whitespace is trimmed to the word. + def test_single_word(self): + assert normalize_whitespace(" hello ") == "hello" + # endregion test_single_word + +# endregion TestNormalizeWhitespace + + +# region TestTruncateText [TYPE Class] +# @BRIEF Test suite for truncate_text. +class TestTruncateText: + + # region test_empty_string [TYPE Function] + # @BRIEF Empty string stays empty regardless of max_length. + def test_empty_string(self): + assert truncate_text("") == "" + assert truncate_text("", max_length=0) == "" + # endregion test_empty_string + + # region test_shorter_than_max [TYPE Function] + # @BRIEF Text shorter than max_length is returned unchanged. + def test_shorter_than_max(self): + assert truncate_text("hi", max_length=5) == "hi" + # endregion test_shorter_than_max + + # region test_exactly_at_max [TYPE Function] + # @BRIEF Text exactly at max_length is returned unchanged (no "..." appended). + def test_exactly_at_max(self): + assert truncate_text("hello", max_length=5) == "hello" + # endregion test_exactly_at_max + + # region test_longer_than_max [TYPE Function] + # @BRIEF Text longer than max_length is truncated with "..." appended. + def test_longer_than_max(self): + result = truncate_text("hello world", max_length=5) + assert result == "hello..." + assert len(result) == 8 # 5 chars + "..." + # endregion test_longer_than_max + + # region test_zero_max_length [TYPE Function] + # @BRIEF max_length=0 truncates any non-empty text to "...". + def test_zero_max_length(self): + assert truncate_text("hello", max_length=0) == "..." + # endregion test_zero_max_length + + # region test_default_max_length [TYPE Function] + # @BRIEF Default max_length of 500 does not truncate short text. + def test_default_max_length(self): + text = "x" * 100 + assert truncate_text(text) == text + # endregion test_default_max_length + + # region test_truncation_at_default_boundary [TYPE Function] + # @BRIEF Text exactly 500 chars is unchanged; 501 chars is truncated. + def test_truncation_at_default_boundary(self): + text_500 = "x" * 500 + text_501 = "x" * 501 + assert truncate_text(text_500) == text_500 + result = truncate_text(text_501) + assert result == "x" * 500 + "..." + assert len(result) == 503 + # endregion test_truncation_at_default_boundary + +# endregion TestTruncateText + + +# region TestCleanText [TYPE Class] +# @BRIEF Test suite for clean_text (combined normalize + truncate). +class TestCleanText: + + # region test_empty_string [TYPE Function] + # @BRIEF Empty string returns empty string. + def test_empty_string(self): + assert clean_text("") == "" + # endregion test_empty_string + + # region test_whitespace_only [TYPE Function] + # @BRIEF Whitespace-only input returns empty string. + def test_whitespace_only(self): + assert clean_text(" \n \t ") == "" + # endregion test_whitespace_only + + # region test_clean_and_short [TYPE Function] + # @BRIEF Already clean, short text is returned unchanged. + def test_clean_and_short(self): + assert clean_text("hello world") == "hello world" + # endregion test_clean_and_short + + # region test_normalize_then_truncate [TYPE Function] + # @BRIEF Whitespace is normalized before truncation — counting clean chars. + def test_normalize_then_truncate(self): + # "hello world" has 15 chars raw, normalized to "hello world" (11 chars) + # With max_length=5, should truncate normalized text to "hello..." + result = clean_text("hello world", max_length=5) + assert result == "hello..." + # endregion test_normalize_then_truncate + + # region test_normalize_keeps_text_under_limit [TYPE Function] + # @BRIEF Normalization reduces length enough that truncation is not needed. + def test_normalize_keeps_text_under_limit(self): + # "A B" has 4 chars raw, normalizes to "A B" (3 chars), still under 5 + result = clean_text("A B", max_length=5) + assert result == "A B" + # endregion test_normalize_keeps_text_under_limit + + # region test_leading_whitespace_then_truncate [TYPE Function] + # @BRIEF Leading whitespace is removed before truncation, avoiding wasted chars. + def test_leading_whitespace_then_truncate(self): + # " hello world" normalizes to "hello world" (11 chars) + # With max_length=5: truncates to "hello..." + result = clean_text(" hello world", max_length=5) + assert result == "hello..." + # endregion test_leading_whitespace_then_truncate + + # region test_default_max_length_passthrough [TYPE Function] + # @BRIEF With default max_length=500, text under 500 after normalize is unchanged. + def test_default_max_length_passthrough(self): + text = " hello world " + expected = "hello world" + assert clean_text(text) == expected + # endregion test_default_max_length_passthrough + +# endregion TestCleanText +# #endregion TestTextCleaner diff --git a/backend/src/plugins/translate/_lang_detect.py b/backend/src/plugins/translate/_lang_detect.py index dce82882..1760680b 100644 --- a/backend/src/plugins/translate/_lang_detect.py +++ b/backend/src/plugins/translate/_lang_detect.py @@ -11,8 +11,15 @@ # langdetect — only 55 languages, less accurate on short texts. # fasttext — requires native compilation, no pre-built wheel for this platform. +import re + from lingua import Language, LanguageDetector, LanguageDetectorBuilder +# Cyrillic character range (Basic + Supplement) +_CYRILLIC_RE = re.compile(r"[\u0400-\u04FF\u0500-\u052F]") +# ASCII letter range (Latin-based) +_ASCII_LETTER_RE = re.compile(r"[a-zA-Z]") + # --------------------------------------------------------------------------- # Build BCP-47 → lingua Language mapping from the lingua Language enum # --------------------------------------------------------------------------- @@ -128,10 +135,53 @@ def detect_language(text: str, detector: LanguageDetector) -> str: # #endregion detect_language +# #region _character_block_fallback [C:2] [TYPE Function] [SEMANTICS language, heuristic, cyrillic] +# @BRIEF When lingua returns "und", check if character-block dominance (e.g. Cyrillic) +# suggests a match to one of the target languages. Catches texts with many numbers +# and punctuation where n-gram-based detection is unreliable. +# @RATIONALE Russian texts with numbers/punctuation (e.g. "По договору поставки. в объёме +# 67 890 123,45 руб..") frequently get "und" from lingua because digits dilute +# the n-gram signal. A simple letter-block ratio catches these reliably. +# @REJECTED Relying solely on lingua for all cases — too many false "und" on number-heavy texts. +# Using LLM for re-detection — defeats the purpose of local detection. +def _character_block_fallback( + texts: list[str], + results: list[str], + target_languages: list[str] | None, +) -> list[str]: + """Apply character-block heuristics for 'und' results. + + For each 'und' result, if the target list includes a Cyrillic language (ru/uk/be) + and the text is Cyrillic-dominant (≥35% of letter chars), assign that language. + """ + if not target_languages: + return results + + tls_lower = [t.lower() for t in target_languages] + cyrillic_targets = [t for t in tls_lower if t in ("ru", "uk", "be")] + if not cyrillic_targets: + return results # no Cyrillic target → nothing to do + + for i, (text, result) in enumerate(zip(texts, results)): + if result != "und" or not text: + continue + + ascii_letters = len(_ASCII_LETTER_RE.findall(text)) + cyrillic_letters = len(_CYRILLIC_RE.findall(text)) + total_letters = ascii_letters + cyrillic_letters + + if total_letters > 0 and cyrillic_letters / total_letters >= 0.35: + results[i] = cyrillic_targets[0] + + return results +# #endregion _character_block_fallback + + # #region batch_detect [C:3] [TYPE Function] [SEMANTICS language, detection, batch] # @BRIEF Detect language for multiple texts in batch. Builds detector if not provided. # @RELATION DEPENDS_ON -> [detect_language] # @RELATION DEPENDS_ON -> [get_detector] +# @RELATION DEPENDS_ON -> [_character_block_fallback] def batch_detect( texts: list[str], target_languages: list[str] | None = None, @@ -150,6 +200,8 @@ def batch_detect( if detector is None: detector = get_detector(target_languages) - return [detect_language(t, detector) for t in texts] + results = [detect_language(t, detector) for t in texts] + results = _character_block_fallback(texts, results, target_languages) + return results # #endregion batch_detect # #endregion LanguageDetectService diff --git a/backend/src/plugins/translate/_llm_call.py b/backend/src/plugins/translate/_llm_call.py index 0e00a44e..2ad39fcc 100644 --- a/backend/src/plugins/translate/_llm_call.py +++ b/backend/src/plugins/translate/_llm_call.py @@ -19,12 +19,13 @@ import json import time import uuid +from datetime import UTC, datetime from typing import Any from sqlalchemy.orm import Session from ...core.logger import belief_scope, logger -from ...models.translate import TranslationJob, TranslationLanguage, TranslationRecord +from ...models.translate import TranslationBatch, TranslationJob, TranslationLanguage, TranslationRecord from ...services.llm_prompt_templates import render_prompt from ...services.llm_provider import LLMProviderService from ._llm_http import call_openai_compatible @@ -148,8 +149,42 @@ class LLMTranslationService: mid = len(batch_rows) // 2 logger.explore("LLM output truncated — splitting batch", {"batch_id": batch_id, "batch_size": len(batch_rows), "split_at": mid, "depth": depth}) - left = self.call_llm_for_batch(job, run_id, batch_rows[:mid], dict_matches, batch_id + "_L", max_tokens, depth + 1) - right = self.call_llm_for_batch(job, run_id, batch_rows[mid:], dict_matches, batch_id + "_R", max_tokens, depth + 1) + + # Create proper TranslationBatch rows for child halves (fixes FK violation + # where _L/_R string suffixes referenced non-existent translation_batches rows). + # @RATIONALE Previous code appended "_L"/"_R" to batch_id string, violating + # FK translation_records.batch_id → translation_batches.id. + # @REJECTED Continuing with string-suffixed batch_ids — causes FK violation + # when any child record is flushed. + left_batch = TranslationBatch( + id=str(uuid.uuid4()), run_id=run_id, batch_index=-1, + status="RUNNING", total_records=len(batch_rows[:mid]), + started_at=datetime.now(UTC), + ) + right_batch = TranslationBatch( + id=str(uuid.uuid4()), run_id=run_id, batch_index=-1, + status="RUNNING", total_records=len(batch_rows[mid:]), + started_at=datetime.now(UTC), + ) + self.db.add_all([left_batch, right_batch]) + self.db.flush() + + left = self.call_llm_for_batch(job, run_id, batch_rows[:mid], dict_matches, + left_batch.id, max_tokens, depth + 1) + right = self.call_llm_for_batch(job, run_id, batch_rows[mid:], dict_matches, + right_batch.id, max_tokens, depth + 1) + + # Finalise child batch stats + left_batch.completed_at = datetime.now(UTC) + right_batch.completed_at = datetime.now(UTC) + left_batch.successful_records = left["successful"] + right_batch.successful_records = right["successful"] + left_batch.failed_records = left["failed"] + right_batch.failed_records = right["failed"] + left_batch.status = "COMPLETED" if left["failed"] == 0 else "COMPLETED_WITH_ERRORS" + right_batch.status = "COMPLETED" if right["failed"] == 0 else "COMPLETED_WITH_ERRORS" + self.db.flush() + return {"successful": left["successful"] + right["successful"], "failed": left["failed"] + right["failed"], "skipped": left["skipped"] + right["skipped"], diff --git a/backend/src/plugins/translate/_text_cleaner.py b/backend/src/plugins/translate/_text_cleaner.py new file mode 100644 index 00000000..25922182 --- /dev/null +++ b/backend/src/plugins/translate/_text_cleaner.py @@ -0,0 +1,63 @@ +# #region TextCleaner [C:3] [TYPE Module] [SEMANTICS translate, text, cleaning, whitespace, truncation] +# @BRIEF Text cleaning utilities for the translation pipeline — whitespace normalization +# and character-length truncation. Used as a preprocessing step before LLM calls. +# @LAYER Domain +# @REJECTED Doing normalization inside the LLM prompt — wastes tokens and produces +# inconsistent results. Preprocessing text before the LLM call is more reliable. +# @RATIONALE Centralizes text preprocessing logic used across the translation pipeline. +# Normalizing whitespace before truncation ensures accurate character counts. + +# #region normalize_whitespace [C:2] [TYPE Function] [SEMANTICS text, whitespace, normalize] +# @BRIEF Collapse multiple spaces, tabs, and newlines into single spaces; trim leading/trailing +# whitespace. Returns empty string for empty or whitespace-only input. +# @PRE text is a string (empty string allowed). +# @POST Returns a string with no leading/trailing whitespace and single internal spaces. +# @EXAMPLE normalize_whitespace(" hello world ") -> "hello world" +# @EXAMPLE normalize_whitespace("") -> "" +def normalize_whitespace(text: str) -> str: + """Collapse multiple spaces/newlines, trim, return clean text. + + All Unicode whitespace characters (spaces, tabs, newlines, etc.) are + collapsed into single ASCII spaces. Leading/trailing whitespace is removed. + """ + if not text: + return "" + return " ".join(text.split()) +# #endregion normalize_whitespace + + +# #region truncate_text [C:2] [TYPE Function] [SEMANTICS text, truncation, length] +# @BRIEF Truncate text to max_length characters, appending "..." if truncation occurs. +# @PRE text is a string. max_length >= 0. +# @POST When len(text) <= max_length, returns text unchanged. +# When len(text) > max_length, returns text[:max_length] + "...". +# @EXAMPLE truncate_text("hello world", 5) -> "hello..." +# @EXAMPLE truncate_text("hi", 5) -> "hi" +def truncate_text(text: str, max_length: int = 500) -> str: + """Truncate text to max_length characters, adding '...' if truncated. + + If the text is shorter than or equal to max_length, it is returned unchanged. + If it exceeds max_length, it is cut at max_length and '...' is appended. + """ + if len(text) > max_length: + return text[:max_length] + "..." + return text +# #endregion truncate_text + + +# #region clean_text [C:2] [TYPE Function] [SEMANTICS text, clean, normalize, truncate] +# @BRIEF Combine whitespace normalization and truncation into one step. +# Applies normalize_whitespace first, then truncate_text. +# @PRE text is a string. max_length >= 0. +# @POST Returns normalized and optionally truncated text. +# @RELATION CALLS -> [normalize_whitespace] +# @RELATION CALLS -> [truncate_text] +# @EXAMPLE clean_text(" hello world ", 5) -> "hello..." (normalized to "hello world", +# then truncated since "hello world" has 11 chars > 5) +def clean_text(text: str, max_length: int = 500) -> str: + """Normalize whitespace then truncate to max_length.""" + normalized = normalize_whitespace(text) + return truncate_text(normalized, max_length) +# #endregion clean_text + +# #endregion TextCleaner diff --git a/backend/src/plugins/translate/service_target_schema.py b/backend/src/plugins/translate/service_target_schema.py index 4aeb71e9..778b3a2f 100644 --- a/backend/src/plugins/translate/service_target_schema.py +++ b/backend/src/plugins/translate/service_target_schema.py @@ -1,4 +1,4 @@ -# #region TargetSchemaValidation [C:3] [TYPE Module] [SEMANTICS translate, schema, validation, target-table] +# #region TargetSchemaValidation [C:4] [TYPE Module] [SEMANTICS translate, schema, validation, target-table] # @BRIEF Проверка схемы целевой таблицы: запрос колонок через Superset SQL Lab, # сравнение с ожидаемыми (из build_columns), возврат diff. # @LAYER Service @@ -8,15 +8,13 @@ # @PRE Superset окружение доступно, target_database_id валиден. # @POST Возвращает актуальные, ожидаемые, отсутствующие и лишние колонки. # @SIDE_EFFECT Выполняет SQL-запрос через Superset SQL Lab. -# @INVARIANT Если таблица не существует — table_exists=False без ошибки. -# @INVARIANT Если Superset недоступен — error с описанием, all_present=False. -# @INVARIANT Логика ожидаемых колонок sync с build_columns() в orchestrator_sql_rows.py. +# @RATIONALE C4 — оркестрация: вызов executor, парсинг ответа, diff и построение ответа. # #endregion TargetSchemaValidation -import logging import re from typing import Any +from ...core.logger import belief_scope, logger from ...schemas.translate import ( TargetSchemaColumnInfo, TargetSchemaValidationRequest, @@ -25,48 +23,34 @@ from ...schemas.translate import ( from ...core.config_manager import ConfigManager from .superset_executor import SupersetSqlLabExecutor -logger = logging.getLogger(__name__) - _TABLE_NAME_RE = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$') # #region _build_expected_columns [C:2] [TYPE Function] # @BRIEF Собирает список ожидаемых колонок по конфигурации column mapping. -# Логика идентична build_columns() из orchestrator_sql_rows.py, -# но не требует объекта TranslationJob. def _build_expected_columns(req: TargetSchemaValidationRequest) -> list[TargetSchemaColumnInfo]: - """ - Определяет, какие колонки INSERT будет пытаться заполнить, - на основе конфигурации column mapping. - """ + """Определяет, какие колонки INSERT будет пытаться заполнить.""" names: list[str] = [] - # Ключевые колонки (user-defined, для upsert matching) if req.target_key_cols: names.extend(req.target_key_cols) - # Целевая колонка для переведённого текста effective_target = req.target_column or req.translation_column if effective_target: names.append(effective_target) - # Колонка с кодом языка перевода if req.target_language_column: names.append(req.target_language_column) - # Колонка с исходным текстом (source reference) if req.target_source_column: names.append(req.target_source_column) - # Колонка с определённым исходным языком if req.target_source_language_column: names.append(req.target_source_language_column) - # Служебные колонки — всегда names.append("context") names.append("is_original") - # Дедупликация с сохранением порядка seen: set[str] = set() deduped: list[str] = [] for c in names: @@ -78,192 +62,264 @@ def _build_expected_columns(req: TargetSchemaValidationRequest) -> list[TargetSc # #endregion _build_expected_columns -# #region _parse_column_results [C:2] [TYPE Function] -# @BRIEF Извлекает информацию о колонках из ответа Superset SQL Lab. +# #region _extract_columns_from_rows [C:2] [TYPE Function] +# @BRIEF Извлекает список колонок таблицы из data-строк результата SQL Lab. # -# Superset SQL Lab возвращает разные форматы в зависимости от версии. -# Эта функция ищет колонки во всех возможных местах ответа. -def _parse_column_results(result: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]: +# Для PostgreSQL (information_schema.columns) каждая строка содержит: +# {"column_name": "id", "data_type": "integer", "is_nullable": "YES"} +# +# Для ClickHouse (DESCRIBE TABLE) каждая строка содержит: +# {"name": "id", "type": "Int32"} +# +# Функция ищет имя колонки по нескольким возможным ключам и возвращает +# унифицированный список TargetSchemaColumnInfo. +def _extract_columns_from_rows(data_rows: list[dict[str, Any]]) -> list[dict[str, Any]]: """ - Парсит результат SQL Lab запроса для извлечения списка колонок таблицы. - Returns (columns_info, table_exists). + Извлекает информацию о колонках таблицы из data-строк SQL Lab ответа. + """ + columns_info: list[dict[str, Any]] = [] + + for row in data_rows: + if not isinstance(row, dict): + continue + + # Имя колонки: ищем по всем возможным ключам (PG, CH, разные версии Superset) + col_name = ( + row.get("column_name") # PostgreSQL information_schema + or row.get("name") # ClickHouse DESCRIBE TABLE / MySQL SHOW COLUMNS + or row.get("Field") # MySQL SHOW COLUMNS + ) + if not col_name: + continue + + # Тип данных + col_type = ( + row.get("data_type") # PG information_schema + or row.get("type") # ClickHouse DESCRIBE TABLE + or row.get("Type") # MySQL SHOW COLUMNS + ) + + # Nullable + is_nullable = row.get("is_nullable") # PG + if isinstance(is_nullable, str): + is_nullable = is_nullable.upper() == "YES" + elif not isinstance(is_nullable, bool): + is_nullable = True # ClickHouse не возвращает is_nullable, считаем True + + columns_info.append({ + "name": str(col_name), + "type": str(col_type) if col_type else None, + "is_nullable": is_nullable, + }) + + return columns_info +# #endregion _extract_columns_from_rows + + +# #region _parse_sqllab_result [C:2] [TYPE Function] +# @BRIEF Извлекает data-строки из ответа Superset SQL Lab. +# Поддерживает 3 формата: sync mode, async polling, get_query_results. +def _parse_sqllab_result(result: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]: + """ + Извлекает строки данных из ответа SQL Lab. + Returns (data_rows, table_exists). """ if not result: return [], False raw = result.get("raw_response", result) + if not raw: + return [], False - # Пробуем извлечь данные из разных форматов ответа Superset - columns_data = None data_rows = None - # Формат 1: прямой ответ с колонками (sync mode) - if "columns" in raw and isinstance(raw["columns"], list): - columns_data = raw["columns"] - data_rows = raw.get("data") if isinstance(raw.get("data"), list) else None + # Формат 1: прямой ответ с data + if "data" in raw and isinstance(raw["data"], list): + data_rows = raw["data"] # Формат 2: вложенный result (async polling) elif "result" in raw and isinstance(raw["result"], dict): res = raw["result"] - if "columns" in res: - columns_data = res["columns"] data_rows = res.get("data") if isinstance(res.get("data"), list) else None # Формат 3: ключ results (get_query_results) elif "results" in raw and isinstance(raw["results"], dict): res = raw["results"] - if "columns" in res: - columns_data = res["columns"] - if "data" in res: + if "data" in res and isinstance(res["data"], list): data_rows = res["data"] - if isinstance(res.get("result"), dict): + elif isinstance(res.get("result"), dict): sub = res["result"] - if not columns_data and "columns" in sub: - columns_data = sub["columns"] - if not data_rows and "data" in sub: - data_rows = sub["data"] + data_rows = sub.get("data") if isinstance(sub.get("data"), list) else None - if not columns_data: + # Формат 4: results — список строк (execute_and_poll sync mode возвращает так) + elif "results" in raw and isinstance(raw["results"], list): + data_rows = raw["results"] + + if data_rows is None: return [], False - # Проверяем существование таблицы: есть ли хотя бы одна строка - table_exists = data_rows is not None and len(data_rows) > 0 + table_exists = len(data_rows) > 0 - # Извлекаем имена и типы колонок из метаданных ответа - columns_info: list[dict[str, Any]] = [] - for col in columns_data: - col_name = ( - col.get("name") - or col.get("column_name") - or col.get("Column", "") - ) - col_type = ( - col.get("type") - or col.get("data_type") - or col.get("Type", "") - or col.get("col_type", "") - ) - is_nullable = col.get("is_nullable") - # PostgreSQL returns 'YES'/'NO' for is_nullable - if isinstance(is_nullable, str): - is_nullable = is_nullable.upper() == "YES" + # Нормализуем формат строк: если data_rows — список списков, а не словарей + # (бывает в некоторых версиях Superset), конвертируем через columns metadata + if data_rows and not isinstance(data_rows[0], dict): + columns_raw = raw.get("columns") or (raw.get("result") or {}).get("columns") or [] + col_names = [] + for c in columns_raw: + cn = c.get("name") or c.get("column_name") or "" + if cn: + col_names.append(cn) + if col_names: + normalized = [] + for row in data_rows: + if isinstance(row, list) and len(row) == len(col_names): + normalized.append(dict(zip(col_names, row))) + data_rows = normalized - if col_name: - columns_info.append({ - "name": str(col_name), - "type": str(col_type) if col_type else None, - "is_nullable": is_nullable if isinstance(is_nullable, bool) else True, - }) - - return columns_info, table_exists -# #endregion _parse_column_results + return data_rows, table_exists +# #endregion _parse_sqllab_result -# #region validate_target_table_schema [C:3] [TYPE Function] +# #region validate_target_table_schema [C:4] [TYPE Function] [SEMANTICS translate, schema, validate, orchestrate] # @BRIEF Основная функция: проверяет схему целевой таблицы через Superset SQL Lab. +# @PRE Superset окружение и target_database_id валидны. +# @POST Возвращает TargetSchemaValidationResponse с diff-анализом. +# @SIDE_EFFECT Выполняет SQL-запрос к information_schema через Superset SQL Lab API. +# @RELATION DEPENDS_ON -> [_build_expected_columns] +# @RELATION DEPENDS_ON -> [_extract_columns_from_rows] +# @RELATION DEPENDS_ON -> [_parse_sqllab_result] +# @RELATION DEPENDS_ON -> [SupersetSqlLabExecutor] def validate_target_table_schema( req: TargetSchemaValidationRequest, config_manager: ConfigManager, ) -> TargetSchemaValidationResponse: """ Проверяет схему целевой таблицы: - 1. Вычисляет ожидаемые колонки (build_columns логика) - 2. Запрашивает актуальные колонки через Superset SQL Lab - (SELECT column_name, data_type, is_nullable FROM information_schema.columns ...) + 1. Вычисляет ожидаемые колонки + 2. Запрашивает актуальные через Superset SQL Lab 3. Сравнивает и возвращает diff """ - # 1. Ожидаемые колонки - expected = _build_expected_columns(req) - expected_names = {c.name for c in expected} + with belief_scope("validate_target_table_schema"): + # 1. Ожидаемые колонки + expected = _build_expected_columns(req) + expected_names = {c.name for c in expected} + logger.reason("Built expected columns", + extra={"payload": {"count": len(expected), "columns": [c.name for c in expected]}}) - # 2. Валидация имён схемы и таблицы (защита от SQL injection в identifier) - if req.target_table and not _TABLE_NAME_RE.match(req.target_table): - return TargetSchemaValidationResponse( - table_exists=False, - error=f"Invalid target table name: '{req.target_table}'. Only alphanumeric and underscore allowed.", - expected_columns=expected, - actual_columns=[], - missing_columns=expected, - extra_columns=[], - all_present=False, - ) - if req.target_schema and not _TABLE_NAME_RE.match(req.target_schema): - return TargetSchemaValidationResponse( - table_exists=False, - error=f"Invalid target schema name: '{req.target_schema}'. Only alphanumeric and underscore allowed.", - expected_columns=expected, - actual_columns=[], - missing_columns=expected, - extra_columns=[], - all_present=False, - ) - - # 3. Запрос через Superset SQL Lab - try: - executor = SupersetSqlLabExecutor(config_manager, req.environment_id) - executor.resolve_database_id(target_database_id=req.target_database_id) - - safe_schema = (req.target_schema or "public").replace("'", "''") - safe_table = req.target_table.replace("'", "''") - - sql = ( - f"SELECT column_name, data_type, is_nullable " - f"FROM information_schema.columns " - f"WHERE table_schema = '{safe_schema}' " - f"AND table_name = '{safe_table}' " - f"ORDER BY ordinal_position" - ) - - result = executor.execute_and_poll(sql=sql, max_polls=15, poll_interval_seconds=1.0) - - status = result.get("status", "") - if status == "failed": - error_msg = result.get("error_message", "SQL Lab query failed") + # 2. Валидация имён (SQL injection protection) + if req.target_table and not _TABLE_NAME_RE.match(req.target_table): + logger.explore("Invalid target table name rejected", + extra={"payload": {"table": req.target_table}, + "error": "Invalid characters in table name"}) return TargetSchemaValidationResponse( table_exists=False, - error=f"Superset SQL Lab error: {error_msg}", - expected_columns=expected, - actual_columns=[], - missing_columns=expected, - extra_columns=[], - all_present=False, + error=f"Invalid target table name: '{req.target_table}'. Only alphanumeric and underscore allowed.", + expected_columns=expected, actual_columns=[], + missing_columns=expected, extra_columns=[], all_present=False, + ) + if req.target_schema and not _TABLE_NAME_RE.match(req.target_schema): + logger.explore("Invalid target schema name rejected", + extra={"payload": {"schema": req.target_schema}, + "error": "Invalid characters in schema name"}) + return TargetSchemaValidationResponse( + table_exists=False, + error=f"Invalid target schema name: '{req.target_schema}'. Only alphanumeric and underscore allowed.", + expected_columns=expected, actual_columns=[], + missing_columns=expected, extra_columns=[], all_present=False, ) - raw_columns, table_exists = _parse_column_results(result) + # 3. Запрос через Superset SQL Lab + target_ref = f"{req.target_schema or 'public'}.{req.target_table}" + logger.reason("Querying target table schema", + extra={"payload": {"table": target_ref, "env": req.environment_id, "db_id": req.target_database_id}}) - actual_cols = [] - for col in raw_columns: - actual_cols.append(TargetSchemaColumnInfo( - name=col["name"], - data_type=col.get("type"), - is_nullable=col.get("is_nullable", True), - )) + try: + executor = SupersetSqlLabExecutor(config_manager, req.environment_id) + executor.resolve_database_id(target_database_id=req.target_database_id) + backend = executor.get_database_backend() or "" - actual_names = {c.name for c in actual_cols} + safe_schema = (req.target_schema or "public").replace("'", "''") + safe_table = req.target_table.replace("'", "''") - # 3. Diff - missing = [col for col in expected if col.name not in actual_names] - extra = [col for col in actual_cols if col.name not in expected_names] + # ClickHouse использует system.columns, всё остальное — information_schema.columns + is_clickhouse = any(kw in backend.lower() for kw in ("clickhouse", "ch")) + if is_clickhouse: + sql = ( + f"SELECT name, type, default_expression AS data_default " + f"FROM system.columns " + f"WHERE database = '{safe_schema}' " + f"AND table = '{safe_table}' " + f"ORDER BY position" + ) + else: + sql = ( + f"SELECT column_name, data_type, is_nullable " + f"FROM information_schema.columns " + f"WHERE table_schema = '{safe_schema}' " + f"AND table_name = '{safe_table}' " + f"ORDER BY ordinal_position" + ) - return TargetSchemaValidationResponse( - table_exists=table_exists, - expected_columns=expected, - actual_columns=actual_cols, - missing_columns=missing, - extra_columns=extra, - all_present=len(missing) == 0, - ) + logger.reason("Executing SQL Lab query", + extra={"payload": {"sql": sql[:200], "backend": backend}}) - except Exception as e: - logger.warning("Target schema validation failed: %s", str(e), exc_info=True) - return TargetSchemaValidationResponse( - table_exists=False, - error=f"Failed to validate target schema: {e}", - expected_columns=expected, - actual_columns=[], - missing_columns=expected, - extra_columns=[], - all_present=False, - ) + result = executor.execute_and_poll(sql=sql, max_polls=15, poll_interval_seconds=1.0) + status = result.get("status", "") + + if status == "failed": + error_msg = result.get("error_message", "SQL Lab query failed") + logger.explore("SQL Lab query failed", + extra={"payload": {"table": target_ref}, "error": error_msg}) + return TargetSchemaValidationResponse( + table_exists=False, error=f"Superset SQL Lab error: {error_msg}", + expected_columns=expected, actual_columns=[], + missing_columns=expected, extra_columns=[], all_present=False, + ) + + # Парсим data-строки результата (реальные колонки таблицы) + data_rows, table_exists = _parse_sqllab_result(result) + actual_cols_raw = _extract_columns_from_rows(data_rows) + + actual_cols = [] + for col in actual_cols_raw: + actual_cols.append(TargetSchemaColumnInfo( + name=col["name"], + data_type=col.get("type"), + is_nullable=col.get("is_nullable", True), + )) + + actual_names = {c.name for c in actual_cols} + + # 4. Diff + missing = [col for col in expected if col.name not in actual_names] + extra = [col for col in actual_cols if col.name not in expected_names] + + logger.reflect("Schema check complete", + extra={"payload": { + "table": target_ref, "table_exists": table_exists, + "expected": len(expected), "found": len(actual_cols), + "missing": len(missing), "extra": len(extra), + "all_present": len(missing) == 0, + "actual_columns": [c.name for c in actual_cols], + }}) + + return TargetSchemaValidationResponse( + table_exists=table_exists, + expected_columns=expected, + actual_columns=actual_cols, + missing_columns=missing, + extra_columns=extra, + all_present=len(missing) == 0, + ) + + except Exception as e: + logger.explore("Target schema validation failed", + extra={"payload": {"table": target_ref, "env": req.environment_id}, + "error": str(e)}) + return TargetSchemaValidationResponse( + table_exists=False, error=f"Failed to validate target schema: {e}", + expected_columns=expected, actual_columns=[], + missing_columns=expected, extra_columns=[], all_present=False, + ) # #endregion validate_target_table_schema # #endregion TargetSchemaValidation diff --git a/backend/src/services/__tests__/test_llm_plugin_persistence.py b/backend/src/services/__tests__/test_llm_plugin_persistence.py index 1ae7d991..94c3d68c 100644 --- a/backend/src/services/__tests__/test_llm_plugin_persistence.py +++ b/backend/src/services/__tests__/test_llm_plugin_persistence.py @@ -75,6 +75,7 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( base_url="https://example.invalid/v1", default_model="gpt-4o", is_active=True, + is_multimodal=True, ) # region _FakeProviderService [TYPE Class] diff --git a/backend/src/services/__tests__/test_llm_provider.py b/backend/src/services/__tests__/test_llm_provider.py index 44203910..a40ccb02 100644 --- a/backend/src/services/__tests__/test_llm_provider.py +++ b/backend/src/services/__tests__/test_llm_provider.py @@ -258,3 +258,125 @@ def test_is_masked_or_placeholder(): assert is_masked_or_placeholder("...xyz") is True assert is_masked_or_placeholder("sk-real-key-1234") is False assert is_masked_or_placeholder("short") is False + + +# endregion test_is_masked_or_placeholder + + +# region test_create_provider_with_multimodal_flag [TYPE Function] +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Verify provider creation persists the is_multimodal flag. +def test_create_provider_with_multimodal_flag(service, mock_db): + """Verify is_multimodal=True is stored during provider creation.""" + config = LLMProviderConfig( + provider_type=LLMProviderType.OPENAI, + name="Vision Provider", + base_url="https://api.openai.com", + api_key="sk-vision-key", + default_model="gpt-4o", + is_active=True, + is_multimodal=True, + ) + + provider = service.create_provider(config) + + assert provider.is_multimodal is True + mock_db.add.assert_called() + mock_db.commit.assert_called() + + +# endregion test_create_provider_with_multimodal_flag + + +# region test_create_provider_without_multimodal_flag [TYPE Function] +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Verify is_multimodal defaults to False when not specified. +def test_create_provider_without_multimodal_flag(service, mock_db): + """Verify is_multimodal defaults to False.""" + config = LLMProviderConfig( + provider_type=LLMProviderType.OPENAI, + name="Text Provider", + base_url="https://api.openai.com", + api_key="sk-text-key", + default_model="gpt-4.1-mini", + is_active=True, + ) + + provider = service.create_provider(config) + + assert provider.is_multimodal is False + + +# endregion test_create_provider_without_multimodal_flag + + +# region test_update_provider_preserves_is_multimodal [TYPE Function] +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Verify updating a provider preserves and correctly sets is_multimodal. +def test_update_provider_preserves_is_multimodal(service, mock_db): + """Verify is_multimodal is updated correctly.""" + existing_encrypted = EncryptionManager().encrypt("secret-value") + mock_provider = LLMProvider( + id="p1", + provider_type="openai", + name="Vision", + base_url="https://api.openai.com/v1", + api_key=existing_encrypted, + default_model="gpt-4o", + is_active=True, + is_multimodal=True, + ) + mock_db.query().filter().first.return_value = mock_provider + + # Update with is_multimodal=False + config = LLMProviderConfig( + id="p1", + provider_type=LLMProviderType.OPENAI, + name="Vision", + base_url="https://api.openai.com/v1", + api_key="********", + default_model="gpt-4o", + is_active=True, + is_multimodal=False, + ) + + updated = service.update_provider("p1", config) + + assert updated.is_multimodal is False + + +# endregion test_update_provider_preserves_is_multimodal + + +# region test_llm_provider_config_multimodal_default [TYPE Function] +# @RELATION: VERIFIES -> [LLMProviderConfig] +# @PURPOSE: Verify LLMProviderConfig.is_multimodal defaults to False for schema stability. +def test_llm_provider_config_multimodal_default(): + """Verify default is_multimodal is False in schema.""" + config = LLMProviderConfig( + provider_type=LLMProviderType.OPENAI, + name="Default Test", + base_url="https://api.openai.com/v1", + api_key="sk-test", + default_model="gpt-4", + ) + assert config.is_multimodal is False + + +# endregion test_llm_provider_config_multimodal_default + + +# region test_llm_provider_config_multimodal_explicit [TYPE Function] +# @RELATION: VERIFIES -> [LLMProviderConfig] +# @PURPOSE: Verify LLMProviderConfig accepts explicit is_multimodal=True. +def test_llm_provider_config_multimodal_explicit(): + """Verify setting is_multimodal=True explicitly works.""" + config = LLMProviderConfig( + provider_type=LLMProviderType.OPENAI, + name="Vision Test", + base_url="https://api.openai.com/v1", + api_key="sk-test", + default_model="gpt-4o", + is_multimodal=True, + ) + assert config.is_multimodal is True diff --git a/backend/src/services/llm_prompt_templates.py b/backend/src/services/llm_prompt_templates.py index 51f0a32b..6c36992f 100644 --- a/backend/src/services/llm_prompt_templates.py +++ b/backend/src/services/llm_prompt_templates.py @@ -131,7 +131,6 @@ def is_multimodal_model(model_name: str, provider_type: str | None = None) -> bo token = (model_name or "").strip().lower() if not token: return False - provider = (provider_type or "").strip().lower() text_only_markers = ( "text-only", "embedding", @@ -158,9 +157,7 @@ def is_multimodal_model(model_name: str, provider_type: str | None = None) -> bo "qwen-vl", "qwen2-vl", ) - if any(marker in token for marker in multimodal_markers): - return True - return False + return any(marker in token for marker in multimodal_markers) # #endregion is_multimodal_model diff --git a/backend/src/services/llm_provider.py b/backend/src/services/llm_provider.py index 9e4aaa9a..d82677f8 100644 --- a/backend/src/services/llm_provider.py +++ b/backend/src/services/llm_provider.py @@ -197,6 +197,7 @@ class LLMProviderService: api_key=encrypted_key, default_model=config.default_model, is_active=config.is_active, + is_multimodal=config.is_multimodal, ) self.db.add(db_provider) self.db.commit() @@ -228,6 +229,7 @@ class LLMProviderService: db_provider.api_key = self.encryption.encrypt(config.api_key) db_provider.default_model = config.default_model db_provider.is_active = config.is_active + db_provider.is_multimodal = config.is_multimodal self.db.commit() self.db.refresh(db_provider) diff --git a/frontend/src/components/llm/ProviderConfig.svelte b/frontend/src/components/llm/ProviderConfig.svelte index b615bda9..cbb09a62 100644 --- a/frontend/src/components/llm/ProviderConfig.svelte +++ b/frontend/src/components/llm/ProviderConfig.svelte @@ -36,6 +36,7 @@ api_key: "", default_model: "gpt-4o", is_active: true, + is_multimodal: false, }); let testStatus = $state({ type: "", message: "" }); @@ -50,20 +51,6 @@ let fetchDebounceTimer = $state(null); - function isMultimodalModel(modelName) { - const token = (modelName || "").toLowerCase(); - if (!token) return false; - return ( - token.includes("gpt-4o") || - token.includes("gpt-4.1") || - token.includes("vision") || - token.includes("vl") || - token.includes("gemini") || - token.includes("claude-3") || - token.includes("claude-sonnet-4") - ); - } - function isMaskedKey(key) { if (!key) return false; return key === "********" || key.includes("..."); @@ -87,6 +74,7 @@ api_key: "", default_model: "gpt-4o", is_active: true, + is_multimodal: false, }; editingProvider = null; testStatus = { type: "", message: "" }; @@ -108,6 +96,7 @@ api_key: provider?.api_key ?? "", default_model: provider?.default_model ?? "gpt-4o", is_active: Boolean(provider?.is_active), + is_multimodal: Boolean(provider?.is_multimodal), }; testStatus = { type: "", message: "" }; availableModels = []; @@ -257,6 +246,7 @@ base_url: provider.base_url, default_model: provider.default_model, is_active: provider.is_active, + is_multimodal: Boolean(provider.is_multimodal), }; await requestApi(`/llm/providers/${provider.id}`, "PUT", updatePayload); } catch (err) { @@ -476,6 +466,21 @@ class="text-sm font-medium text-gray-700">{$t.llm.active} + +
+ + +
{#if testStatus.message} @@ -530,9 +535,9 @@ {provider.is_active ? $t.llm.active : "Inactive"} - {isMultimodalModel(provider.default_model) + {provider.is_multimodal ? ($t.llm?.multimodal ) : ($t.llm?.text_only )} diff --git a/frontend/src/lib/components/translate/TargetSchemaHint.svelte b/frontend/src/lib/components/translate/TargetSchemaHint.svelte index 405e091b..26370abb 100644 --- a/frontend/src/lib/components/translate/TargetSchemaHint.svelte +++ b/frontend/src/lib/components/translate/TargetSchemaHint.svelte @@ -33,6 +33,7 @@ let state = $state('idle'); let result = $state(null); let abortController = $state(null); + let renderKey = $state(0); /** CSS-класс body в зависимости от состояния */ let bodyClass = $derived.by(() => { @@ -42,6 +43,9 @@ return 'px-3 py-2.5'; }); + /** Колонки с типами для отображения — $state, наполняется в handleCheck() */ + let actualColumns = $state([]); + /** Есть ли минимальные данные для проверки? */ let canCheck = $derived( !!environmentId @@ -56,6 +60,24 @@ return _('translate.target_schema.check_button') || 'Check Target Schema'; }); + /** + * Выводит тип колонки по соглашению именования, + * когда реальный тип из БД недоступен. + */ + function inferType(colName) { + const n = colName.toLowerCase(); + // Служебные флаги и boolean + if (n === 'is_original' || n.startsWith('is_') || n.startsWith('has_') || n.endsWith('_flag')) return 'INT'; + // Внешние ключи + if (n.endsWith('_id') || n.endsWith('_key')) return 'INT'; + // Код языка + if (n.endsWith('_lang') || n === 'lang_code' || n.endsWith('_language')) return 'STRING'; + // Даты + if (n.endsWith('_date') || n.endsWith('_time') || n.endsWith('_at')) return 'DATETIME'; + // Всё остальное — текст + return 'STRING'; + } + /** Основная проверка */ async function handleCheck() { if (!canCheck || state === 'checking') return; @@ -83,7 +105,20 @@ // Если запрос был отменён — не обновляем state if (signal.aborted) return; result = res; + // Build merged column list with types from actual_columns + const ac = res?.actual_columns || []; + const actualMap = {}; + for (const c of ac) actualMap[c.name] = c; + actualColumns = (res?.expected_columns || []).map(col => { + const match = actualMap[col.name]; + return { + name: col.name, + data_type: match ? match.data_type : inferType(col.name), + is_missing: (res?.missing_columns || []).some(m => m.name === col.name), + }; + }); state = 'complete'; + renderKey++; } catch (err) { if (signal.aborted) return; state = 'error'; @@ -179,64 +214,66 @@

{:else if state === 'complete' && result} - {#if !result.table_exists} -
- - - -

{_('translate.target_schema.table_not_found') || 'Target table not found in the database.'}

-
- - {:else if result.all_present} -
- - - -

{_('translate.target_schema.all_present') || 'Target table has all required columns.'}

-
- - {:else} -
- - - -

{_('translate.target_schema.missing_columns_title') || 'Target table is missing some required columns.'}

-
- {/if} - - - {#if result.expected_columns?.length > 0} -
-

- {_('translate.target_schema.expected_columns') || 'Required columns for translation inserts:'} -

-
- {#each result.expected_columns as col} - {@const isMissing = result.missing_columns?.some(m => m.name === col.name)} - - {#if isMissing} - - - - {:else} - - - - {/if} - {col.name} - - {/each} + {#key renderKey} + {#if !result.table_exists} +
+ + + +

{_('translate.target_schema.table_not_found') || 'Target table not found in the database.'}

-
- {/if} + + {:else if result.all_present} +
+ + + +

{_('translate.target_schema.all_present') || 'Target table has all required columns.'}

+
+ + {:else} +
+ + + +

{_('translate.target_schema.missing_columns_title') || 'Target table is missing some required columns.'}

+
+ {/if} + + + {#if actualColumns.length > 0} +
+

+ {_('translate.target_schema.expected_columns') || 'Required columns for translation inserts:'} +

+
+ {#each actualColumns as col} + + {#if col.is_missing} + + + + {:else} + + + + {/if} + {col.name} + {#if col.data_type}({col.data_type}){/if} + + {/each} +
+
+ {/if} + {/key} {#if result.table_exists && !result.all_present} diff --git a/frontend/src/lib/i18n/locales/en/llm.json b/frontend/src/lib/i18n/locales/en/llm.json index f3ca665c..949bf0b8 100644 --- a/frontend/src/lib/i18n/locales/en/llm.json +++ b/frontend/src/lib/i18n/locales/en/llm.json @@ -32,5 +32,6 @@ "models_loaded": "{count} models loaded", "models_load_failed": "Failed to load models: {error}", "select_model": "Select a model...", - "refresh_models": "Refresh models" + "refresh_models": "Refresh models", + "multimodal_checkbox": "Multimodal (supports images)" } diff --git a/frontend/src/lib/i18n/locales/ru/llm.json b/frontend/src/lib/i18n/locales/ru/llm.json index 1a51856a..f2006e77 100644 --- a/frontend/src/lib/i18n/locales/ru/llm.json +++ b/frontend/src/lib/i18n/locales/ru/llm.json @@ -32,5 +32,6 @@ "models_loaded": "Загружено {count} моделей", "models_load_failed": "Не удалось загрузить модели: {error}", "select_model": "Выберите модель...", - "refresh_models": "Обновить модели" + "refresh_models": "Обновить модели", + "multimodal_checkbox": "Мультимодальная (поддержка изображений)" } diff --git a/frontend/src/routes/admin/settings/llm/+page.svelte b/frontend/src/routes/admin/settings/llm/+page.svelte index e3fd52bb..1685e785 100644 --- a/frontend/src/routes/admin/settings/llm/+page.svelte +++ b/frontend/src/routes/admin/settings/llm/+page.svelte @@ -47,20 +47,6 @@ git_commit: '', }; - function isMultimodalModel(modelName) { - const token = (modelName || '').toLowerCase(); - if (!token) return false; - return ( - token.includes('gpt-4o') || - token.includes('gpt-4.1') || - token.includes('vision') || - token.includes('vl') || - token.includes('gemini') || - token.includes('claude-3') || - token.includes('claude-sonnet-4') - ); - } - function getProviderById(providerId) { if (!providerId) return null; return providers.find((item) => item.id === providerId) || null; @@ -209,7 +195,7 @@ {/each} - {#if bindings.dashboard_validation && !isMultimodalModel(getProviderById(bindings.dashboard_validation)?.default_model)} + {#if bindings.dashboard_validation && !(getProviderById(bindings.dashboard_validation)?.is_multimodal)}

{$t.settings?.llm_multimodal_warning || 'Dashboard validation requires a multimodal model (image input).'} diff --git a/frontend/src/routes/settings/settings-utils.js b/frontend/src/routes/settings/settings-utils.js index cdc1bb4a..d0ed9eba 100644 --- a/frontend/src/routes/settings/settings-utils.js +++ b/frontend/src/routes/settings/settings-utils.js @@ -109,24 +109,7 @@ export function isDashboardValidationBindingValid(settings) { const providerId = settings.llm.provider_bindings.dashboard_validation; const providers = settings.providers || settings.llm_providers || []; const provider = providers.find((p) => p.id === providerId); - return !!provider && isMultimodalModel(provider.default_model); -} - -/** - * Return `true` when the model name looks like a multimodal/vision model. - */ -export function isMultimodalModel(modelName) { - const token = (modelName || "").toLowerCase(); - if (!token) return false; - return ( - token.includes("gpt-4o") || - token.includes("gpt-4.1") || - token.includes("vision") || - token.includes("vl") || - token.includes("gemini") || - token.includes("claude-3") || - token.includes("claude-sonnet-4") - ); + return !!provider && Boolean(provider.is_multimodal); } /**