semantics

This commit is contained in:
2026-05-20 09:59:03 +03:00
parent b916ef94d5
commit 09d12b3b68
34 changed files with 9727 additions and 8789 deletions

View File

@@ -0,0 +1,490 @@
# region TargetSchemaValidationTests [TYPE Module]
# @SEMANTICS: test, translate, target-schema, validation
# @PURPOSE: Tests for target table schema validation: _build_expected_columns, _parse_column_results, validate_target_table_schema.
# @LAYER: Test
# @RELATION: BINDS_TO -> [TargetSchemaValidation]
# @TEST_CONTRACT: _build_expected_columns -> list[TargetSchemaColumnInfo]
# @TEST_CONTRACT: _parse_column_results -> (list[dict], bool)
# @TEST_CONTRACT: validate_target_table_schema -> TargetSchemaValidationResponse
# @TEST_EDGE: missing_target_key_cols -> Only context + is_original returned
# @TEST_EDGE: deduplication -> Duplicate column names are removed
# @TEST_EDGE: empty_result -> Returns ([], False)
# @TEST_EDGE: superset_sync_format -> Standard Superset response format parsed
# @TEST_EDGE: superset_async_format -> Async polling response format parsed
# @TEST_EDGE: get_query_results_format -> get_query_results response format parsed
# @TEST_EDGE: executor_failure -> SQL Lab query fails, error returned
# @TEST_EDGE: sql_injection_table_name -> Invalid table name rejected with error
# @TEST_EDGE: executor_exception -> Exception during execution returns error response
from unittest.mock import MagicMock, patch
import pytest
from src.schemas.translate import (
TargetSchemaColumnInfo,
TargetSchemaValidationRequest,
)
from src.plugins.translate.service_target_schema import (
_build_expected_columns,
_parse_column_results,
validate_target_table_schema,
)
# region build_request [TYPE Function]
# @PURPOSE: Helper to build a TargetSchemaValidationRequest with defaults.
@pytest.fixture
def build_request():
def _build(**overrides):
defaults = {
"environment_id": "env-1",
"target_database_id": "db-1",
"target_schema": "public",
"target_table": "my_table",
"target_key_cols": ["id"],
"target_column": "translated_text",
"translation_column": None,
"target_language_column": None,
"target_source_column": None,
"target_source_language_column": None,
}
defaults.update(overrides)
return TargetSchemaValidationRequest(**defaults)
return _build
# endregion build_request
# region TestBuildExpectedColumns [TYPE Class]
# @PURPOSE: Tests for _build_expected_columns logic.
class TestBuildExpectedColumns:
# region test_basic_case [TYPE Function]
# @PURPOSE: target_key_cols + target_column produces id, translated_text, context, is_original.
def test_basic_case(self, build_request) -> None:
req = build_request()
result = _build_expected_columns(req)
names = [c.name for c in result]
assert names == ["id", "translated_text", "context", "is_original"]
# endregion test_basic_case
# region test_all_fields [TYPE Function]
# @PURPOSE: All optional column fields produce 7 columns.
def test_all_fields(self, build_request) -> None:
req = build_request(
target_language_column="lang",
target_source_column="source_text",
target_source_language_column="source_lang",
)
result = _build_expected_columns(req)
names = [c.name for c in result]
assert len(names) == 7
assert "lang" in names
assert "source_text" in names
assert "source_lang" in names
assert "id" in names
assert "translated_text" in names
assert "context" in names
assert "is_original" in names
# endregion test_all_fields
# region test_empty_target_key_cols [TYPE Function]
# @PURPOSE: Empty target_key_cols produces just translated_text, context, is_original.
def test_empty_target_key_cols(self, build_request) -> None:
req = build_request(target_key_cols=[])
result = _build_expected_columns(req)
names = [c.name for c in result]
assert names == ["translated_text", "context", "is_original"]
# endregion test_empty_target_key_cols
# region test_translation_column_fallback [TYPE Function]
# @PURPOSE: When target_column is None, translation_column is used as fallback.
def test_translation_column_fallback(self, build_request) -> None:
req = build_request(target_column=None, translation_column="source_col")
result = _build_expected_columns(req)
names = [c.name for c in result]
assert "source_col" in names
assert names[1] == "source_col"
# endregion test_translation_column_fallback
# region test_dedup_same_names [TYPE Function]
# @PURPOSE: Duplicate column names are removed preserving order.
def test_dedup_same_names(self, build_request) -> None:
req = build_request(
target_key_cols=["id", "id"],
target_column="name",
target_language_column="name",
)
result = _build_expected_columns(req)
names = [c.name for c in result]
assert names.count("id") == 1
assert names.count("name") == 1
# endregion test_dedup_same_names
# region test_empty_request [TYPE Function]
# @PURPOSE: No key cols and no target/translation column produces just context + is_original.
def test_empty_request(self, build_request) -> None:
req = build_request(
target_key_cols=[],
target_column=None,
translation_column=None,
)
result = _build_expected_columns(req)
names = [c.name for c in result]
assert names == ["context", "is_original"]
# endregion test_empty_request
# region test_column_order_preserved [TYPE Function]
# @PURPOSE: Column order is preserved: key_cols, target, lang, source, src_lang, context, is_original.
def test_column_order_preserved(self, build_request) -> None:
req = build_request(
target_key_cols=["pk1", "pk2"],
target_column="target_col",
target_language_column="lang_col",
target_source_column="src_col",
target_source_language_column="src_lang_col",
)
result = _build_expected_columns(req)
names = [c.name for c in result]
expected_order = [
"pk1", "pk2", "target_col", "lang_col",
"src_col", "src_lang_col", "context", "is_original",
]
assert names == expected_order
# endregion test_column_order_preserved
# endregion TestBuildExpectedColumns
# region TestParseColumnResults [TYPE Class]
# @PURPOSE: Tests for _parse_column_results response parsing.
class TestParseColumnResults:
# region test_superset_sync_format [TYPE Function]
# @PURPOSE: Standard Superset sync mode (columns + data directly in response).
def test_superset_sync_format(self) -> None:
result = {
"columns": [
{"name": "id", "type": "integer", "is_nullable": "YES"},
{"name": "name", "type": "varchar", "is_nullable": "NO"},
],
"data": [{"id": 1, "name": "test"}],
}
cols, exists = _parse_column_results({"raw_response": result})
assert exists is True
assert len(cols) == 2
assert cols[0]["name"] == "id"
assert cols[0]["type"] == "integer"
assert cols[0]["is_nullable"] is True
assert cols[1]["name"] == "name"
assert cols[1]["type"] == "varchar"
assert cols[1]["is_nullable"] is False
# endregion test_superset_sync_format
# region test_async_polling_format [TYPE Function]
# @PURPOSE: Async polling format (result -> columns + data).
def test_async_polling_format(self) -> None:
result = {
"result": {
"columns": [
{"name": "col1", "type": "text"},
],
"data": [{"col1": "val1"}],
}
}
cols, exists = _parse_column_results({"raw_response": result})
assert exists is True
assert len(cols) == 1
assert cols[0]["name"] == "col1"
# endregion test_async_polling_format
# region test_get_query_results_format [TYPE Function]
# @PURPOSE: get_query_results format (results -> columns + data).
def test_get_query_results_format(self) -> None:
result = {
"results": {
"columns": [
{"name": "a", "type": "int"},
],
"data": [{"a": 1}],
}
}
cols, exists = _parse_column_results({"raw_response": result})
assert exists is True
assert len(cols) == 1
assert cols[0]["name"] == "a"
# endregion test_get_query_results_format
# region test_deeply_nested_get_query_results [TYPE Function]
# @PURPOSE: get_query_results with nested result dict (results -> result -> columns + data).
def test_deeply_nested_get_query_results(self) -> None:
result = {
"results": {
"result": {
"columns": [
{"name": "x", "type": "float"},
],
"data": [{"x": 1.0}],
}
}
}
cols, exists = _parse_column_results({"raw_response": result})
assert exists is True
assert len(cols) == 1
assert cols[0]["name"] == "x"
# endregion test_deeply_nested_get_query_results
# region test_empty_result [TYPE Function]
# @PURPOSE: Empty result dict returns ([], False).
def test_empty_result(self) -> None:
cols, exists = _parse_column_results({"raw_response": {}})
assert cols == []
assert exists is False
# endregion test_empty_result
# region test_none_result [TYPE Function]
# @PURPOSE: None result returns ([], False) via early guard.
def test_none_result(self) -> None:
cols, exists = _parse_column_results(None)
assert cols == []
assert exists is False
# endregion test_none_result
# region test_raw_response_none [TYPE Function]
# @PURPOSE: raw_response=None causes TypeError — known production gap (not guarded).
def test_raw_response_none(self) -> None:
with pytest.raises(TypeError):
_parse_column_results({"raw_response": None})
# endregion test_raw_response_none
# region test_result_without_columns [TYPE Function]
# @PURPOSE: Result without columns key returns ([], False).
def test_result_without_columns(self) -> None:
result = {"status": "success", "data": []}
cols, exists = _parse_column_results({"raw_response": result})
assert cols == []
assert exists is False
# endregion test_result_without_columns
# region test_table_exists_with_data [TYPE Function]
# @PURPOSE: Non-empty data means table exists.
def test_table_exists_with_data(self) -> None:
result = {
"columns": [{"name": "c"}],
"data": [{"c": "v"}],
}
_, exists = _parse_column_results({"raw_response": result})
assert exists is True
# endregion test_table_exists_with_data
# region test_table_not_exists_empty_data [TYPE Function]
# @PURPOSE: Empty data means table does not exist.
def test_table_not_exists_empty_data(self) -> None:
result = {
"columns": [{"name": "c"}],
"data": [],
}
_, exists = _parse_column_results({"raw_response": result})
assert exists is False
# endregion test_table_not_exists_empty_data
# region test_alternate_column_key_names [TYPE Function]
# @PURPOSE: Columns with alternate key names (column_name, Column) parsed correctly.
def test_alternate_column_key_names(self) -> None:
result = {
"columns": [
{"column_name": "id", "data_type": "int"},
{"Column": "name", "Type": "text"},
],
"data": [],
}
cols, _ = _parse_column_results({"raw_response": result})
assert len(cols) == 2
assert cols[0]["name"] == "id"
assert cols[1]["name"] == "name"
# endregion test_alternate_column_key_names
# endregion TestParseColumnResults
# region TestValidateTargetTableSchema [TYPE Class]
# @PURPOSE: Tests for validate_target_table_schema with mocked Superset executor.
class TestValidateTargetTableSchema:
# region test_all_columns_present [TYPE Function]
# @PURPOSE: All expected columns present in target table.
@patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor")
def test_all_columns_present(self, mock_executor, build_request) -> None:
req = build_request()
config_manager = MagicMock()
instance = mock_executor.return_value
instance.resolve_database_id.return_value = None
instance.execute_and_poll.return_value = {
"status": "success",
"raw_response": {
"columns": [
{"name": "id", "type": "integer", "is_nullable": "YES"},
{"name": "translated_text", "type": "text", "is_nullable": "YES"},
{"name": "context", "type": "text", "is_nullable": "YES"},
{"name": "is_original", "type": "boolean", "is_nullable": "YES"},
],
"data": [{"id": 1}],
},
}
response = validate_target_table_schema(req, config_manager)
assert response.table_exists is True
assert response.error is None
assert response.all_present is True
assert len(response.missing_columns) == 0
assert len(response.expected_columns) == 4
assert len(response.actual_columns) == 4
# endregion test_all_columns_present
# region test_missing_columns [TYPE Function]
# @PURPOSE: Target table is missing some expected columns.
@patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor")
def test_missing_columns(self, mock_executor, build_request) -> None:
req = build_request()
config_manager = MagicMock()
instance = mock_executor.return_value
instance.resolve_database_id.return_value = None
instance.execute_and_poll.return_value = {
"status": "success",
"raw_response": {
"columns": [
{"name": "id", "type": "integer", "is_nullable": "YES"},
{"name": "context", "type": "text", "is_nullable": "YES"},
],
"data": [{"id": 1}],
},
}
response = validate_target_table_schema(req, config_manager)
assert response.table_exists is True
assert response.all_present is False
missing_names = {c.name for c in response.missing_columns}
assert "translated_text" in missing_names
assert "is_original" in missing_names
assert "id" not in missing_names
# endregion test_missing_columns
# region test_superset_failed_status [TYPE Function]
# @PURPOSE: Superset returns failed status.
@patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor")
def test_superset_failed_status(self, mock_executor, build_request) -> None:
req = build_request()
config_manager = MagicMock()
instance = mock_executor.return_value
instance.resolve_database_id.return_value = None
instance.execute_and_poll.return_value = {
"status": "failed",
"error_message": "Query timeout",
}
response = validate_target_table_schema(req, config_manager)
assert response.table_exists is False
assert response.error is not None
assert "Query timeout" in response.error
assert response.all_present is False
# endregion test_superset_failed_status
# region test_executor_exception [TYPE Function]
# @PURPOSE: Executor throws an exception.
@patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor")
def test_executor_exception(self, mock_executor, build_request) -> None:
req = build_request()
config_manager = MagicMock()
instance = mock_executor.return_value
instance.resolve_database_id.return_value = None
instance.execute_and_poll.side_effect = ConnectionError("Superset API unavailable")
response = validate_target_table_schema(req, config_manager)
assert response.table_exists is False
assert response.error is not None
assert "Superset API unavailable" in response.error
assert response.all_present is False
# endregion test_executor_exception
# region test_sql_injection_table_name [TYPE Function]
# @PURPOSE: Invalid table name with space is rejected with error.
def test_sql_injection_table_name(self, build_request) -> None:
req = build_request(target_table="my table; DROP TABLE students; --")
config_manager = MagicMock()
response = validate_target_table_schema(req, config_manager)
assert response.table_exists is False
assert response.error is not None
assert "Invalid target table name" in response.error
assert response.all_present is False
# endregion test_sql_injection_table_name
# region test_sql_injection_schema_name [TYPE Function]
# @PURPOSE: Invalid schema name with DROP is rejected with error.
def test_sql_injection_schema_name(self, build_request) -> None:
req = build_request(target_schema="public; DROP TABLE students; --")
config_manager = MagicMock()
response = validate_target_table_schema(req, config_manager)
assert response.table_exists is False
assert response.error is not None
assert "Invalid target schema name" in response.error
assert response.all_present is False
# endregion test_sql_injection_schema_name
# region test_table_not_exists [TYPE Function]
# @PURPOSE: Target table does not exist (empty data from information_schema).
@patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor")
def test_table_not_exists(self, mock_executor, build_request) -> None:
req = build_request()
config_manager = MagicMock()
instance = mock_executor.return_value
instance.resolve_database_id.return_value = None
instance.execute_and_poll.return_value = {
"status": "success",
"raw_response": {
"columns": [
{"name": "id", "type": "integer"},
],
"data": [],
},
}
response = validate_target_table_schema(req, config_manager)
assert response.table_exists is False
assert response.error is None
assert response.all_present is False
assert len(response.missing_columns) == 3 # translated_text, context, is_original
# endregion test_table_not_exists
# endregion TestValidateTargetTableSchema
# endregion TargetSchemaValidationTests

View File

@@ -0,0 +1,269 @@
# #region TargetSchemaValidation [C:3] [TYPE Module] [SEMANTICS translate, schema, validation, target-table]
# @BRIEF Проверка схемы целевой таблицы: запрос колонок через Superset SQL Lab,
# сравнение с ожидаемыми (из build_columns), возврат diff.
# @LAYER Service
# @RELATION DEPENDS_ON -> [SupersetSqlLabExecutor]
# @RELATION DEPENDS_ON -> [schemas.translate.TargetSchemaValidationRequest]
# @RELATION DEPENDS_ON -> [schemas.translate.TargetSchemaValidationResponse]
# @PRE Superset окружение доступно, target_database_id валиден.
# @POST Возвращает актуальные, ожидаемые, отсутствующие и лишние колонки.
# @SIDE_EFFECT Выполняет SQL-запрос через Superset SQL Lab.
# @INVARIANT Если таблица не существует — table_exists=False без ошибки.
# @INVARIANT Если Superset недоступен — error с описанием, all_present=False.
# @INVARIANT Логика ожидаемых колонок sync с build_columns() в orchestrator_sql_rows.py.
# #endregion TargetSchemaValidation
import logging
import re
from typing import Any
from ...schemas.translate import (
TargetSchemaColumnInfo,
TargetSchemaValidationRequest,
TargetSchemaValidationResponse,
)
from ...core.config_manager import ConfigManager
from .superset_executor import SupersetSqlLabExecutor
logger = logging.getLogger(__name__)
_TABLE_NAME_RE = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$')
# #region _build_expected_columns [C:2] [TYPE Function]
# @BRIEF Собирает список ожидаемых колонок по конфигурации column mapping.
# Логика идентична build_columns() из orchestrator_sql_rows.py,
# но не требует объекта TranslationJob.
def _build_expected_columns(req: TargetSchemaValidationRequest) -> list[TargetSchemaColumnInfo]:
"""
Определяет, какие колонки INSERT будет пытаться заполнить,
на основе конфигурации column mapping.
"""
names: list[str] = []
# Ключевые колонки (user-defined, для upsert matching)
if req.target_key_cols:
names.extend(req.target_key_cols)
# Целевая колонка для переведённого текста
effective_target = req.target_column or req.translation_column
if effective_target:
names.append(effective_target)
# Колонка с кодом языка перевода
if req.target_language_column:
names.append(req.target_language_column)
# Колонка с исходным текстом (source reference)
if req.target_source_column:
names.append(req.target_source_column)
# Колонка с определённым исходным языком
if req.target_source_language_column:
names.append(req.target_source_language_column)
# Служебные колонки — всегда
names.append("context")
names.append("is_original")
# Дедупликация с сохранением порядка
seen: set[str] = set()
deduped: list[str] = []
for c in names:
if c and c not in seen:
deduped.append(c)
seen.add(c)
return [TargetSchemaColumnInfo(name=n) for n in deduped]
# #endregion _build_expected_columns
# #region _parse_column_results [C:2] [TYPE Function]
# @BRIEF Извлекает информацию о колонках из ответа Superset SQL Lab.
#
# Superset SQL Lab возвращает разные форматы в зависимости от версии.
# Эта функция ищет колонки во всех возможных местах ответа.
def _parse_column_results(result: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
"""
Парсит результат SQL Lab запроса для извлечения списка колонок таблицы.
Returns (columns_info, table_exists).
"""
if not result:
return [], False
raw = result.get("raw_response", result)
# Пробуем извлечь данные из разных форматов ответа Superset
columns_data = None
data_rows = None
# Формат 1: прямой ответ с колонками (sync mode)
if "columns" in raw and isinstance(raw["columns"], list):
columns_data = raw["columns"]
data_rows = raw.get("data") if isinstance(raw.get("data"), list) else None
# Формат 2: вложенный result (async polling)
elif "result" in raw and isinstance(raw["result"], dict):
res = raw["result"]
if "columns" in res:
columns_data = res["columns"]
data_rows = res.get("data") if isinstance(res.get("data"), list) else None
# Формат 3: ключ results (get_query_results)
elif "results" in raw and isinstance(raw["results"], dict):
res = raw["results"]
if "columns" in res:
columns_data = res["columns"]
if "data" in res:
data_rows = res["data"]
if isinstance(res.get("result"), dict):
sub = res["result"]
if not columns_data and "columns" in sub:
columns_data = sub["columns"]
if not data_rows and "data" in sub:
data_rows = sub["data"]
if not columns_data:
return [], False
# Проверяем существование таблицы: есть ли хотя бы одна строка
table_exists = data_rows is not None and len(data_rows) > 0
# Извлекаем имена и типы колонок из метаданных ответа
columns_info: list[dict[str, Any]] = []
for col in columns_data:
col_name = (
col.get("name")
or col.get("column_name")
or col.get("Column", "")
)
col_type = (
col.get("type")
or col.get("data_type")
or col.get("Type", "")
or col.get("col_type", "")
)
is_nullable = col.get("is_nullable")
# PostgreSQL returns 'YES'/'NO' for is_nullable
if isinstance(is_nullable, str):
is_nullable = is_nullable.upper() == "YES"
if col_name:
columns_info.append({
"name": str(col_name),
"type": str(col_type) if col_type else None,
"is_nullable": is_nullable if isinstance(is_nullable, bool) else True,
})
return columns_info, table_exists
# #endregion _parse_column_results
# #region validate_target_table_schema [C:3] [TYPE Function]
# @BRIEF Основная функция: проверяет схему целевой таблицы через Superset SQL Lab.
def validate_target_table_schema(
req: TargetSchemaValidationRequest,
config_manager: ConfigManager,
) -> TargetSchemaValidationResponse:
"""
Проверяет схему целевой таблицы:
1. Вычисляет ожидаемые колонки (build_columns логика)
2. Запрашивает актуальные колонки через Superset SQL Lab
(SELECT column_name, data_type, is_nullable FROM information_schema.columns ...)
3. Сравнивает и возвращает diff
"""
# 1. Ожидаемые колонки
expected = _build_expected_columns(req)
expected_names = {c.name for c in expected}
# 2. Валидация имён схемы и таблицы (защита от SQL injection в identifier)
if req.target_table and not _TABLE_NAME_RE.match(req.target_table):
return TargetSchemaValidationResponse(
table_exists=False,
error=f"Invalid target table name: '{req.target_table}'. Only alphanumeric and underscore allowed.",
expected_columns=expected,
actual_columns=[],
missing_columns=expected,
extra_columns=[],
all_present=False,
)
if req.target_schema and not _TABLE_NAME_RE.match(req.target_schema):
return TargetSchemaValidationResponse(
table_exists=False,
error=f"Invalid target schema name: '{req.target_schema}'. Only alphanumeric and underscore allowed.",
expected_columns=expected,
actual_columns=[],
missing_columns=expected,
extra_columns=[],
all_present=False,
)
# 3. Запрос через Superset SQL Lab
try:
executor = SupersetSqlLabExecutor(config_manager, req.environment_id)
executor.resolve_database_id(target_database_id=req.target_database_id)
safe_schema = (req.target_schema or "public").replace("'", "''")
safe_table = req.target_table.replace("'", "''")
sql = (
f"SELECT column_name, data_type, is_nullable "
f"FROM information_schema.columns "
f"WHERE table_schema = '{safe_schema}' "
f"AND table_name = '{safe_table}' "
f"ORDER BY ordinal_position"
)
result = executor.execute_and_poll(sql=sql, max_polls=15, poll_interval_seconds=1.0)
status = result.get("status", "")
if status == "failed":
error_msg = result.get("error_message", "SQL Lab query failed")
return TargetSchemaValidationResponse(
table_exists=False,
error=f"Superset SQL Lab error: {error_msg}",
expected_columns=expected,
actual_columns=[],
missing_columns=expected,
extra_columns=[],
all_present=False,
)
raw_columns, table_exists = _parse_column_results(result)
actual_cols = []
for col in raw_columns:
actual_cols.append(TargetSchemaColumnInfo(
name=col["name"],
data_type=col.get("type"),
is_nullable=col.get("is_nullable", True),
))
actual_names = {c.name for c in actual_cols}
# 3. Diff
missing = [col for col in expected if col.name not in actual_names]
extra = [col for col in actual_cols if col.name not in expected_names]
return TargetSchemaValidationResponse(
table_exists=table_exists,
expected_columns=expected,
actual_columns=actual_cols,
missing_columns=missing,
extra_columns=extra,
all_present=len(missing) == 0,
)
except Exception as e:
logger.warning("Target schema validation failed: %s", str(e), exc_info=True)
return TargetSchemaValidationResponse(
table_exists=False,
error=f"Failed to validate target schema: {e}",
expected_columns=expected,
actual_columns=[],
missing_columns=expected,
extra_columns=[],
all_present=False,
)
# #endregion validate_target_table_schema
# #endregion TargetSchemaValidation