semantics
This commit is contained in:
490
backend/src/plugins/translate/__tests__/test_target_schema.py
Normal file
490
backend/src/plugins/translate/__tests__/test_target_schema.py
Normal file
@@ -0,0 +1,490 @@
|
||||
# region TargetSchemaValidationTests [TYPE Module]
|
||||
# @SEMANTICS: test, translate, target-schema, validation
|
||||
# @PURPOSE: Tests for target table schema validation: _build_expected_columns, _parse_column_results, validate_target_table_schema.
|
||||
# @LAYER: Test
|
||||
# @RELATION: BINDS_TO -> [TargetSchemaValidation]
|
||||
# @TEST_CONTRACT: _build_expected_columns -> list[TargetSchemaColumnInfo]
|
||||
# @TEST_CONTRACT: _parse_column_results -> (list[dict], bool)
|
||||
# @TEST_CONTRACT: validate_target_table_schema -> TargetSchemaValidationResponse
|
||||
# @TEST_EDGE: missing_target_key_cols -> Only context + is_original returned
|
||||
# @TEST_EDGE: deduplication -> Duplicate column names are removed
|
||||
# @TEST_EDGE: empty_result -> Returns ([], False)
|
||||
# @TEST_EDGE: superset_sync_format -> Standard Superset response format parsed
|
||||
# @TEST_EDGE: superset_async_format -> Async polling response format parsed
|
||||
# @TEST_EDGE: get_query_results_format -> get_query_results response format parsed
|
||||
# @TEST_EDGE: executor_failure -> SQL Lab query fails, error returned
|
||||
# @TEST_EDGE: sql_injection_table_name -> Invalid table name rejected with error
|
||||
# @TEST_EDGE: executor_exception -> Exception during execution returns error response
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.schemas.translate import (
|
||||
TargetSchemaColumnInfo,
|
||||
TargetSchemaValidationRequest,
|
||||
)
|
||||
from src.plugins.translate.service_target_schema import (
|
||||
_build_expected_columns,
|
||||
_parse_column_results,
|
||||
validate_target_table_schema,
|
||||
)
|
||||
|
||||
|
||||
# region build_request [TYPE Function]
|
||||
# @PURPOSE: Helper to build a TargetSchemaValidationRequest with defaults.
|
||||
@pytest.fixture
|
||||
def build_request():
|
||||
def _build(**overrides):
|
||||
defaults = {
|
||||
"environment_id": "env-1",
|
||||
"target_database_id": "db-1",
|
||||
"target_schema": "public",
|
||||
"target_table": "my_table",
|
||||
"target_key_cols": ["id"],
|
||||
"target_column": "translated_text",
|
||||
"translation_column": None,
|
||||
"target_language_column": None,
|
||||
"target_source_column": None,
|
||||
"target_source_language_column": None,
|
||||
}
|
||||
defaults.update(overrides)
|
||||
return TargetSchemaValidationRequest(**defaults)
|
||||
|
||||
return _build
|
||||
|
||||
|
||||
# endregion build_request
|
||||
|
||||
|
||||
# region TestBuildExpectedColumns [TYPE Class]
|
||||
# @PURPOSE: Tests for _build_expected_columns logic.
|
||||
class TestBuildExpectedColumns:
|
||||
|
||||
# region test_basic_case [TYPE Function]
|
||||
# @PURPOSE: target_key_cols + target_column produces id, translated_text, context, is_original.
|
||||
def test_basic_case(self, build_request) -> None:
|
||||
req = build_request()
|
||||
result = _build_expected_columns(req)
|
||||
names = [c.name for c in result]
|
||||
assert names == ["id", "translated_text", "context", "is_original"]
|
||||
|
||||
# endregion test_basic_case
|
||||
|
||||
# region test_all_fields [TYPE Function]
|
||||
# @PURPOSE: All optional column fields produce 7 columns.
|
||||
def test_all_fields(self, build_request) -> None:
|
||||
req = build_request(
|
||||
target_language_column="lang",
|
||||
target_source_column="source_text",
|
||||
target_source_language_column="source_lang",
|
||||
)
|
||||
result = _build_expected_columns(req)
|
||||
names = [c.name for c in result]
|
||||
assert len(names) == 7
|
||||
assert "lang" in names
|
||||
assert "source_text" in names
|
||||
assert "source_lang" in names
|
||||
assert "id" in names
|
||||
assert "translated_text" in names
|
||||
assert "context" in names
|
||||
assert "is_original" in names
|
||||
|
||||
# endregion test_all_fields
|
||||
|
||||
# region test_empty_target_key_cols [TYPE Function]
|
||||
# @PURPOSE: Empty target_key_cols produces just translated_text, context, is_original.
|
||||
def test_empty_target_key_cols(self, build_request) -> None:
|
||||
req = build_request(target_key_cols=[])
|
||||
result = _build_expected_columns(req)
|
||||
names = [c.name for c in result]
|
||||
assert names == ["translated_text", "context", "is_original"]
|
||||
|
||||
# endregion test_empty_target_key_cols
|
||||
|
||||
# region test_translation_column_fallback [TYPE Function]
|
||||
# @PURPOSE: When target_column is None, translation_column is used as fallback.
|
||||
def test_translation_column_fallback(self, build_request) -> None:
|
||||
req = build_request(target_column=None, translation_column="source_col")
|
||||
result = _build_expected_columns(req)
|
||||
names = [c.name for c in result]
|
||||
assert "source_col" in names
|
||||
assert names[1] == "source_col"
|
||||
|
||||
# endregion test_translation_column_fallback
|
||||
|
||||
# region test_dedup_same_names [TYPE Function]
|
||||
# @PURPOSE: Duplicate column names are removed preserving order.
|
||||
def test_dedup_same_names(self, build_request) -> None:
|
||||
req = build_request(
|
||||
target_key_cols=["id", "id"],
|
||||
target_column="name",
|
||||
target_language_column="name",
|
||||
)
|
||||
result = _build_expected_columns(req)
|
||||
names = [c.name for c in result]
|
||||
assert names.count("id") == 1
|
||||
assert names.count("name") == 1
|
||||
|
||||
# endregion test_dedup_same_names
|
||||
|
||||
# region test_empty_request [TYPE Function]
|
||||
# @PURPOSE: No key cols and no target/translation column produces just context + is_original.
|
||||
def test_empty_request(self, build_request) -> None:
|
||||
req = build_request(
|
||||
target_key_cols=[],
|
||||
target_column=None,
|
||||
translation_column=None,
|
||||
)
|
||||
result = _build_expected_columns(req)
|
||||
names = [c.name for c in result]
|
||||
assert names == ["context", "is_original"]
|
||||
|
||||
# endregion test_empty_request
|
||||
|
||||
# region test_column_order_preserved [TYPE Function]
|
||||
# @PURPOSE: Column order is preserved: key_cols, target, lang, source, src_lang, context, is_original.
|
||||
def test_column_order_preserved(self, build_request) -> None:
|
||||
req = build_request(
|
||||
target_key_cols=["pk1", "pk2"],
|
||||
target_column="target_col",
|
||||
target_language_column="lang_col",
|
||||
target_source_column="src_col",
|
||||
target_source_language_column="src_lang_col",
|
||||
)
|
||||
result = _build_expected_columns(req)
|
||||
names = [c.name for c in result]
|
||||
expected_order = [
|
||||
"pk1", "pk2", "target_col", "lang_col",
|
||||
"src_col", "src_lang_col", "context", "is_original",
|
||||
]
|
||||
assert names == expected_order
|
||||
|
||||
# endregion test_column_order_preserved
|
||||
# endregion TestBuildExpectedColumns
|
||||
|
||||
|
||||
# region TestParseColumnResults [TYPE Class]
|
||||
# @PURPOSE: Tests for _parse_column_results response parsing.
|
||||
class TestParseColumnResults:
|
||||
|
||||
# region test_superset_sync_format [TYPE Function]
|
||||
# @PURPOSE: Standard Superset sync mode (columns + data directly in response).
|
||||
def test_superset_sync_format(self) -> None:
|
||||
result = {
|
||||
"columns": [
|
||||
{"name": "id", "type": "integer", "is_nullable": "YES"},
|
||||
{"name": "name", "type": "varchar", "is_nullable": "NO"},
|
||||
],
|
||||
"data": [{"id": 1, "name": "test"}],
|
||||
}
|
||||
cols, exists = _parse_column_results({"raw_response": result})
|
||||
assert exists is True
|
||||
assert len(cols) == 2
|
||||
assert cols[0]["name"] == "id"
|
||||
assert cols[0]["type"] == "integer"
|
||||
assert cols[0]["is_nullable"] is True
|
||||
assert cols[1]["name"] == "name"
|
||||
assert cols[1]["type"] == "varchar"
|
||||
assert cols[1]["is_nullable"] is False
|
||||
|
||||
# endregion test_superset_sync_format
|
||||
|
||||
# region test_async_polling_format [TYPE Function]
|
||||
# @PURPOSE: Async polling format (result -> columns + data).
|
||||
def test_async_polling_format(self) -> None:
|
||||
result = {
|
||||
"result": {
|
||||
"columns": [
|
||||
{"name": "col1", "type": "text"},
|
||||
],
|
||||
"data": [{"col1": "val1"}],
|
||||
}
|
||||
}
|
||||
cols, exists = _parse_column_results({"raw_response": result})
|
||||
assert exists is True
|
||||
assert len(cols) == 1
|
||||
assert cols[0]["name"] == "col1"
|
||||
|
||||
# endregion test_async_polling_format
|
||||
|
||||
# region test_get_query_results_format [TYPE Function]
|
||||
# @PURPOSE: get_query_results format (results -> columns + data).
|
||||
def test_get_query_results_format(self) -> None:
|
||||
result = {
|
||||
"results": {
|
||||
"columns": [
|
||||
{"name": "a", "type": "int"},
|
||||
],
|
||||
"data": [{"a": 1}],
|
||||
}
|
||||
}
|
||||
cols, exists = _parse_column_results({"raw_response": result})
|
||||
assert exists is True
|
||||
assert len(cols) == 1
|
||||
assert cols[0]["name"] == "a"
|
||||
|
||||
# endregion test_get_query_results_format
|
||||
|
||||
# region test_deeply_nested_get_query_results [TYPE Function]
|
||||
# @PURPOSE: get_query_results with nested result dict (results -> result -> columns + data).
|
||||
def test_deeply_nested_get_query_results(self) -> None:
|
||||
result = {
|
||||
"results": {
|
||||
"result": {
|
||||
"columns": [
|
||||
{"name": "x", "type": "float"},
|
||||
],
|
||||
"data": [{"x": 1.0}],
|
||||
}
|
||||
}
|
||||
}
|
||||
cols, exists = _parse_column_results({"raw_response": result})
|
||||
assert exists is True
|
||||
assert len(cols) == 1
|
||||
assert cols[0]["name"] == "x"
|
||||
|
||||
# endregion test_deeply_nested_get_query_results
|
||||
|
||||
# region test_empty_result [TYPE Function]
|
||||
# @PURPOSE: Empty result dict returns ([], False).
|
||||
def test_empty_result(self) -> None:
|
||||
cols, exists = _parse_column_results({"raw_response": {}})
|
||||
assert cols == []
|
||||
assert exists is False
|
||||
|
||||
# endregion test_empty_result
|
||||
|
||||
# region test_none_result [TYPE Function]
|
||||
# @PURPOSE: None result returns ([], False) via early guard.
|
||||
def test_none_result(self) -> None:
|
||||
cols, exists = _parse_column_results(None)
|
||||
assert cols == []
|
||||
assert exists is False
|
||||
|
||||
# endregion test_none_result
|
||||
|
||||
# region test_raw_response_none [TYPE Function]
|
||||
# @PURPOSE: raw_response=None causes TypeError — known production gap (not guarded).
|
||||
def test_raw_response_none(self) -> None:
|
||||
with pytest.raises(TypeError):
|
||||
_parse_column_results({"raw_response": None})
|
||||
|
||||
# endregion test_raw_response_none
|
||||
|
||||
# region test_result_without_columns [TYPE Function]
|
||||
# @PURPOSE: Result without columns key returns ([], False).
|
||||
def test_result_without_columns(self) -> None:
|
||||
result = {"status": "success", "data": []}
|
||||
cols, exists = _parse_column_results({"raw_response": result})
|
||||
assert cols == []
|
||||
assert exists is False
|
||||
|
||||
# endregion test_result_without_columns
|
||||
|
||||
# region test_table_exists_with_data [TYPE Function]
|
||||
# @PURPOSE: Non-empty data means table exists.
|
||||
def test_table_exists_with_data(self) -> None:
|
||||
result = {
|
||||
"columns": [{"name": "c"}],
|
||||
"data": [{"c": "v"}],
|
||||
}
|
||||
_, exists = _parse_column_results({"raw_response": result})
|
||||
assert exists is True
|
||||
|
||||
# endregion test_table_exists_with_data
|
||||
|
||||
# region test_table_not_exists_empty_data [TYPE Function]
|
||||
# @PURPOSE: Empty data means table does not exist.
|
||||
def test_table_not_exists_empty_data(self) -> None:
|
||||
result = {
|
||||
"columns": [{"name": "c"}],
|
||||
"data": [],
|
||||
}
|
||||
_, exists = _parse_column_results({"raw_response": result})
|
||||
assert exists is False
|
||||
|
||||
# endregion test_table_not_exists_empty_data
|
||||
|
||||
# region test_alternate_column_key_names [TYPE Function]
|
||||
# @PURPOSE: Columns with alternate key names (column_name, Column) parsed correctly.
|
||||
def test_alternate_column_key_names(self) -> None:
|
||||
result = {
|
||||
"columns": [
|
||||
{"column_name": "id", "data_type": "int"},
|
||||
{"Column": "name", "Type": "text"},
|
||||
],
|
||||
"data": [],
|
||||
}
|
||||
cols, _ = _parse_column_results({"raw_response": result})
|
||||
assert len(cols) == 2
|
||||
assert cols[0]["name"] == "id"
|
||||
assert cols[1]["name"] == "name"
|
||||
|
||||
# endregion test_alternate_column_key_names
|
||||
# endregion TestParseColumnResults
|
||||
|
||||
|
||||
# region TestValidateTargetTableSchema [TYPE Class]
|
||||
# @PURPOSE: Tests for validate_target_table_schema with mocked Superset executor.
|
||||
class TestValidateTargetTableSchema:
|
||||
|
||||
# region test_all_columns_present [TYPE Function]
|
||||
# @PURPOSE: All expected columns present in target table.
|
||||
@patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor")
|
||||
def test_all_columns_present(self, mock_executor, build_request) -> None:
|
||||
req = build_request()
|
||||
config_manager = MagicMock()
|
||||
|
||||
instance = mock_executor.return_value
|
||||
instance.resolve_database_id.return_value = None
|
||||
instance.execute_and_poll.return_value = {
|
||||
"status": "success",
|
||||
"raw_response": {
|
||||
"columns": [
|
||||
{"name": "id", "type": "integer", "is_nullable": "YES"},
|
||||
{"name": "translated_text", "type": "text", "is_nullable": "YES"},
|
||||
{"name": "context", "type": "text", "is_nullable": "YES"},
|
||||
{"name": "is_original", "type": "boolean", "is_nullable": "YES"},
|
||||
],
|
||||
"data": [{"id": 1}],
|
||||
},
|
||||
}
|
||||
|
||||
response = validate_target_table_schema(req, config_manager)
|
||||
assert response.table_exists is True
|
||||
assert response.error is None
|
||||
assert response.all_present is True
|
||||
assert len(response.missing_columns) == 0
|
||||
assert len(response.expected_columns) == 4
|
||||
assert len(response.actual_columns) == 4
|
||||
|
||||
# endregion test_all_columns_present
|
||||
|
||||
# region test_missing_columns [TYPE Function]
|
||||
# @PURPOSE: Target table is missing some expected columns.
|
||||
@patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor")
|
||||
def test_missing_columns(self, mock_executor, build_request) -> None:
|
||||
req = build_request()
|
||||
config_manager = MagicMock()
|
||||
|
||||
instance = mock_executor.return_value
|
||||
instance.resolve_database_id.return_value = None
|
||||
instance.execute_and_poll.return_value = {
|
||||
"status": "success",
|
||||
"raw_response": {
|
||||
"columns": [
|
||||
{"name": "id", "type": "integer", "is_nullable": "YES"},
|
||||
{"name": "context", "type": "text", "is_nullable": "YES"},
|
||||
],
|
||||
"data": [{"id": 1}],
|
||||
},
|
||||
}
|
||||
|
||||
response = validate_target_table_schema(req, config_manager)
|
||||
assert response.table_exists is True
|
||||
assert response.all_present is False
|
||||
missing_names = {c.name for c in response.missing_columns}
|
||||
assert "translated_text" in missing_names
|
||||
assert "is_original" in missing_names
|
||||
assert "id" not in missing_names
|
||||
|
||||
# endregion test_missing_columns
|
||||
|
||||
# region test_superset_failed_status [TYPE Function]
|
||||
# @PURPOSE: Superset returns failed status.
|
||||
@patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor")
|
||||
def test_superset_failed_status(self, mock_executor, build_request) -> None:
|
||||
req = build_request()
|
||||
config_manager = MagicMock()
|
||||
|
||||
instance = mock_executor.return_value
|
||||
instance.resolve_database_id.return_value = None
|
||||
instance.execute_and_poll.return_value = {
|
||||
"status": "failed",
|
||||
"error_message": "Query timeout",
|
||||
}
|
||||
|
||||
response = validate_target_table_schema(req, config_manager)
|
||||
assert response.table_exists is False
|
||||
assert response.error is not None
|
||||
assert "Query timeout" in response.error
|
||||
assert response.all_present is False
|
||||
|
||||
# endregion test_superset_failed_status
|
||||
|
||||
# region test_executor_exception [TYPE Function]
|
||||
# @PURPOSE: Executor throws an exception.
|
||||
@patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor")
|
||||
def test_executor_exception(self, mock_executor, build_request) -> None:
|
||||
req = build_request()
|
||||
config_manager = MagicMock()
|
||||
|
||||
instance = mock_executor.return_value
|
||||
instance.resolve_database_id.return_value = None
|
||||
instance.execute_and_poll.side_effect = ConnectionError("Superset API unavailable")
|
||||
|
||||
response = validate_target_table_schema(req, config_manager)
|
||||
assert response.table_exists is False
|
||||
assert response.error is not None
|
||||
assert "Superset API unavailable" in response.error
|
||||
assert response.all_present is False
|
||||
|
||||
# endregion test_executor_exception
|
||||
|
||||
# region test_sql_injection_table_name [TYPE Function]
|
||||
# @PURPOSE: Invalid table name with space is rejected with error.
|
||||
def test_sql_injection_table_name(self, build_request) -> None:
|
||||
req = build_request(target_table="my table; DROP TABLE students; --")
|
||||
config_manager = MagicMock()
|
||||
|
||||
response = validate_target_table_schema(req, config_manager)
|
||||
assert response.table_exists is False
|
||||
assert response.error is not None
|
||||
assert "Invalid target table name" in response.error
|
||||
assert response.all_present is False
|
||||
|
||||
# endregion test_sql_injection_table_name
|
||||
|
||||
# region test_sql_injection_schema_name [TYPE Function]
|
||||
# @PURPOSE: Invalid schema name with DROP is rejected with error.
|
||||
def test_sql_injection_schema_name(self, build_request) -> None:
|
||||
req = build_request(target_schema="public; DROP TABLE students; --")
|
||||
config_manager = MagicMock()
|
||||
|
||||
response = validate_target_table_schema(req, config_manager)
|
||||
assert response.table_exists is False
|
||||
assert response.error is not None
|
||||
assert "Invalid target schema name" in response.error
|
||||
assert response.all_present is False
|
||||
|
||||
# endregion test_sql_injection_schema_name
|
||||
|
||||
# region test_table_not_exists [TYPE Function]
|
||||
# @PURPOSE: Target table does not exist (empty data from information_schema).
|
||||
@patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor")
|
||||
def test_table_not_exists(self, mock_executor, build_request) -> None:
|
||||
req = build_request()
|
||||
config_manager = MagicMock()
|
||||
|
||||
instance = mock_executor.return_value
|
||||
instance.resolve_database_id.return_value = None
|
||||
instance.execute_and_poll.return_value = {
|
||||
"status": "success",
|
||||
"raw_response": {
|
||||
"columns": [
|
||||
{"name": "id", "type": "integer"},
|
||||
],
|
||||
"data": [],
|
||||
},
|
||||
}
|
||||
|
||||
response = validate_target_table_schema(req, config_manager)
|
||||
assert response.table_exists is False
|
||||
assert response.error is None
|
||||
assert response.all_present is False
|
||||
assert len(response.missing_columns) == 3 # translated_text, context, is_original
|
||||
|
||||
# endregion test_table_not_exists
|
||||
# endregion TestValidateTargetTableSchema
|
||||
# endregion TargetSchemaValidationTests
|
||||
269
backend/src/plugins/translate/service_target_schema.py
Normal file
269
backend/src/plugins/translate/service_target_schema.py
Normal file
@@ -0,0 +1,269 @@
|
||||
# #region TargetSchemaValidation [C:3] [TYPE Module] [SEMANTICS translate, schema, validation, target-table]
|
||||
# @BRIEF Проверка схемы целевой таблицы: запрос колонок через Superset SQL Lab,
|
||||
# сравнение с ожидаемыми (из build_columns), возврат diff.
|
||||
# @LAYER Service
|
||||
# @RELATION DEPENDS_ON -> [SupersetSqlLabExecutor]
|
||||
# @RELATION DEPENDS_ON -> [schemas.translate.TargetSchemaValidationRequest]
|
||||
# @RELATION DEPENDS_ON -> [schemas.translate.TargetSchemaValidationResponse]
|
||||
# @PRE Superset окружение доступно, target_database_id валиден.
|
||||
# @POST Возвращает актуальные, ожидаемые, отсутствующие и лишние колонки.
|
||||
# @SIDE_EFFECT Выполняет SQL-запрос через Superset SQL Lab.
|
||||
# @INVARIANT Если таблица не существует — table_exists=False без ошибки.
|
||||
# @INVARIANT Если Superset недоступен — error с описанием, all_present=False.
|
||||
# @INVARIANT Логика ожидаемых колонок sync с build_columns() в orchestrator_sql_rows.py.
|
||||
# #endregion TargetSchemaValidation
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from ...schemas.translate import (
|
||||
TargetSchemaColumnInfo,
|
||||
TargetSchemaValidationRequest,
|
||||
TargetSchemaValidationResponse,
|
||||
)
|
||||
from ...core.config_manager import ConfigManager
|
||||
from .superset_executor import SupersetSqlLabExecutor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_TABLE_NAME_RE = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$')
|
||||
|
||||
|
||||
# #region _build_expected_columns [C:2] [TYPE Function]
|
||||
# @BRIEF Собирает список ожидаемых колонок по конфигурации column mapping.
|
||||
# Логика идентична build_columns() из orchestrator_sql_rows.py,
|
||||
# но не требует объекта TranslationJob.
|
||||
def _build_expected_columns(req: TargetSchemaValidationRequest) -> list[TargetSchemaColumnInfo]:
|
||||
"""
|
||||
Определяет, какие колонки INSERT будет пытаться заполнить,
|
||||
на основе конфигурации column mapping.
|
||||
"""
|
||||
names: list[str] = []
|
||||
|
||||
# Ключевые колонки (user-defined, для upsert matching)
|
||||
if req.target_key_cols:
|
||||
names.extend(req.target_key_cols)
|
||||
|
||||
# Целевая колонка для переведённого текста
|
||||
effective_target = req.target_column or req.translation_column
|
||||
if effective_target:
|
||||
names.append(effective_target)
|
||||
|
||||
# Колонка с кодом языка перевода
|
||||
if req.target_language_column:
|
||||
names.append(req.target_language_column)
|
||||
|
||||
# Колонка с исходным текстом (source reference)
|
||||
if req.target_source_column:
|
||||
names.append(req.target_source_column)
|
||||
|
||||
# Колонка с определённым исходным языком
|
||||
if req.target_source_language_column:
|
||||
names.append(req.target_source_language_column)
|
||||
|
||||
# Служебные колонки — всегда
|
||||
names.append("context")
|
||||
names.append("is_original")
|
||||
|
||||
# Дедупликация с сохранением порядка
|
||||
seen: set[str] = set()
|
||||
deduped: list[str] = []
|
||||
for c in names:
|
||||
if c and c not in seen:
|
||||
deduped.append(c)
|
||||
seen.add(c)
|
||||
|
||||
return [TargetSchemaColumnInfo(name=n) for n in deduped]
|
||||
# #endregion _build_expected_columns
|
||||
|
||||
|
||||
# #region _parse_column_results [C:2] [TYPE Function]
|
||||
# @BRIEF Извлекает информацию о колонках из ответа Superset SQL Lab.
|
||||
#
|
||||
# Superset SQL Lab возвращает разные форматы в зависимости от версии.
|
||||
# Эта функция ищет колонки во всех возможных местах ответа.
|
||||
def _parse_column_results(result: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
|
||||
"""
|
||||
Парсит результат SQL Lab запроса для извлечения списка колонок таблицы.
|
||||
Returns (columns_info, table_exists).
|
||||
"""
|
||||
if not result:
|
||||
return [], False
|
||||
|
||||
raw = result.get("raw_response", result)
|
||||
|
||||
# Пробуем извлечь данные из разных форматов ответа Superset
|
||||
columns_data = None
|
||||
data_rows = None
|
||||
|
||||
# Формат 1: прямой ответ с колонками (sync mode)
|
||||
if "columns" in raw and isinstance(raw["columns"], list):
|
||||
columns_data = raw["columns"]
|
||||
data_rows = raw.get("data") if isinstance(raw.get("data"), list) else None
|
||||
|
||||
# Формат 2: вложенный result (async polling)
|
||||
elif "result" in raw and isinstance(raw["result"], dict):
|
||||
res = raw["result"]
|
||||
if "columns" in res:
|
||||
columns_data = res["columns"]
|
||||
data_rows = res.get("data") if isinstance(res.get("data"), list) else None
|
||||
|
||||
# Формат 3: ключ results (get_query_results)
|
||||
elif "results" in raw and isinstance(raw["results"], dict):
|
||||
res = raw["results"]
|
||||
if "columns" in res:
|
||||
columns_data = res["columns"]
|
||||
if "data" in res:
|
||||
data_rows = res["data"]
|
||||
if isinstance(res.get("result"), dict):
|
||||
sub = res["result"]
|
||||
if not columns_data and "columns" in sub:
|
||||
columns_data = sub["columns"]
|
||||
if not data_rows and "data" in sub:
|
||||
data_rows = sub["data"]
|
||||
|
||||
if not columns_data:
|
||||
return [], False
|
||||
|
||||
# Проверяем существование таблицы: есть ли хотя бы одна строка
|
||||
table_exists = data_rows is not None and len(data_rows) > 0
|
||||
|
||||
# Извлекаем имена и типы колонок из метаданных ответа
|
||||
columns_info: list[dict[str, Any]] = []
|
||||
for col in columns_data:
|
||||
col_name = (
|
||||
col.get("name")
|
||||
or col.get("column_name")
|
||||
or col.get("Column", "")
|
||||
)
|
||||
col_type = (
|
||||
col.get("type")
|
||||
or col.get("data_type")
|
||||
or col.get("Type", "")
|
||||
or col.get("col_type", "")
|
||||
)
|
||||
is_nullable = col.get("is_nullable")
|
||||
# PostgreSQL returns 'YES'/'NO' for is_nullable
|
||||
if isinstance(is_nullable, str):
|
||||
is_nullable = is_nullable.upper() == "YES"
|
||||
|
||||
if col_name:
|
||||
columns_info.append({
|
||||
"name": str(col_name),
|
||||
"type": str(col_type) if col_type else None,
|
||||
"is_nullable": is_nullable if isinstance(is_nullable, bool) else True,
|
||||
})
|
||||
|
||||
return columns_info, table_exists
|
||||
# #endregion _parse_column_results
|
||||
|
||||
|
||||
# #region validate_target_table_schema [C:3] [TYPE Function]
|
||||
# @BRIEF Основная функция: проверяет схему целевой таблицы через Superset SQL Lab.
|
||||
def validate_target_table_schema(
|
||||
req: TargetSchemaValidationRequest,
|
||||
config_manager: ConfigManager,
|
||||
) -> TargetSchemaValidationResponse:
|
||||
"""
|
||||
Проверяет схему целевой таблицы:
|
||||
1. Вычисляет ожидаемые колонки (build_columns логика)
|
||||
2. Запрашивает актуальные колонки через Superset SQL Lab
|
||||
(SELECT column_name, data_type, is_nullable FROM information_schema.columns ...)
|
||||
3. Сравнивает и возвращает diff
|
||||
"""
|
||||
# 1. Ожидаемые колонки
|
||||
expected = _build_expected_columns(req)
|
||||
expected_names = {c.name for c in expected}
|
||||
|
||||
# 2. Валидация имён схемы и таблицы (защита от SQL injection в identifier)
|
||||
if req.target_table and not _TABLE_NAME_RE.match(req.target_table):
|
||||
return TargetSchemaValidationResponse(
|
||||
table_exists=False,
|
||||
error=f"Invalid target table name: '{req.target_table}'. Only alphanumeric and underscore allowed.",
|
||||
expected_columns=expected,
|
||||
actual_columns=[],
|
||||
missing_columns=expected,
|
||||
extra_columns=[],
|
||||
all_present=False,
|
||||
)
|
||||
if req.target_schema and not _TABLE_NAME_RE.match(req.target_schema):
|
||||
return TargetSchemaValidationResponse(
|
||||
table_exists=False,
|
||||
error=f"Invalid target schema name: '{req.target_schema}'. Only alphanumeric and underscore allowed.",
|
||||
expected_columns=expected,
|
||||
actual_columns=[],
|
||||
missing_columns=expected,
|
||||
extra_columns=[],
|
||||
all_present=False,
|
||||
)
|
||||
|
||||
# 3. Запрос через Superset SQL Lab
|
||||
try:
|
||||
executor = SupersetSqlLabExecutor(config_manager, req.environment_id)
|
||||
executor.resolve_database_id(target_database_id=req.target_database_id)
|
||||
|
||||
safe_schema = (req.target_schema or "public").replace("'", "''")
|
||||
safe_table = req.target_table.replace("'", "''")
|
||||
|
||||
sql = (
|
||||
f"SELECT column_name, data_type, is_nullable "
|
||||
f"FROM information_schema.columns "
|
||||
f"WHERE table_schema = '{safe_schema}' "
|
||||
f"AND table_name = '{safe_table}' "
|
||||
f"ORDER BY ordinal_position"
|
||||
)
|
||||
|
||||
result = executor.execute_and_poll(sql=sql, max_polls=15, poll_interval_seconds=1.0)
|
||||
|
||||
status = result.get("status", "")
|
||||
if status == "failed":
|
||||
error_msg = result.get("error_message", "SQL Lab query failed")
|
||||
return TargetSchemaValidationResponse(
|
||||
table_exists=False,
|
||||
error=f"Superset SQL Lab error: {error_msg}",
|
||||
expected_columns=expected,
|
||||
actual_columns=[],
|
||||
missing_columns=expected,
|
||||
extra_columns=[],
|
||||
all_present=False,
|
||||
)
|
||||
|
||||
raw_columns, table_exists = _parse_column_results(result)
|
||||
|
||||
actual_cols = []
|
||||
for col in raw_columns:
|
||||
actual_cols.append(TargetSchemaColumnInfo(
|
||||
name=col["name"],
|
||||
data_type=col.get("type"),
|
||||
is_nullable=col.get("is_nullable", True),
|
||||
))
|
||||
|
||||
actual_names = {c.name for c in actual_cols}
|
||||
|
||||
# 3. Diff
|
||||
missing = [col for col in expected if col.name not in actual_names]
|
||||
extra = [col for col in actual_cols if col.name not in expected_names]
|
||||
|
||||
return TargetSchemaValidationResponse(
|
||||
table_exists=table_exists,
|
||||
expected_columns=expected,
|
||||
actual_columns=actual_cols,
|
||||
missing_columns=missing,
|
||||
extra_columns=extra,
|
||||
all_present=len(missing) == 0,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Target schema validation failed: %s", str(e), exc_info=True)
|
||||
return TargetSchemaValidationResponse(
|
||||
table_exists=False,
|
||||
error=f"Failed to validate target schema: {e}",
|
||||
expected_columns=expected,
|
||||
actual_columns=[],
|
||||
missing_columns=expected,
|
||||
extra_columns=[],
|
||||
all_present=False,
|
||||
)
|
||||
# #endregion validate_target_table_schema
|
||||
# #endregion TargetSchemaValidation
|
||||
Reference in New Issue
Block a user