busya
8a4310169b
12 known failures — all from Agent 3 new unverified tests (mock setup issues): - 3 dataset_review_routes_extended (DTO field mismatches) - 1 settings_consolidated (dict key access) - 1 llm_analysis_service_coverage (rate_limit mock) - 1 migration_plugin (SessionLocal side_effect exhaustion) - 1 preview (DB query vs dict key) - 5 scheduler (datetime timezone + async mock mismatches) NEW TEST FILES THIS SESSION: - test_batch_insert_coverage.py — 3 tests - test_storage_plugin.py — +3 tests - test_search.py — +2 tests - test_mapper.py — already 100% - test_llm_analysis_migration_v1_to_v2.py — 14 tests - test_llm_async_http.py — +1 test - test_prompt_builder.py — +1 test - test_service_datasource.py — +1 test - test_lang_detect.py — +1 test - test_scheduler.py — +6 tests - test_llm_analysis_service_coverage.py — +15 tests - test_dataset_review_routes_extended.py — +14 tests - test_settings_consolidated.py — +13 tests Modules pushed to 100%: _batch_insert, dictionary_entries, service_datasource, _llm_async_http, prompt_builder, dictionary_crud, _batch_sizer, storage/plugin, mapper.py Session: 7194→7643 tests (+449), 80%→83% raw (+3pp), 93.4%→96.9% real (+3.5pp). Remaining: 12 failures to fix + ~300 statements to reach 98% real coverage.
345 lines
14 KiB
Python
345 lines
14 KiB
Python
# #region Test.DatasourceMetadataService [C:3] [TYPE Module] [SEMANTICS test,datasource,metadata]
|
|
# @BRIEF Tests for service_datasource.py — datasource metadata fetching.
|
|
# @RELATION BINDS_TO -> [DatasourceMetadataService]
|
|
# @TEST_EDGE: empty_backend -> ValueError
|
|
# @TEST_EDGE: unsupported_dialect -> ValueError
|
|
# @TEST_EDGE: no_env_config -> ValueError
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "src"))
|
|
|
|
import pytest
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
from src.plugins.translate.service_datasource import (
|
|
get_dialect_from_database,
|
|
detect_virtual_columns,
|
|
get_datasource_columns,
|
|
fetch_datasource_metadata,
|
|
SUPPORTED_DIALECTS,
|
|
)
|
|
|
|
|
|
class TestGetDialectFromDatabase:
|
|
"""get_dialect_from_database — extract dialect from Superset DB record."""
|
|
|
|
def test_backend_key(self):
|
|
"""Uses 'backend' key."""
|
|
result = get_dialect_from_database({"backend": "postgresql"})
|
|
assert result == "postgresql"
|
|
|
|
def test_engine_fallback(self):
|
|
"""Falls back to 'engine' when 'backend' missing."""
|
|
result = get_dialect_from_database({"engine": "mysql"})
|
|
assert result == "mysql"
|
|
|
|
def test_empty_raises(self):
|
|
"""Empty backend/engine raises ValueError."""
|
|
with pytest.raises(ValueError, match="Could not determine"):
|
|
get_dialect_from_database({})
|
|
|
|
def test_dialect_mapping(self):
|
|
"""Dialect map normalizes known backends."""
|
|
assert get_dialect_from_database({"backend": "greenplum"}) == "postgresql"
|
|
assert get_dialect_from_database({"backend": "clickhousedb"}) == "clickhouse"
|
|
|
|
def test_unsupported_dialect_raises(self):
|
|
"""Unsupported dialect raises ValueError."""
|
|
with pytest.raises(ValueError, match="Unsupported database dialect"):
|
|
get_dialect_from_database({"backend": "mongodb"})
|
|
|
|
def test_case_insensitive(self):
|
|
"""Backend is case-insensitive."""
|
|
result = get_dialect_from_database({"backend": "PostgreSQL"})
|
|
assert result == "postgresql"
|
|
|
|
|
|
class TestDetectVirtualColumns:
|
|
"""detect_virtual_columns — identify computed columns."""
|
|
|
|
def test_detects_virtual(self):
|
|
"""Returns names of non-physical columns."""
|
|
columns = [
|
|
{"name": "id", "is_physical": True},
|
|
{"name": "full_name", "is_physical": False},
|
|
{"name": "email", "is_physical": True},
|
|
]
|
|
virtual = detect_virtual_columns(columns)
|
|
assert virtual == ["full_name"]
|
|
|
|
def test_all_physical(self):
|
|
"""All physical returns empty list."""
|
|
columns = [
|
|
{"name": "id", "is_physical": True},
|
|
{"name": "name", "is_physical": True},
|
|
]
|
|
assert detect_virtual_columns(columns) == []
|
|
|
|
def test_all_virtual(self):
|
|
"""All virtual returns all names."""
|
|
columns = [
|
|
{"name": "calc1", "is_physical": False},
|
|
{"name": "calc2", "is_physical": False},
|
|
]
|
|
virtual = detect_virtual_columns(columns)
|
|
assert virtual == ["calc1", "calc2"]
|
|
|
|
def test_default_is_physical(self):
|
|
"""Default is_physical is True when missing."""
|
|
columns = [{"name": "id"}]
|
|
assert detect_virtual_columns(columns) == []
|
|
|
|
|
|
class TestFetchDatasourceMetadata:
|
|
"""fetch_datasource_metadata — fetch columns + dialect."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_success_with_database_dict(self):
|
|
"""Success when database is a dict."""
|
|
config_manager = MagicMock()
|
|
env_config = MagicMock()
|
|
env_config.id = "env-1"
|
|
config_manager.get_environments.return_value = [env_config]
|
|
|
|
with patch("src.plugins.translate.service_datasource.get_superset_client") as mock_get:
|
|
mock_client = AsyncMock()
|
|
mock_get.return_value = mock_client
|
|
mock_client.get_dataset_detail.return_value = {
|
|
"columns": [
|
|
{"name": "id", "type": "INTEGER", "is_physical": True},
|
|
{"name": "name", "type": "VARCHAR", "is_physical": True},
|
|
],
|
|
"database": {"backend": "postgresql"},
|
|
}
|
|
|
|
columns, dialect = await fetch_datasource_metadata(1, "env-1", config_manager)
|
|
assert len(columns) == 2
|
|
assert dialect == "postgresql"
|
|
assert columns[0]["name"] == "id"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_env_config(self):
|
|
"""Non-existent env raises ValueError."""
|
|
config_manager = MagicMock()
|
|
config_manager.get_environments.return_value = []
|
|
|
|
with pytest.raises(ValueError, match="not found"):
|
|
await fetch_datasource_metadata(1, "env-1", config_manager)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_columns_with_name_fallback(self):
|
|
"""Uses 'column_name' when 'name' is missing."""
|
|
config_manager = MagicMock()
|
|
env_config = MagicMock()
|
|
env_config.id = "env-1"
|
|
config_manager.get_environments.return_value = [env_config]
|
|
|
|
with patch("src.plugins.translate.service_datasource.get_superset_client") as mock_get:
|
|
mock_client = AsyncMock()
|
|
mock_get.return_value = mock_client
|
|
mock_client.get_dataset_detail.return_value = {
|
|
"columns": [
|
|
{"column_name": "my_col", "type": "VARCHAR"},
|
|
],
|
|
"database": {"backend": "postgresql"},
|
|
}
|
|
|
|
columns, dialect = await fetch_datasource_metadata(1, "env-1", config_manager)
|
|
assert len(columns) == 1
|
|
assert columns[0]["name"] == "my_col"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_skips_columns_without_name(self):
|
|
"""Skips columns without name or column_name."""
|
|
config_manager = MagicMock()
|
|
env_config = MagicMock()
|
|
env_config.id = "env-1"
|
|
config_manager.get_environments.return_value = [env_config]
|
|
|
|
with patch("src.plugins.translate.service_datasource.get_superset_client") as mock_get:
|
|
mock_client = AsyncMock()
|
|
mock_get.return_value = mock_client
|
|
mock_client.get_dataset_detail.return_value = {
|
|
"columns": [
|
|
{"name": "valid_col"},
|
|
{"extra": "no_name"},
|
|
{},
|
|
],
|
|
"database": {"backend": "postgresql"},
|
|
}
|
|
|
|
columns, dialect = await fetch_datasource_metadata(1, "env-1", config_manager)
|
|
assert len(columns) == 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_database_info_as_non_dict(self):
|
|
"""When database_info is not a dict (e.g., str), fetches from database_id."""
|
|
config_manager = MagicMock()
|
|
env_config = MagicMock()
|
|
env_config.id = "env-1"
|
|
config_manager.get_environments.return_value = [env_config]
|
|
|
|
with patch("src.plugins.translate.service_datasource.get_superset_client") as mock_get:
|
|
mock_client = AsyncMock()
|
|
mock_get.return_value = mock_client
|
|
mock_client.get_dataset_detail.return_value = {
|
|
"columns": [],
|
|
"database": "not a dict",
|
|
"database_id": 123,
|
|
}
|
|
mock_client.get_database.return_value = {"result": {"backend": "mysql"}}
|
|
|
|
columns, dialect = await fetch_datasource_metadata(1, "env-1", config_manager)
|
|
assert dialect == "mysql"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_database_info_not_dict_no_db_id(self):
|
|
"""When database_info not a dict and no database_id, raises."""
|
|
config_manager = MagicMock()
|
|
env_config = MagicMock()
|
|
env_config.id = "env-1"
|
|
config_manager.get_environments.return_value = [env_config]
|
|
|
|
with patch("src.plugins.translate.service_datasource.get_superset_client") as mock_get:
|
|
mock_client = AsyncMock()
|
|
mock_get.return_value = mock_client
|
|
mock_client.get_dataset_detail.return_value = {
|
|
"columns": [],
|
|
"database": "not a dict",
|
|
}
|
|
|
|
with pytest.raises(ValueError, match="No database information"):
|
|
await fetch_datasource_metadata(1, "env-1", config_manager)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_database_info_not_dict_db_error(self):
|
|
"""Exception when fetching database info raises."""
|
|
config_manager = MagicMock()
|
|
env_config = MagicMock()
|
|
env_config.id = "env-1"
|
|
config_manager.get_environments.return_value = [env_config]
|
|
|
|
with patch("src.plugins.translate.service_datasource.get_superset_client") as mock_get:
|
|
mock_client = AsyncMock()
|
|
mock_get.return_value = mock_client
|
|
mock_client.get_dataset_detail.return_value = {
|
|
"columns": [],
|
|
"database": "not a dict",
|
|
"database_id": 123,
|
|
}
|
|
mock_client.get_database.side_effect = Exception("API error")
|
|
|
|
with pytest.raises(ValueError, match="Could not determine"):
|
|
await fetch_datasource_metadata(1, "env-1", config_manager)
|
|
|
|
|
|
class TestGetDatasourceColumns:
|
|
"""get_datasource_columns — structured column metadata response."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_success(self):
|
|
"""Returns structured response with columns."""
|
|
config_manager = MagicMock()
|
|
env_config = MagicMock()
|
|
env_config.id = "env-1"
|
|
config_manager.get_environments.return_value = [env_config]
|
|
|
|
with patch("src.plugins.translate.service_datasource.get_superset_client") as mock_get:
|
|
mock_client = AsyncMock()
|
|
mock_get.return_value = mock_client
|
|
mock_client.get_dataset_detail.return_value = {
|
|
"columns": [
|
|
{"name": "id", "type": "INTEGER"},
|
|
],
|
|
"database": {"backend": "postgresql"},
|
|
"table_name": "my_table",
|
|
"schema": "public",
|
|
}
|
|
|
|
response = await get_datasource_columns(1, "env-1", config_manager)
|
|
assert response.datasource_id == 1
|
|
assert response.database_dialect == "postgresql"
|
|
assert response.datasource_name == "my_table"
|
|
assert len(response.columns) == 1
|
|
assert response.columns[0].name == "id"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_env_config(self):
|
|
"""Non-existent env raises ValueError."""
|
|
config_manager = MagicMock()
|
|
config_manager.get_environments.return_value = []
|
|
|
|
with pytest.raises(ValueError, match="not found"):
|
|
await get_datasource_columns(1, "env-1", config_manager)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_dialect_from_database_fallback(self):
|
|
"""When dialect from database dict fails, fetches from database_id."""
|
|
config_manager = MagicMock()
|
|
env_config = MagicMock()
|
|
env_config.id = "env-1"
|
|
config_manager.get_environments.return_value = [env_config]
|
|
|
|
with patch("src.plugins.translate.service_datasource.get_superset_client") as mock_get:
|
|
mock_client = AsyncMock()
|
|
mock_get.return_value = mock_client
|
|
# get_dialect_from_database will fail on empty backend
|
|
mock_client.get_dataset_detail.return_value = {
|
|
"columns": [],
|
|
"database": {"not_backend": "mysql"},
|
|
"database_id": 456,
|
|
}
|
|
mock_client.get_database.return_value = {"result": {"backend": "mysql"}}
|
|
|
|
response = await get_datasource_columns(1, "env-1", config_manager)
|
|
assert response.database_dialect == "mysql"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_database_id_raises(self):
|
|
"""When dialect fails and no database_id, raises."""
|
|
config_manager = MagicMock()
|
|
env_config = MagicMock()
|
|
env_config.id = "env-1"
|
|
config_manager.get_environments.return_value = [env_config]
|
|
|
|
with patch("src.plugins.translate.service_datasource.get_superset_client") as mock_get:
|
|
mock_client = AsyncMock()
|
|
mock_get.return_value = mock_client
|
|
mock_client.get_dataset_detail.return_value = {
|
|
"columns": [],
|
|
"database": {"not_backend": "xxx"},
|
|
}
|
|
|
|
with pytest.raises(ValueError, match="Could not determine"):
|
|
await get_datasource_columns(1, "env-1", config_manager)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_skips_column_without_name(self):
|
|
"""Columns without name or column_name are skipped (line 159)."""
|
|
config_manager = MagicMock()
|
|
env_config = MagicMock()
|
|
env_config.id = "env-1"
|
|
config_manager.get_environments.return_value = [env_config]
|
|
|
|
with patch("src.plugins.translate.service_datasource.get_superset_client") as mock_get:
|
|
mock_client = AsyncMock()
|
|
mock_get.return_value = mock_client
|
|
mock_client.get_dataset_detail.return_value = {
|
|
"columns": [
|
|
{"name": "valid_col", "type": "INTEGER"},
|
|
{"not_name": "should_skip", "type": "VARCHAR"},
|
|
{},
|
|
{"column_name": "also_valid", "type": "TEXT"},
|
|
],
|
|
"database": {"backend": "postgresql"},
|
|
"table_name": "t1",
|
|
"schema": "public",
|
|
}
|
|
|
|
response = await get_datasource_columns(1, "env-1", config_manager)
|
|
assert len(response.columns) == 2
|
|
assert response.columns[0].name == "valid_col"
|
|
assert response.columns[1].name == "also_valid"
|
|
# #endregion Test.DatasourceMetadataService
|