busya
154c9bb3b1
QA AGENT FINDINGS (new issues not in audit): 1. Legacy @PURPOSE→@BRIEF: test_datasets.py (44 occurrences) 2. Legacy @SEMANTICS:→[SEMANTICS]: test_superset_matrix.py, test_smoke_app.py 3. @PRE/@POST on C2 functions: test_models.py violation 4. Unclosed #endregion anchors: test_datasets.py (56→1), test_db_executor.py, etc. FIXES APPLIED: - Module #region anchors added: test_smoke_plugins.py, test_models.py, api/test_tasks.py, core/test_defensive_guards.py - @RELATION BINDS_TO added: 14 files - @TEST_EDGE added (≥3 each): 16 files - Legacy syntax converted: test_datasets.py, test_superset_matrix.py, test_smoke_app.py - @PRE/@POST removed from C2 functions: test_models.py - Unclosed #endregion fixed: test_smoke_app.py, test_db_executor.py, test_connection_service.py, test_orchestrator_direct_db.py VERIFIED: 7778/7778 tests pass, 0 new failures REMAINING: 947 @BRIEF gaps, 165 @TEST_EDGE gaps, 38 oversized files
1047 lines
40 KiB
Python
1047 lines
40 KiB
Python
# #region Test.DatasetsApi [C:3] [TYPE Module] [SEMANTICS test, datasets, api, metrics, inline-edit]
|
|
# @BRIEF Contract tests for dataset endpoints — list+stats, detail+metrics, inline-edit descriptions, map-columns, generate-docs.
|
|
# @RELATION BINDS_TO -> [DatasetsApi]
|
|
# @TEST_EDGE: missing_env -> 404 when env not found
|
|
# @TEST_EDGE: invalid_page -> 400 when page < 1 or page_size > 100
|
|
# @TEST_EDGE: external_superset_fail -> 503 when Superset/API fails
|
|
# @TEST_EDGE: html_injection -> HTML tags stripped from descriptions
|
|
# @TEST_EDGE: oversized_description -> 400 when description exceeds 2000 chars
|
|
import pytest
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
from fastapi.testclient import TestClient
|
|
|
|
from src.app import app
|
|
from src.dependencies import (
|
|
get_config_manager,
|
|
get_current_user,
|
|
get_resource_service,
|
|
get_task_manager,
|
|
has_permission,
|
|
)
|
|
|
|
client = TestClient(app)
|
|
|
|
# #region mock_user [C:2] [TYPE Function]
|
|
# @BRIEF Create a mock user with Admin role and all permissions for testing.
|
|
def _make_mock_user():
|
|
admin_role = MagicMock()
|
|
admin_role.name = "Admin"
|
|
admin_role.permissions = []
|
|
|
|
mock_user = MagicMock()
|
|
mock_user.username = "test-user"
|
|
mock_user.roles = [admin_role]
|
|
return mock_user
|
|
|
|
|
|
# #region mock_deps [C:2] [TYPE Function]
|
|
# @BRIEF Provide dependency override fixture for dataset route tests.
|
|
# @TEST_FIXTURE: dataset_route_overrides -> INLINE_JSON
|
|
@pytest.fixture
|
|
def mock_deps():
|
|
config_manager = MagicMock()
|
|
task_manager = MagicMock()
|
|
resource_service = MagicMock()
|
|
|
|
env = MagicMock()
|
|
env.id = "ss-dev"
|
|
env.name = "SS Dev"
|
|
env.superset_url = "http://superset:8088"
|
|
env.superset_token = "mock-token"
|
|
config_manager.get_environments.return_value = [env]
|
|
|
|
task_manager.get_all_tasks.return_value = []
|
|
|
|
resource_service.get_datasets_with_status = AsyncMock(
|
|
return_value=[
|
|
{
|
|
"id": 1,
|
|
"table_name": "users",
|
|
"schema": "public",
|
|
"database": "postgres",
|
|
"mapped_fields": {"total": 10, "mapped": 7},
|
|
"last_task": {"task_id": "t1", "status": "SUCCESS"},
|
|
"linked_dashboard_count": 2,
|
|
},
|
|
{
|
|
"id": 2,
|
|
"table_name": "orders",
|
|
"schema": "public",
|
|
"database": "postgres",
|
|
"mapped_fields": {"total": 5, "mapped": 0},
|
|
"last_task": None,
|
|
"linked_dashboard_count": 0,
|
|
},
|
|
{
|
|
"id": 3,
|
|
"table_name": "products",
|
|
"schema": "public",
|
|
"database": "postgres",
|
|
"mapped_fields": None,
|
|
"last_task": None,
|
|
"linked_dashboard_count": 0,
|
|
},
|
|
]
|
|
)
|
|
|
|
mock_user = _make_mock_user()
|
|
|
|
app.dependency_overrides[get_config_manager] = lambda: config_manager
|
|
app.dependency_overrides[get_task_manager] = lambda: task_manager
|
|
app.dependency_overrides[get_resource_service] = lambda: resource_service
|
|
app.dependency_overrides[get_current_user] = lambda: mock_user
|
|
# Override specific permission dependencies used by datasets routes
|
|
app.dependency_overrides[has_permission("plugin:migration", "READ")] = lambda: mock_user
|
|
app.dependency_overrides[has_permission("plugin:migration", "WRITE")] = lambda: mock_user
|
|
|
|
yield
|
|
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
# #region test_get_datasets_returns_stats [C:2] [TYPE Function]
|
|
# @BRIEF Verify GET /api/datasets returns stats object with correct counts.
|
|
# @TEST_CONTRACT: datasets_list -> datasets with stats payload
|
|
# @TEST_SCENARIO: datasets_with_stats -> HTTP 200 returns datasets, stats, and pagination.
|
|
# @TEST_INVARIANT: stats_counts_correct -> VERIFIED_BY: [datasets_with_stats]
|
|
def test_get_datasets_returns_stats(mock_deps):
|
|
response = client.get(
|
|
"/api/datasets?env_id=ss-dev&page=1&page_size=10"
|
|
)
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert "datasets" in data
|
|
assert "stats" in data
|
|
assert data["stats"]["total"] == 3
|
|
assert data["stats"]["mapped_count"] == 1 # users: 7/10
|
|
assert data["stats"]["unmapped_count"] == 1 # orders: 0/5
|
|
assert data["stats"]["linked_count"] == 1 # users has 2 linked dashboards
|
|
assert data["total"] == 3
|
|
assert data["page"] == 1
|
|
assert len(data["datasets"]) == 3
|
|
|
|
|
|
# #region test_get_datasets_filter_unmapped [C:2] [TYPE Function]
|
|
# @BRIEF Verify server-side filtering by unmapped datasets.
|
|
# @TEST_SCENARIO: filter_unmapped -> Only returns datasets with mapped=0.
|
|
def test_get_datasets_filter_unmapped(mock_deps):
|
|
response = client.get(
|
|
"/api/datasets?env_id=ss-dev&filter=unmapped&page=1&page_size=10"
|
|
)
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
# stats are global (unfiltered)
|
|
assert data["stats"]["total"] == 3
|
|
# filtered list: only orders (mapped=0)
|
|
assert data["total"] == 1
|
|
assert len(data["datasets"]) == 1
|
|
assert data["datasets"][0]["table_name"] == "orders"
|
|
|
|
|
|
# #region test_get_datasets_filter_mapped [C:2] [TYPE Function]
|
|
# @BRIEF Verify server-side filtering by mapped datasets.
|
|
def test_get_datasets_filter_mapped(mock_deps):
|
|
response = client.get(
|
|
"/api/datasets?env_id=ss-dev&filter=mapped&page=1&page_size=10"
|
|
)
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["stats"]["total"] == 3
|
|
assert data["total"] == 1
|
|
assert data["datasets"][0]["table_name"] == "users"
|
|
|
|
|
|
# #region test_get_datasets_filter_linked [C:2] [TYPE Function]
|
|
# @BRIEF Verify server-side filtering by datasets linked to dashboards.
|
|
def test_get_datasets_filter_linked(mock_deps):
|
|
response = client.get(
|
|
"/api/datasets?env_id=ss-dev&filter=linked&page=1&page_size=10"
|
|
)
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["stats"]["total"] == 3
|
|
assert data["total"] == 1
|
|
assert data["datasets"][0]["table_name"] == "users"
|
|
|
|
|
|
# #region test_get_datasets_filter_all [C:2] [TYPE Function]
|
|
# @BRIEF Verify filter=all returns all datasets (same as no filter).
|
|
def test_get_datasets_filter_all(mock_deps):
|
|
response = client.get(
|
|
"/api/datasets?env_id=ss-dev&filter=all&page=1&page_size=10"
|
|
)
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["total"] == 3
|
|
|
|
|
|
# #region test_get_dataset_detail_returns_metrics [C:2] [TYPE Function]
|
|
# @BRIEF Verify GET /api/datasets/{id} returns metrics and metric_count.
|
|
# @TEST_SCENARIO: detail_with_metrics -> Response includes metrics list and metric_count.
|
|
def test_get_dataset_detail_returns_metrics(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
|
|
mock_client.get_dataset_detail.return_value = {
|
|
"id": 1,
|
|
"table_name": "users",
|
|
"schema": "public",
|
|
"database": "postgres",
|
|
"description": "User table",
|
|
"columns": [
|
|
{"id": 10, "name": "id", "type": "INT", "is_dttm": False, "is_active": True, "description": "PK"},
|
|
{"id": 11, "name": "name", "type": "STRING", "is_dttm": False, "is_active": True, "description": ""},
|
|
],
|
|
"column_count": 2,
|
|
"metrics": [
|
|
{"id": 100, "metric_name": "count", "expression": "COUNT(*)", "verbose_name": "Row Count", "description": "Total rows", "metric_type": "count"},
|
|
],
|
|
"metric_count": 1,
|
|
"sql": "SELECT * FROM users",
|
|
"linked_dashboards": [{"id": 5, "title": "Users Dashboard", "slug": "users-dash"}],
|
|
"linked_dashboard_count": 1,
|
|
"is_sqllab_view": False,
|
|
"created_on": "2024-01-01T00:00:00",
|
|
"changed_on": "2024-06-01T00:00:00",
|
|
}
|
|
|
|
response = client.get(
|
|
"/api/datasets/1?env_id=ss-dev"
|
|
)
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert "metrics" in data
|
|
assert "metric_count" in data
|
|
assert data["metric_count"] == 1
|
|
assert len(data["metrics"]) == 1
|
|
assert data["metrics"][0]["metric_name"] == "count"
|
|
assert data["metrics"][0]["expression"] == "COUNT(*)"
|
|
|
|
|
|
# #region test_update_column_description [C:2] [TYPE Function]
|
|
# @BRIEF Verify PUT /api/datasets/{id}/columns/{col_id}/description saves description.
|
|
# @TEST_SCENARIO: save_column_desc -> Saves and returns updated description.
|
|
def test_update_column_description(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
|
|
mock_client.get_dataset.return_value = {
|
|
"id": 1,
|
|
"result": {
|
|
"id": 1,
|
|
"columns": [
|
|
{"id": 10, "column_name": "id", "type": "INT", "description": "old desc"},
|
|
{"id": 11, "column_name": "name", "type": "STRING", "description": ""},
|
|
],
|
|
"metrics": [],
|
|
},
|
|
}
|
|
mock_client.update_dataset.return_value = {"id": 1}
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/columns/10/description?env_id=ss-dev",
|
|
json={"description": "new description"},
|
|
)
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["success"] is True
|
|
assert data["description"] == "new description"
|
|
|
|
# Verify update_dataset was called with correct column modified
|
|
update_call = mock_client.update_dataset.call_args
|
|
assert update_call[0][0] == 1 # dataset_id
|
|
# Check override_columns=false was passed
|
|
assert update_call[1]["override_columns"] is False
|
|
|
|
|
|
# #region test_update_metric_description [C:2] [TYPE Function]
|
|
# @BRIEF Verify PUT /api/datasets/{id}/metrics/{metric_id}/description saves correctly.
|
|
# @TEST_SCENARIO: save_metric_desc -> Saves and returns updated description.
|
|
def test_update_metric_description(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
|
|
mock_client.get_dataset.return_value = {
|
|
"id": 1,
|
|
"result": {
|
|
"id": 1,
|
|
"columns": [],
|
|
"metrics": [
|
|
{"id": 100, "metric_name": "count", "expression": "COUNT(*)", "description": ""},
|
|
],
|
|
},
|
|
}
|
|
mock_client.update_dataset.return_value = {"id": 1}
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/metrics/100/description?env_id=ss-dev",
|
|
json={"description": "Total row count"},
|
|
)
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["success"] is True
|
|
assert data["description"] == "Total row count"
|
|
|
|
|
|
# #region test_update_column_description_not_found [C:2] [TYPE Function]
|
|
# @BRIEF Verify 404 when column not found.
|
|
def test_update_column_description_not_found(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
|
|
mock_client.get_dataset.return_value = {
|
|
"id": 1,
|
|
"result": {
|
|
"id": 1,
|
|
"columns": [{"id": 10, "column_name": "id"}],
|
|
"metrics": [],
|
|
},
|
|
}
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/columns/999/description?env_id=ss-dev",
|
|
json={"description": "test"},
|
|
)
|
|
assert response.status_code == 404
|
|
|
|
|
|
# #region test_update_column_description_validation [C:2] [TYPE Function]
|
|
# @BRIEF Verify 400 when description exceeds max length.
|
|
def test_update_column_description_validation(mock_deps):
|
|
response = client.put(
|
|
"/api/datasets/1/columns/10/description?env_id=ss-dev",
|
|
json={"description": "x" * 2001},
|
|
)
|
|
assert response.status_code == 400
|
|
|
|
|
|
# #region test_get_datasets_superset_503 [C:2] [TYPE Function]
|
|
# @BRIEF Verify GET /api/datasets returns 503 when Superset/ResourceService throws.
|
|
# @TEST_SCENARIO: upstream_superset_failure -> 503 with error detail.
|
|
# @TEST_INVARIANT: upstream_failure_503 -> VERIFIED_BY: [superset_failure]
|
|
def test_get_datasets_superset_503(mock_deps):
|
|
# Override resource_service to throw during get_datasets_with_status
|
|
from src.dependencies import get_resource_service
|
|
|
|
failing_resource_service = MagicMock()
|
|
failing_resource_service.get_datasets_with_status = AsyncMock(
|
|
side_effect=Exception("Superset connection refused")
|
|
)
|
|
|
|
app.dependency_overrides[get_resource_service] = lambda: failing_resource_service
|
|
|
|
try:
|
|
response = client.get(
|
|
"/api/datasets?env_id=ss-dev&page=1&page_size=10"
|
|
)
|
|
assert response.status_code == 503
|
|
assert "Superset" in response.text or "Failed" in response.text
|
|
finally:
|
|
app.dependency_overrides[get_resource_service] = None
|
|
|
|
|
|
# #region test_update_column_description_superset_502 [C:2] [TYPE Function]
|
|
# @BRIEF Verify PUT column description returns 502 when SupersetClient.update_dataset throws.
|
|
# @TEST_SCENARIO: superset_put_failure -> 502 with error detail.
|
|
def test_update_column_description_superset_502(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
|
|
# GET succeeds
|
|
mock_client.get_dataset.return_value = {
|
|
"id": 1,
|
|
"result": {
|
|
"id": 1,
|
|
"columns": [{"id": 10, "column_name": "id", "type": "INT", "description": "old desc"}],
|
|
"metrics": [],
|
|
},
|
|
}
|
|
# PUT fails
|
|
mock_client.update_dataset.side_effect = Exception("Superset write failure")
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/columns/10/description?env_id=ss-dev",
|
|
json={"description": "new description"},
|
|
)
|
|
assert response.status_code == 502
|
|
data = response.json()
|
|
assert "Superset" in data.get("detail", "")
|
|
|
|
|
|
# #region test_update_column_description_strips_html [C:2] [TYPE Function]
|
|
# @BRIEF Verify HTML tags are stripped from description before saving.
|
|
# @TEST_SCENARIO: html_in_description -> HTML tags stripped, clean text saved.
|
|
def test_update_column_description_strips_html(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
|
|
mock_client.get_dataset.return_value = {
|
|
"id": 1,
|
|
"result": {
|
|
"id": 1,
|
|
"columns": [{"id": 10, "column_name": "id", "type": "INT", "description": ""}],
|
|
"metrics": [],
|
|
},
|
|
}
|
|
mock_client.update_dataset.return_value = {"id": 1}
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/columns/10/description?env_id=ss-dev",
|
|
json={"description": "Hello <b>World</b> with <script>alert('xss')</script>"},
|
|
)
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
# HTML stripped
|
|
assert data["description"] == "Hello World with alert('xss')"
|
|
assert "<b>" not in data["description"]
|
|
assert "<script>" not in data["description"]
|
|
|
|
# Verify the stripped description was sent to Superset
|
|
update_call = mock_client.update_dataset.call_args
|
|
updated_columns = update_call[0][1]["columns"]
|
|
assert updated_columns[0]["description"] == "Hello World with alert('xss')"
|
|
|
|
|
|
# #region test_dataset_item_has_metric_count [C:2] [TYPE Function]
|
|
# @BRIEF Verify DatasetItem in list response includes metric_count field.
|
|
def test_dataset_item_has_metric_count(mock_deps):
|
|
response = client.get(
|
|
"/api/datasets?env_id=ss-dev&page=1&page_size=10"
|
|
)
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
for ds in data["datasets"]:
|
|
assert "metric_count" in ds
|
|
assert isinstance(ds["metric_count"], int)
|
|
|
|
|
|
# #region test_get_dataset_ids [C:2] [TYPE Function]
|
|
# @BRIEF Verify GET /api/datasets/ids returns list of dataset IDs.
|
|
def test_get_dataset_ids(mock_deps):
|
|
response = client.get("/api/datasets/ids?env_id=ss-dev")
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert "dataset_ids" in data
|
|
assert isinstance(data["dataset_ids"], list)
|
|
assert len(data["dataset_ids"]) == 3
|
|
assert 1 in data["dataset_ids"]
|
|
|
|
|
|
# #region test_get_dataset_ids_search [C:2] [TYPE Function]
|
|
# @BRIEF Verify GET /api/datasets/ids with search filter.
|
|
def test_get_dataset_ids_search(mock_deps):
|
|
response = client.get("/api/datasets/ids?env_id=ss-dev&search=users")
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["dataset_ids"] == [1]
|
|
|
|
|
|
# #region test_get_dataset_ids_env_not_found [C:2] [TYPE Function]
|
|
# @BRIEF Verify 404 when env not found.
|
|
def test_get_dataset_ids_env_not_found(mock_deps):
|
|
response = client.get("/api/datasets/ids?env_id=nonexistent")
|
|
assert response.status_code == 404
|
|
|
|
|
|
# #region test_get_dataset_ids_503 [C:2] [TYPE Function]
|
|
# @BRIEF Verify 503 when Superset fails.
|
|
def test_get_dataset_ids_503(mock_deps):
|
|
from src.dependencies import get_resource_service
|
|
failing = MagicMock()
|
|
failing.get_datasets_with_status = AsyncMock(
|
|
side_effect=Exception("Superset unavailable")
|
|
)
|
|
app.dependency_overrides[get_resource_service] = lambda: failing
|
|
try:
|
|
response = client.get("/api/datasets/ids?env_id=ss-dev")
|
|
assert response.status_code == 503
|
|
finally:
|
|
app.dependency_overrides[get_resource_service] = None
|
|
|
|
|
|
# #region test_get_datasets_invalid_page [C:2] [TYPE Function]
|
|
# @BRIEF Verify 400 when page < 1.
|
|
def test_get_datasets_invalid_page(mock_deps):
|
|
response = client.get("/api/datasets?env_id=ss-dev&page=0")
|
|
assert response.status_code == 400
|
|
|
|
|
|
# #region test_get_datasets_invalid_page_size [C:2] [TYPE Function]
|
|
# @BRIEF Verify 400 when page_size > 100.
|
|
def test_get_datasets_invalid_page_size(mock_deps):
|
|
response = client.get("/api/datasets?env_id=ss-dev&page_size=200")
|
|
assert response.status_code == 400
|
|
|
|
|
|
# #region test_get_datasets_env_not_found [C:2] [TYPE Function]
|
|
# @BRIEF Verify 404 when environment not found.
|
|
def test_get_datasets_env_not_found(mock_deps):
|
|
response = client.get("/api/datasets?env_id=nonexistent&page=1&page_size=10")
|
|
assert response.status_code == 404
|
|
|
|
|
|
# #region test_get_datasets_search_filter [C:2] [TYPE Function]
|
|
# @BRIEF Verify search filter narrows results.
|
|
def test_get_datasets_search_filter(mock_deps):
|
|
response = client.get("/api/datasets?env_id=ss-dev&search=users&page=1&page_size=10")
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["total"] == 1
|
|
assert data["datasets"][0]["table_name"] == "users"
|
|
|
|
|
|
# #region test_get_datasets_pagination [C:2] [TYPE Function]
|
|
# @BRIEF Verify pagination works.
|
|
def test_get_datasets_pagination(mock_deps):
|
|
response = client.get("/api/datasets?env_id=ss-dev&page=1&page_size=2")
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert len(data["datasets"]) == 2
|
|
assert data["total_pages"] == 2
|
|
|
|
|
|
# #region test_map_columns [C:2] [TYPE Function]
|
|
# @BRIEF Verify POST /api/datasets/map-columns creates mapping task.
|
|
def test_map_columns(mock_deps):
|
|
from src.dependencies import get_task_manager
|
|
task_manager_mock = MagicMock()
|
|
task_obj = MagicMock()
|
|
task_obj.id = "mapping-task-1"
|
|
task_manager_mock.create_task = AsyncMock(return_value=task_obj)
|
|
|
|
overrides = {
|
|
"plugin:mapper": MagicMock(),
|
|
}
|
|
|
|
app.dependency_overrides[get_task_manager] = lambda: task_manager_mock
|
|
|
|
from src.dependencies import has_permission
|
|
app.dependency_overrides[has_permission("plugin:mapper", "EXECUTE")] = lambda: True
|
|
|
|
try:
|
|
response = client.post(
|
|
"/api/datasets/map-columns",
|
|
json={
|
|
"env_id": "ss-dev",
|
|
"dataset_ids": [1, 2],
|
|
"source_type": "sqllab",
|
|
"database_id": 42,
|
|
},
|
|
)
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert "task_id" in data
|
|
assert data["task_id"] == "mapping-task-1"
|
|
finally:
|
|
app.dependency_overrides[get_task_manager] = None
|
|
app.dependency_overrides.pop(has_permission("plugin:mapper", "EXECUTE"), None)
|
|
|
|
|
|
# #region test_map_columns_no_dataset_ids [C:2] [TYPE Function]
|
|
# @BRIEF Verify 400 when no dataset IDs provided.
|
|
def test_map_columns_no_dataset_ids(mock_deps):
|
|
from src.dependencies import has_permission
|
|
app.dependency_overrides[has_permission("plugin:mapper", "EXECUTE")] = lambda: True
|
|
|
|
try:
|
|
response = client.post(
|
|
"/api/datasets/map-columns",
|
|
json={"env_id": "ss-dev", "dataset_ids": [], "source_type": "sqllab"},
|
|
)
|
|
assert response.status_code == 400
|
|
finally:
|
|
app.dependency_overrides.pop(has_permission("plugin:mapper", "EXECUTE"), None)
|
|
|
|
|
|
# #region test_map_columns_invalid_source_type [C:2] [TYPE Function]
|
|
# @BRIEF Verify 400 when invalid source type.
|
|
def test_map_columns_invalid_source_type(mock_deps):
|
|
from src.dependencies import has_permission
|
|
app.dependency_overrides[has_permission("plugin:mapper", "EXECUTE")] = lambda: True
|
|
|
|
try:
|
|
response = client.post(
|
|
"/api/datasets/map-columns",
|
|
json={"env_id": "ss-dev", "dataset_ids": [1], "source_type": "invalid"},
|
|
)
|
|
assert response.status_code == 400
|
|
finally:
|
|
app.dependency_overrides.pop(has_permission("plugin:mapper", "EXECUTE"), None)
|
|
|
|
|
|
# #region test_map_columns_missing_database_id [C:2] [TYPE Function]
|
|
# @BRIEF Verify 400 when sqllab source without database_id.
|
|
def test_map_columns_missing_database_id(mock_deps):
|
|
from src.dependencies import has_permission
|
|
app.dependency_overrides[has_permission("plugin:mapper", "EXECUTE")] = lambda: True
|
|
|
|
try:
|
|
response = client.post(
|
|
"/api/datasets/map-columns",
|
|
json={"env_id": "ss-dev", "dataset_ids": [1], "source_type": "sqllab"},
|
|
)
|
|
assert response.status_code == 400
|
|
finally:
|
|
app.dependency_overrides.pop(has_permission("plugin:mapper", "EXECUTE"), None)
|
|
|
|
|
|
# #region test_map_columns_env_not_found [C:2] [TYPE Function]
|
|
# @BRIEF Verify 404 when environment not found.
|
|
def test_map_columns_env_not_found(mock_deps):
|
|
from src.dependencies import has_permission
|
|
app.dependency_overrides[has_permission("plugin:mapper", "EXECUTE")] = lambda: True
|
|
|
|
try:
|
|
response = client.post(
|
|
"/api/datasets/map-columns",
|
|
json={
|
|
"env_id": "nonexistent",
|
|
"dataset_ids": [1],
|
|
"source_type": "sqllab",
|
|
"database_id": 42,
|
|
},
|
|
)
|
|
assert response.status_code == 404
|
|
finally:
|
|
app.dependency_overrides.pop(has_permission("plugin:mapper", "EXECUTE"), None)
|
|
|
|
|
|
# #region test_map_columns_xlsx_source [C:2] [TYPE Function]
|
|
# @BRIEF Verify xlsx source type works.
|
|
def test_map_columns_xlsx_source(mock_deps):
|
|
from src.dependencies import get_task_manager, has_permission
|
|
task_manager_mock = MagicMock()
|
|
task_obj = MagicMock()
|
|
task_obj.id = "mapping-task-2"
|
|
task_manager_mock.create_task = AsyncMock(return_value=task_obj)
|
|
|
|
app.dependency_overrides[get_task_manager] = lambda: task_manager_mock
|
|
app.dependency_overrides[has_permission("plugin:mapper", "EXECUTE")] = lambda: True
|
|
|
|
try:
|
|
response = client.post(
|
|
"/api/datasets/map-columns",
|
|
json={
|
|
"env_id": "ss-dev",
|
|
"dataset_ids": [1],
|
|
"source_type": "xlsx",
|
|
"file_data": "/tmp/test.xlsx",
|
|
},
|
|
)
|
|
assert response.status_code == 200
|
|
assert response.json()["task_id"] == "mapping-task-2"
|
|
finally:
|
|
app.dependency_overrides[get_task_manager] = None
|
|
app.dependency_overrides.pop(has_permission("plugin:mapper", "EXECUTE"), None)
|
|
|
|
|
|
# #region test_map_columns_503 [C:2] [TYPE Function]
|
|
# @BRIEF Verify 503 when task creation fails.
|
|
def test_map_columns_503(mock_deps):
|
|
from src.dependencies import get_task_manager, has_permission
|
|
task_manager_mock = MagicMock()
|
|
task_manager_mock.create_task = AsyncMock(side_effect=Exception("Task creation failed"))
|
|
|
|
app.dependency_overrides[get_task_manager] = lambda: task_manager_mock
|
|
app.dependency_overrides[has_permission("plugin:mapper", "EXECUTE")] = lambda: True
|
|
|
|
try:
|
|
response = client.post(
|
|
"/api/datasets/map-columns",
|
|
json={
|
|
"env_id": "ss-dev",
|
|
"dataset_ids": [1],
|
|
"source_type": "sqllab",
|
|
"database_id": 42,
|
|
},
|
|
)
|
|
assert response.status_code == 503
|
|
finally:
|
|
app.dependency_overrides[get_task_manager] = None
|
|
app.dependency_overrides.pop(has_permission("plugin:mapper", "EXECUTE"), None)
|
|
|
|
|
|
# #region test_generate_docs [C:2] [TYPE Function]
|
|
# @BRIEF Verify POST /api/datasets/generate-docs creates doc generation task.
|
|
def test_generate_docs(mock_deps):
|
|
from src.dependencies import get_task_manager, has_permission
|
|
task_manager_mock = MagicMock()
|
|
task_obj = MagicMock()
|
|
task_obj.id = "doc-task-1"
|
|
task_manager_mock.create_task = AsyncMock(return_value=task_obj)
|
|
|
|
app.dependency_overrides[get_task_manager] = lambda: task_manager_mock
|
|
app.dependency_overrides[has_permission("plugin:llm_analysis", "EXECUTE")] = lambda: True
|
|
|
|
try:
|
|
response = client.post(
|
|
"/api/datasets/generate-docs",
|
|
json={
|
|
"env_id": "ss-dev",
|
|
"dataset_ids": [1],
|
|
"llm_provider": "gpt-4",
|
|
},
|
|
)
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["task_id"] == "doc-task-1"
|
|
finally:
|
|
app.dependency_overrides[get_task_manager] = None
|
|
app.dependency_overrides.pop(has_permission("plugin:llm_analysis", "EXECUTE"), None)
|
|
|
|
|
|
# #region test_generate_docs_no_dataset_ids [C:2] [TYPE Function]
|
|
# @BRIEF Verify 400 when no dataset IDs provided.
|
|
def test_generate_docs_no_dataset_ids(mock_deps):
|
|
from src.dependencies import has_permission
|
|
app.dependency_overrides[has_permission("plugin:llm_analysis", "EXECUTE")] = lambda: True
|
|
|
|
try:
|
|
response = client.post(
|
|
"/api/datasets/generate-docs",
|
|
json={"env_id": "ss-dev", "dataset_ids": [], "llm_provider": "gpt-4"},
|
|
)
|
|
assert response.status_code == 400
|
|
finally:
|
|
app.dependency_overrides.pop(has_permission("plugin:llm_analysis", "EXECUTE"), None)
|
|
|
|
|
|
# #region test_generate_docs_env_not_found [C:2] [TYPE Function]
|
|
# @BRIEF Verify 404 when environment not found.
|
|
def test_generate_docs_env_not_found(mock_deps):
|
|
from src.dependencies import has_permission
|
|
app.dependency_overrides[has_permission("plugin:llm_analysis", "EXECUTE")] = lambda: True
|
|
|
|
try:
|
|
response = client.post(
|
|
"/api/datasets/generate-docs",
|
|
json={"env_id": "nonexistent", "dataset_ids": [1], "llm_provider": "gpt-4"},
|
|
)
|
|
assert response.status_code == 404
|
|
finally:
|
|
app.dependency_overrides.pop(has_permission("plugin:llm_analysis", "EXECUTE"), None)
|
|
|
|
|
|
# #region test_generate_docs_503 [C:2] [TYPE Function]
|
|
# @BRIEF Verify 503 when task creation fails.
|
|
def test_generate_docs_503(mock_deps):
|
|
from src.dependencies import get_task_manager, has_permission
|
|
task_manager_mock = MagicMock()
|
|
task_manager_mock.create_task = AsyncMock(side_effect=Exception("Doc gen failed"))
|
|
|
|
app.dependency_overrides[get_task_manager] = lambda: task_manager_mock
|
|
app.dependency_overrides[has_permission("plugin:llm_analysis", "EXECUTE")] = lambda: True
|
|
|
|
try:
|
|
response = client.post(
|
|
"/api/datasets/generate-docs",
|
|
json={"env_id": "ss-dev", "dataset_ids": [1], "llm_provider": "gpt-4"},
|
|
)
|
|
assert response.status_code == 503
|
|
finally:
|
|
app.dependency_overrides[get_task_manager] = None
|
|
app.dependency_overrides.pop(has_permission("plugin:llm_analysis", "EXECUTE"), None)
|
|
|
|
|
|
# #region test_update_metric_description_not_found [C:2] [TYPE Function]
|
|
# @BRIEF Verify 404 when metric not found.
|
|
def test_update_metric_description_not_found(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
|
|
mock_client.get_dataset.return_value = {
|
|
"id": 1,
|
|
"result": {
|
|
"id": 1,
|
|
"columns": [],
|
|
"metrics": [{"id": 100, "metric_name": "count", "description": ""}],
|
|
},
|
|
}
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/metrics/999/description?env_id=ss-dev",
|
|
json={"description": "test"},
|
|
)
|
|
assert response.status_code == 404
|
|
|
|
|
|
# #region test_update_metric_description_validation [C:2] [TYPE Function]
|
|
# @BRIEF Verify 400 when description exceeds max length for metric.
|
|
def test_update_metric_description_validation(mock_deps):
|
|
response = client.put(
|
|
"/api/datasets/1/metrics/100/description?env_id=ss-dev",
|
|
json={"description": "x" * 2001},
|
|
)
|
|
assert response.status_code == 400
|
|
|
|
|
|
# #region test_update_metric_description_superset_502_get [C:2] [TYPE Function]
|
|
# @BRIEF Verify 502 when Superset GET fails for metric.
|
|
def test_update_metric_description_superset_502_get(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
mock_client.get_dataset.side_effect = Exception("Superset GET failed")
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/metrics/100/description?env_id=ss-dev",
|
|
json={"description": "new desc"},
|
|
)
|
|
assert response.status_code == 502
|
|
|
|
|
|
# #region test_update_metric_description_superset_502_put [C:2] [TYPE Function]
|
|
# @BRIEF Verify 502 when Superset PUT fails for metric.
|
|
def test_update_metric_description_superset_502_put(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
mock_client.get_dataset.return_value = {
|
|
"id": 1,
|
|
"result": {
|
|
"id": 1,
|
|
"columns": [],
|
|
"metrics": [{"id": 100, "metric_name": "count", "description": ""}],
|
|
},
|
|
}
|
|
mock_client.update_dataset.side_effect = Exception("Superset PUT failed")
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/metrics/100/description?env_id=ss-dev",
|
|
json={"description": "new desc"},
|
|
)
|
|
assert response.status_code == 502
|
|
|
|
|
|
# #region test_update_column_description_env_not_found [C:2] [TYPE Function]
|
|
# @BRIEF Verify 404 when environment not found in update_column_description.
|
|
def test_update_column_description_env_not_found(mock_deps):
|
|
response = client.put(
|
|
"/api/datasets/1/columns/10/description?env_id=nonexistent",
|
|
json={"description": "test"},
|
|
)
|
|
assert response.status_code == 404
|
|
|
|
|
|
# #region test_update_column_description_get_fails [C:2] [TYPE Function]
|
|
# @BRIEF Verify 502 when Superset GET fails in update_column_description.
|
|
def test_update_column_description_get_fails(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
mock_client.get_dataset.side_effect = Exception("Superset GET error")
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/columns/10/description?env_id=ss-dev",
|
|
json={"description": "new desc"},
|
|
)
|
|
assert response.status_code == 502
|
|
|
|
|
|
# #region test_update_column_description_response_without_result_key [C:2] [TYPE Function]
|
|
# @BRIEF Cover line 562: elif branch when response has no "result" key.
|
|
def test_update_column_description_response_without_result_key(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
# Response has no "result" key
|
|
mock_client.get_dataset.return_value = {"id": 1, "columns": [{"id": 10, "column_name": "id", "type": "INT", "description": ""}]}
|
|
mock_client.update_dataset.return_value = {"id": 1}
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/columns/10/description?env_id=ss-dev",
|
|
json={"description": "test"},
|
|
)
|
|
assert response.status_code == 200
|
|
|
|
|
|
# #region test_update_column_description_dataset_not_found [C:2] [TYPE Function]
|
|
# @BRIEF Verify 404 when dataset has no id after GET.
|
|
def test_update_column_description_dataset_not_found(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
mock_client.get_dataset.return_value = {"result": {}}
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/columns/10/description?env_id=ss-dev",
|
|
json={"description": "test"},
|
|
)
|
|
assert response.status_code == 404
|
|
|
|
|
|
# #region test_update_column_description_unexpected_error [C:2] [TYPE Function]
|
|
# @BRIEF Verify 500 on unexpected error.
|
|
def test_update_column_description_unexpected_error(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
mock_client.get_dataset.side_effect = RuntimeError("Unexpected runtime error")
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/columns/10/description?env_id=ss-dev",
|
|
json={"description": "test"},
|
|
)
|
|
assert response.status_code == 502 # Caught by the inner try/except for GET
|
|
|
|
|
|
# #region test_update_metric_description_env_not_found [C:2] [TYPE Function]
|
|
# @BRIEF Verify 404 when env not found for metric update.
|
|
def test_update_metric_description_env_not_found(mock_deps):
|
|
response = client.put(
|
|
"/api/datasets/1/metrics/100/description?env_id=nonexistent",
|
|
json={"description": "test"},
|
|
)
|
|
assert response.status_code == 404
|
|
|
|
|
|
# #region test_update_metric_description_no_result_key [C:2] [TYPE Function]
|
|
# @BRIEF Cover line 650: elif branch when response has no "result" key for metric.
|
|
def test_update_metric_description_no_result_key(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
mock_client.get_dataset.return_value = {"id": 1, "columns": [], "metrics": [{"id": 100, "metric_name": "count", "description": ""}]}
|
|
mock_client.update_dataset.return_value = {"id": 1}
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/metrics/100/description?env_id=ss-dev",
|
|
json={"description": "new desc"},
|
|
)
|
|
assert response.status_code == 200
|
|
|
|
|
|
# #region test_update_metric_description_dataset_not_found [C:2] [TYPE Function]
|
|
# @BRIEF Verify 404 when dataset not found for metric update.
|
|
def test_update_metric_description_dataset_not_found(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
mock_client.get_dataset.return_value = {"result": {"columns": []}}
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/metrics/100/description?env_id=ss-dev",
|
|
json={"description": "test"},
|
|
)
|
|
assert response.status_code == 404
|
|
|
|
|
|
# #region test_update_metric_description_unexpected_error [C:2] [TYPE Function]
|
|
# @BRIEF Verify 500 on unexpected error for metric update.
|
|
def test_update_metric_description_unexpected_error(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
mock_client.get_dataset.side_effect = RuntimeError("Unexpected metric error")
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/metrics/100/description?env_id=ss-dev",
|
|
json={"description": "test"},
|
|
)
|
|
assert response.status_code == 502 # Caught by inner try/except for GET
|
|
|
|
|
|
# #region test_update_column_description_type_error [C:2] [TYPE Function]
|
|
# @BRIEF Verify 500 on TypeError in update_column_description (covers lines 597-599).
|
|
def test_update_column_description_type_error(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
# Return non-dict that passes inner try but breaks on .get()
|
|
mock_client.get_dataset.return_value = "not-a-dict"
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/columns/10/description?env_id=ss-dev",
|
|
json={"description": "test"},
|
|
)
|
|
assert response.status_code == 500
|
|
|
|
|
|
# #region test_update_metric_description_type_error [C:2] [TYPE Function]
|
|
# @BRIEF Verify 500 on TypeError in update_metric_description (covers lines 684-686).
|
|
def test_update_metric_description_type_error(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
mock_client.get_dataset.return_value = 42 # non-dict triggers TypeError
|
|
|
|
response = client.put(
|
|
"/api/datasets/1/metrics/100/description?env_id=ss-dev",
|
|
json={"description": "test"},
|
|
)
|
|
assert response.status_code == 500
|
|
|
|
|
|
# #region test_get_dataset_detail_database_object [C:2] [TYPE Function]
|
|
# @BRIEF Verify database field normalization when it's a dict.
|
|
def test_get_dataset_detail_database_object(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
|
|
mock_client.get_dataset_detail.return_value = {
|
|
"id": 1,
|
|
"table_name": "users",
|
|
"schema": "public",
|
|
"database": {"database_name": "PostgreSQL", "id": 5},
|
|
"columns": [],
|
|
"column_count": 0,
|
|
"metrics": [],
|
|
"metric_count": 0,
|
|
"sql": None,
|
|
"linked_dashboards": [],
|
|
"linked_dashboard_count": 0,
|
|
"is_sqllab_view": False,
|
|
}
|
|
|
|
response = client.get("/api/datasets/1?env_id=ss-dev")
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["database"] == "PostgreSQL"
|
|
|
|
|
|
# #region test_get_dataset_detail_503 [C:2] [TYPE Function]
|
|
# @BRIEF Verify 503 when Superset fails for dataset detail.
|
|
def test_get_dataset_detail_503(mock_deps):
|
|
with patch("src.api.routes.datasets.AsyncSupersetClient") as MockClient:
|
|
mock_client = AsyncMock()
|
|
MockClient.return_value = mock_client
|
|
mock_client.get_dataset_detail.side_effect = Exception("Superset error")
|
|
|
|
response = client.get("/api/datasets/1?env_id=ss-dev")
|
|
assert response.status_code == 503
|
|
|
|
|
|
# #region test_get_dataset_detail_env_not_found [C:2] [TYPE Function]
|
|
# @BRIEF Verify 404 when env not found for dataset detail.
|
|
def test_get_dataset_detail_env_not_found(mock_deps):
|
|
response = client.get("/api/datasets/1?env_id=nonexistent")
|
|
assert response.status_code == 404
|
|
|
|
|
|
# #region test_strip_html_tags_function [C:2] [TYPE Function]
|
|
# @BRIEF Test _strip_html_tags directly.
|
|
def test_strip_html_tags_empty():
|
|
from src.api.routes.datasets import _strip_html_tags
|
|
assert _strip_html_tags("") == ""
|
|
|
|
|
|
def test_strip_html_tags_plain_text():
|
|
from src.api.routes.datasets import _strip_html_tags
|
|
assert _strip_html_tags("plain text") == "plain text"
|
|
|
|
|
|
def test_strip_html_tags_with_html():
|
|
from src.api.routes.datasets import _strip_html_tags
|
|
result = _strip_html_tags("<b>bold</b> and <i>italic</i>")
|
|
assert result == "bold and italic"
|
|
# #endregion Test.DatasetsApi
|