test: 7 agents — plugins ~158 tests, extractor 98-100%, dashboard routes 99-100%, git services 94-100%, services 94-100%, dataset_review 100%. Fix test_api_key_routes.py sys.modules pollution
This commit is contained in:
280
backend/tests/test_core/test_async_network_extra.py
Normal file
280
backend/tests/test_core/test_async_network_extra.py
Normal file
@@ -0,0 +1,280 @@
|
||||
# #region Test.AsyncNetworkExtra [C:2] [TYPE Module] [SEMANTICS test, async, network, edge, upload]
|
||||
# @BRIEF Extra edge-case tests for async_network.py and network.py — auth CSRF refresh, expired cache, upload 201.
|
||||
# @RELATION BINDS_TO -> [AsyncNetworkModule]
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from src.core.utils.network import SupersetAuthCache
|
||||
|
||||
|
||||
# ── SupersetAuthCache: expired token path —─
|
||||
|
||||
|
||||
class TestSupersetAuthCacheExpired:
|
||||
"""SupersetAuthCache.get — expired token returns None (network.py lines 115-116)."""
|
||||
|
||||
def clear(self):
|
||||
SupersetAuthCache._entries.clear()
|
||||
|
||||
def test_expired_entry_returns_none(self):
|
||||
"""Entry with past expires_at returns None and is removed."""
|
||||
self.clear()
|
||||
key = ("exp_url", "exp_user", False)
|
||||
past_time = time.time() - 100
|
||||
with SupersetAuthCache._lock:
|
||||
SupersetAuthCache._entries[key] = {
|
||||
"tokens": {"access_token": "old", "csrf_token": "old_csrf"},
|
||||
"expires_at": past_time,
|
||||
}
|
||||
result = SupersetAuthCache.get(key)
|
||||
assert result is None
|
||||
|
||||
def test_expired_after_wait_returns_none(self):
|
||||
"""Entry that expired between check and get returns None."""
|
||||
self.clear()
|
||||
key = ("exp2_url", "exp2_user", False)
|
||||
past_time = time.time() - 10
|
||||
with SupersetAuthCache._lock:
|
||||
SupersetAuthCache._entries[key] = {
|
||||
"tokens": {"access_token": "old", "csrf_token": "old_csrf"},
|
||||
"expires_at": past_time,
|
||||
}
|
||||
result = SupersetAuthCache.get(key)
|
||||
assert result is None
|
||||
# Entry should be removed
|
||||
assert key not in SupersetAuthCache._entries
|
||||
|
||||
|
||||
# ── AsyncAPIClient.authenticate: cache-hit-after-wait CSRF refresh ──
|
||||
|
||||
|
||||
class TestAuthenticateCacheHitAfterWaitCSRF:
|
||||
"""authenticate — cache-hit-after-wait CSRF refresh paths (lines 161-177).
|
||||
|
||||
To hit the lock path, we need:
|
||||
- First cache check (line 153) → miss (no cache)
|
||||
- Second cache check (line 159, inside lock) → hit
|
||||
|
||||
We achieve this by patching SupersetAuthCache.get to return different
|
||||
values on successive calls.
|
||||
"""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_cache(self):
|
||||
SupersetAuthCache._entries.clear()
|
||||
yield
|
||||
|
||||
def _make_client(self):
|
||||
from src.core.utils.async_network import AsyncAPIClient
|
||||
|
||||
c = AsyncAPIClient(
|
||||
{
|
||||
"base_url": "https://superset.example.com",
|
||||
"auth": {"username": "admin", "password": "pass"},
|
||||
}
|
||||
)
|
||||
c._client = AsyncMock()
|
||||
return c
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_csrf_refresh_success_path(self):
|
||||
"""Cache hit after wait: CSRF refresh succeeds and updates token."""
|
||||
client = self._make_client()
|
||||
mock_csrf_resp = MagicMock()
|
||||
mock_csrf_resp.is_success = True
|
||||
mock_csrf_resp.json.return_value = {"result": "new_csrf_token"}
|
||||
client._client.get = AsyncMock(return_value=mock_csrf_resp)
|
||||
|
||||
call_count = 0
|
||||
_tokens_cache = {"access_token": "cached_at", "csrf_token": "refresh_me"}
|
||||
|
||||
def _fake_get(key):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
return None # first check — miss
|
||||
return _tokens_cache # second check — hit
|
||||
|
||||
with patch.object(SupersetAuthCache, "get", side_effect=_fake_get):
|
||||
tokens = await client.authenticate()
|
||||
assert tokens["access_token"] == "cached_at"
|
||||
assert tokens["csrf_token"] == "new_csrf_token"
|
||||
assert client._authenticated is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_csrf_refresh_exception_swallowed(self):
|
||||
"""Cache hit after wait: CSRF refresh exception is swallowed (lines 175-176)."""
|
||||
client = self._make_client()
|
||||
mock_csrf_resp = MagicMock()
|
||||
mock_csrf_resp.is_success = True
|
||||
mock_csrf_resp.json.side_effect = ValueError("Bad JSON")
|
||||
client._client.get = AsyncMock(return_value=mock_csrf_resp)
|
||||
|
||||
call_count = 0
|
||||
_tokens_cache = {"access_token": "cached_at", "csrf_token": "old_csrf"}
|
||||
|
||||
def _fake_get(key):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
return None
|
||||
return _tokens_cache
|
||||
|
||||
with patch.object(SupersetAuthCache, "get", side_effect=_fake_get):
|
||||
tokens = await client.authenticate()
|
||||
assert tokens["access_token"] == "cached_at"
|
||||
# CSRF token stays as cached (silently failed)
|
||||
assert tokens["csrf_token"] == "old_csrf"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_csrf_refresh_connection_error_swallowed(self):
|
||||
"""Cache hit after wait: CSRF connection error is swallowed."""
|
||||
client = self._make_client()
|
||||
client._client.get = AsyncMock(side_effect=httpx.ConnectError("No route"))
|
||||
|
||||
call_count = 0
|
||||
_tokens_cache = {"access_token": "cached_at", "csrf_token": "old_csrf"}
|
||||
|
||||
def _fake_get(key):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
return None
|
||||
return _tokens_cache
|
||||
|
||||
with patch.object(SupersetAuthCache, "get", side_effect=_fake_get):
|
||||
tokens = await client.authenticate()
|
||||
assert tokens["access_token"] == "cached_at"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cache_hit_stale_csrf_refreshed(self):
|
||||
"""Cache hit after wait: CSRF token gets refreshed."""
|
||||
client = self._make_client()
|
||||
mock_csrf_resp = MagicMock()
|
||||
mock_csrf_resp.is_success = True
|
||||
mock_csrf_resp.json.return_value = {"result": "new_csrf"}
|
||||
client._client.get = AsyncMock(return_value=mock_csrf_resp)
|
||||
|
||||
call_count = 0
|
||||
_tokens_cache = {"access_token": "cached_at", "csrf_token": "stale_csrf"}
|
||||
|
||||
def _fake_get(key):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
return None
|
||||
return _tokens_cache
|
||||
|
||||
with patch.object(SupersetAuthCache, "get", side_effect=_fake_get):
|
||||
tokens = await client.authenticate()
|
||||
assert tokens["access_token"] == "cached_at"
|
||||
assert tokens["csrf_token"] == "new_csrf"
|
||||
|
||||
|
||||
# ── AsyncAPIClient.upload_file: non-200 success status ──
|
||||
|
||||
|
||||
class TestUploadFileNon200:
|
||||
"""upload_file — response with non-200 success status like 201 (line 494)."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_201_created(self):
|
||||
from src.core.utils.async_network import AsyncAPIClient
|
||||
|
||||
client = AsyncAPIClient(
|
||||
{
|
||||
"base_url": "https://test.com",
|
||||
"auth": {"username": "admin", "password": "pass"},
|
||||
}
|
||||
)
|
||||
client._authenticated = True
|
||||
client._tokens = {"access_token": "acc", "csrf_token": "csrf"}
|
||||
client._client = AsyncMock()
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 201 # Created — NOT 200
|
||||
mock_resp.json.return_value = {"result": "created"}
|
||||
client._client.post = AsyncMock(return_value=mock_resp)
|
||||
|
||||
import io
|
||||
|
||||
buf = io.BytesIO(b"zip content")
|
||||
result = await client.upload_file(
|
||||
endpoint="/dashboard/import/",
|
||||
file_info={"file_obj": buf, "file_name": "test.zip"},
|
||||
)
|
||||
assert result == {"result": "created"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_204_no_content(self):
|
||||
"""204 response with no body — raise_for_status passes, but json() may fail."""
|
||||
from src.core.utils.async_network import AsyncAPIClient
|
||||
from src.core.utils.network import SupersetAPIError
|
||||
|
||||
client = AsyncAPIClient(
|
||||
{
|
||||
"base_url": "https://test.com",
|
||||
"auth": {"username": "admin", "password": "pass"},
|
||||
}
|
||||
)
|
||||
client._authenticated = True
|
||||
client._tokens = {"access_token": "acc", "csrf_token": "csrf"}
|
||||
client._client = AsyncMock()
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
# Empty text means the first response.json() inside try succeeds
|
||||
# We want to test the case where it fails
|
||||
mock_resp.json.side_effect = ValueError("No JSON")
|
||||
client._client.post = AsyncMock(return_value=mock_resp)
|
||||
|
||||
import io
|
||||
|
||||
buf = io.BytesIO(b"zip content")
|
||||
with pytest.raises(SupersetAPIError, match="API error during upload"):
|
||||
await client.upload_file(
|
||||
endpoint="/dashboard/import/",
|
||||
file_info={"file_obj": buf, "file_name": "test.zip"},
|
||||
)
|
||||
|
||||
|
||||
# ── AsyncAPIClient._build_api_url with strip ──
|
||||
|
||||
|
||||
class TestBuildApiUrlEdge:
|
||||
"""_build_api_url — edge cases."""
|
||||
|
||||
@pytest.fixture
|
||||
def client(self):
|
||||
from src.core.utils.async_network import AsyncAPIClient
|
||||
|
||||
return AsyncAPIClient(
|
||||
{"base_url": "https://test.com", "auth": {}}
|
||||
)
|
||||
|
||||
def test_endpoint_with_query(self, client):
|
||||
"""Endpoint with query params."""
|
||||
url = client._build_api_url("/chart/?page=1")
|
||||
assert "page=1" in url
|
||||
|
||||
def test_endpoint_with_hash(self, client):
|
||||
"""Endpoint with hash fragment."""
|
||||
url = client._build_api_url("/chart/#section")
|
||||
assert "#section" in url
|
||||
|
||||
def test_api_v1_with_query_does_not_dedup(self, client):
|
||||
"""Endpoint with api/v1 and query doesn't double-add api/v1."""
|
||||
url = client._build_api_url("/api/v1/chart/?page=1")
|
||||
assert url.count("api/v1") == 1
|
||||
# #endregion Test.AsyncNetworkExtra
|
||||
137
backend/tests/test_core/test_dataset_mapper_extra.py
Normal file
137
backend/tests/test_core/test_dataset_mapper_extra.py
Normal file
@@ -0,0 +1,137 @@
|
||||
# #region Test.DatasetMapperExtra [C:2] [TYPE Module] [SEMANTICS test, dataset, mapper, metrics, edge]
|
||||
# @BRIEF Extra tests for dataset_mapper.py — metrics handling in run_mapping (lines 193-206).
|
||||
# @RELATION BINDS_TO -> [DatasetMapperModule]
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestDatasetMapperMetrics:
|
||||
"""DatasetMapper.run_mapping — metrics transformation."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_mapping_with_metrics(self):
|
||||
"""run_mapping with dataset that has metrics updates them."""
|
||||
from src.core.utils.dataset_mapper import DatasetMapper
|
||||
|
||||
mapper = DatasetMapper()
|
||||
mapper.get_sqllab_mappings = AsyncMock(
|
||||
return_value={"col1": "Column 1"}
|
||||
)
|
||||
client = AsyncMock()
|
||||
client.get_dataset = AsyncMock(
|
||||
return_value={
|
||||
"result": {
|
||||
"table_name": "users",
|
||||
"columns": [{"column_name": "col1", "verbose_name": None}],
|
||||
"metrics": [
|
||||
{"id": 1, "metric_name": "count", "expression": "COUNT(*)", "verbose_name": "Count"},
|
||||
{"id": 2, "metric_name": "sum", "expression": "SUM(amount)", "verbose_name": None},
|
||||
],
|
||||
"owners": [],
|
||||
"database": {"id": 1},
|
||||
}
|
||||
}
|
||||
)
|
||||
client.update_dataset = AsyncMock(return_value={"result": "ok"})
|
||||
|
||||
await mapper.run_mapping(
|
||||
superset_client=client,
|
||||
dataset_id=1,
|
||||
source="sqllab",
|
||||
sqllab_executor=MagicMock(),
|
||||
database_id=42,
|
||||
)
|
||||
client.update_dataset.assert_awaited_once()
|
||||
call_args = client.update_dataset.call_args
|
||||
payload = call_args[0][1]
|
||||
# Should have metrics in the payload
|
||||
assert "metrics" in payload
|
||||
# Metrics with None verbose_name should have the key stripped
|
||||
metric_names = [m["metric_name"] for m in payload["metrics"]]
|
||||
assert "count" in metric_names
|
||||
assert "sum" in metric_names
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_mapping_skips_none_metric_fields(self):
|
||||
"""Metrics with None fields have those keys stripped."""
|
||||
from src.core.utils.dataset_mapper import DatasetMapper
|
||||
|
||||
mapper = DatasetMapper()
|
||||
mapper.get_sqllab_mappings = AsyncMock(
|
||||
return_value={"col1": "Column 1"}
|
||||
)
|
||||
client = AsyncMock()
|
||||
client.get_dataset = AsyncMock(
|
||||
return_value={
|
||||
"result": {
|
||||
"table_name": "users",
|
||||
"columns": [{"column_name": "col1", "verbose_name": None}],
|
||||
"metrics": [
|
||||
{"id": 1, "metric_name": "cnt", "expression": "COUNT(*)", "verbose_name": None, "description": None},
|
||||
],
|
||||
"owners": [],
|
||||
"database": {"id": 1},
|
||||
}
|
||||
}
|
||||
)
|
||||
client.update_dataset = AsyncMock(return_value={"result": "ok"})
|
||||
|
||||
await mapper.run_mapping(
|
||||
superset_client=client,
|
||||
dataset_id=1,
|
||||
source="sqllab",
|
||||
sqllab_executor=MagicMock(),
|
||||
database_id=42,
|
||||
)
|
||||
call_args = client.update_dataset.call_args
|
||||
payload = call_args[0][1]
|
||||
metric = payload["metrics"][0]
|
||||
# None fields like `verbose_name` and `description` should be stripped
|
||||
assert "verbose_name" not in metric
|
||||
assert "description" not in metric
|
||||
assert metric["metric_name"] == "cnt"
|
||||
|
||||
|
||||
class TestDatasetMapperNetworkEdge:
|
||||
"""DatasetMapper edge cases — async_network integration."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_mapping_with_async_client(self):
|
||||
"""run_mapping with AsyncAPIClient (has async get_dataset/update_dataset)."""
|
||||
from src.core.utils.dataset_mapper import DatasetMapper
|
||||
|
||||
mapper = DatasetMapper()
|
||||
mapper.get_sqllab_mappings = AsyncMock(
|
||||
return_value={"col1": "Column 1"}
|
||||
)
|
||||
client = AsyncMock()
|
||||
client.get_dataset = AsyncMock(
|
||||
return_value={
|
||||
"result": {
|
||||
"table_name": "users",
|
||||
"columns": [{"column_name": "col1", "verbose_name": None}],
|
||||
"metrics": [],
|
||||
"owners": [],
|
||||
"database": {"id": 1},
|
||||
}
|
||||
}
|
||||
)
|
||||
client.update_dataset = AsyncMock(return_value={"result": "ok"})
|
||||
|
||||
await mapper.run_mapping(
|
||||
superset_client=client,
|
||||
dataset_id=1,
|
||||
source="sqllab",
|
||||
sqllab_executor=MagicMock(),
|
||||
database_id=42,
|
||||
)
|
||||
client.update_dataset.assert_awaited_once()
|
||||
# #endregion Test.DatasetMapperExtra
|
||||
@@ -0,0 +1,224 @@
|
||||
# #region Test.SupersetContextFiltersExtra [C:2] [TYPE Module] [SEMANTICS test, superset, filter, edge]
|
||||
# @BRIEF Extra tests for _filters.py — edge cases that hit lines 76, 107, 144, 167, 173-178, 182-193.
|
||||
# @RELATION BINDS_TO -> [SupersetContextFiltersExtractMixin]
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
|
||||
|
||||
import os
|
||||
|
||||
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-testing")
|
||||
os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:")
|
||||
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:")
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.utils.superset_context_extractor._filters import (
|
||||
SupersetContextFiltersExtractMixin,
|
||||
)
|
||||
|
||||
|
||||
class _TestExtractor(SupersetContextFiltersExtractMixin):
|
||||
"""Concrete test class for the mixin."""
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def extractor():
|
||||
return _TestExtractor()
|
||||
|
||||
|
||||
class TestExtractImportedFiltersExtra:
|
||||
"""Extra edge cases for _extract_imported_filters."""
|
||||
|
||||
def test_dataMask_non_dict_item_skipped(self, extractor):
|
||||
"""dataMask non-dict item is skipped (line 76)."""
|
||||
result = extractor._extract_imported_filters({
|
||||
"dataMask": {
|
||||
"NATIVE-1": "not_a_dict",
|
||||
"NATIVE-2": {"filterState": {"label": "Country", "value": "US"}},
|
||||
}
|
||||
})
|
||||
assert len(result) == 1
|
||||
assert result[0]["raw_value"] == "US"
|
||||
|
||||
def test_dataMask_filter_state_values_fallback(self, extractor):
|
||||
"""filterState 'values' used when 'value' is None (line 107)."""
|
||||
result = extractor._extract_imported_filters({
|
||||
"dataMask": {
|
||||
"NATIVE-1": {
|
||||
"filterState": {"label": "Country", "values": ["US", "CA"]},
|
||||
"extraFormData": {},
|
||||
}
|
||||
}
|
||||
})
|
||||
assert result[0]["raw_value"] == ["US", "CA"]
|
||||
|
||||
def test_native_filter_state_non_dict_skipped(self, extractor):
|
||||
"""native_filter_state non-dict item skipped (line 144)."""
|
||||
result = extractor._extract_imported_filters({
|
||||
"native_filter_state": {
|
||||
"NATIVE-1": "not_a_dict",
|
||||
"NATIVE-2": {"filterState": {"value": "US"}, "extraFormData": {}},
|
||||
}
|
||||
})
|
||||
assert len(result) == 1
|
||||
assert result[0]["raw_value"] == "US"
|
||||
|
||||
def test_native_filter_state_extra_form_data_filters(self, extractor):
|
||||
"""native_filter_state extraFormData.filters extracts clauses (line 167)."""
|
||||
result = extractor._extract_imported_filters({
|
||||
"native_filter_state": {
|
||||
"NATIVE-1": {
|
||||
"extraFormData": {
|
||||
"filters": [{"col": "country", "op": "IN", "val": "US"}]
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
assert len(result) >= 1
|
||||
clauses = result[0]["normalized_value"]["filter_clauses"]
|
||||
assert len(clauses) == 1
|
||||
assert clauses[0]["col"] == "country"
|
||||
|
||||
def test_native_filter_state_raw_value_from_clauses(self, extractor):
|
||||
"""native_filter_state: raw_value derived from filter_clauses (lines 173-178)."""
|
||||
result = extractor._extract_imported_filters({
|
||||
"native_filter_state": {
|
||||
"NATIVE-1": {
|
||||
"extraFormData": {
|
||||
"filters": [{"col": "country", "op": "IN", "val": "US"}]
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
assert result[0]["raw_value"] == "US"
|
||||
assert result[0]["recovery_status"] == "recovered"
|
||||
|
||||
def test_native_filter_state_raw_value_from_clauses_value_key(self, extractor):
|
||||
"""native_filter_state: raw_value from 'value' key when 'val' absent."""
|
||||
result = extractor._extract_imported_filters({
|
||||
"native_filter_state": {
|
||||
"NATIVE-1": {
|
||||
"extraFormData": {
|
||||
"filters": [{"col": "country", "op": "IN", "value": "Canada"}]
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
assert result[0]["raw_value"] == "Canada"
|
||||
|
||||
def test_native_filter_state_time_fields(self, extractor):
|
||||
"""native_filter_state: time fields provide raw_value (lines 182-193)."""
|
||||
result = extractor._extract_imported_filters({
|
||||
"native_filter_state": {
|
||||
"NATIVE-1": {
|
||||
"extraFormData": {"time_range": "Last 7 days"}
|
||||
}
|
||||
}
|
||||
})
|
||||
assert result[0]["raw_value"] == "Last 7 days"
|
||||
assert result[0]["recovery_status"] == "recovered"
|
||||
|
||||
def test_native_filter_state_time_grain_field(self, extractor):
|
||||
"""native_filter_state: time_grain_sqla field."""
|
||||
result = extractor._extract_imported_filters({
|
||||
"native_filter_state": {
|
||||
"NATIVE-1": {
|
||||
"extraFormData": {"time_grain_sqla": "P1D"}
|
||||
}
|
||||
}
|
||||
})
|
||||
assert result[0]["raw_value"] == "P1D"
|
||||
|
||||
def test_native_filter_state_time_column_field(self, extractor):
|
||||
"""native_filter_state: time_column field."""
|
||||
result = extractor._extract_imported_filters({
|
||||
"native_filter_state": {
|
||||
"NATIVE-1": {
|
||||
"extraFormData": {"time_column": "created_at"}
|
||||
}
|
||||
}
|
||||
})
|
||||
assert result[0]["raw_value"] == "created_at"
|
||||
|
||||
def test_native_filter_state_granularity_field(self, extractor):
|
||||
"""native_filter_state: granularity field."""
|
||||
result = extractor._extract_imported_filters({
|
||||
"native_filter_state": {
|
||||
"NATIVE-1": {
|
||||
"extraFormData": {"granularity": "day"}
|
||||
}
|
||||
}
|
||||
})
|
||||
assert result[0]["raw_value"] == "day"
|
||||
|
||||
def test_dataMask_raw_value_from_clauses_value_key(self, extractor):
|
||||
"""dataMask: raw_value from 'value' key when 'val' absent."""
|
||||
result = extractor._extract_imported_filters({
|
||||
"dataMask": {
|
||||
"NATIVE-1": {
|
||||
"extraFormData": {
|
||||
"filters": [{"col": "country", "op": "IN", "value": "UK"}]
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
assert result[0]["raw_value"] == "UK"
|
||||
assert result[0]["recovery_status"] == "recovered"
|
||||
|
||||
def test_dataMask_time_extra_form_data_time_grain(self, extractor):
|
||||
"""dataMask: time_grain_sqla extraFormData field."""
|
||||
result = extractor._extract_imported_filters({
|
||||
"dataMask": {
|
||||
"NATIVE-1": {
|
||||
"extraFormData": {"time_grain_sqla": "P1M"}
|
||||
}
|
||||
}
|
||||
})
|
||||
assert result[0]["raw_value"] == "P1M"
|
||||
|
||||
def test_dataMask_extra_form_data_has_filters_without_val(self, extractor):
|
||||
"""dataMask: extraFormData filters without val/value yields partial."""
|
||||
result = extractor._extract_imported_filters({
|
||||
"dataMask": {
|
||||
"NATIVE-1": {
|
||||
"extraFormData": {
|
||||
"filters": [{"col": "country"}]
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
# Should still get a filter entry, raw_value is None from filter clauses
|
||||
# and no time fields present
|
||||
assert result[0]["raw_value"] is None
|
||||
assert result[0]["recovery_status"] == "partial"
|
||||
|
||||
def test_native_filter_state_filter_id_fallback(self, extractor):
|
||||
"""native_filter_state: filter uses item id or falls back to key."""
|
||||
result = extractor._extract_imported_filters({
|
||||
"native_filter_state": {
|
||||
"NATIVE-KEY": {
|
||||
"filterState": {"value": "test_val"},
|
||||
"extraFormData": {},
|
||||
}
|
||||
}
|
||||
})
|
||||
# No 'id' in item, so falls back to "NATIVE-KEY"
|
||||
assert result[0]["filter_name"] == "NATIVE-KEY"
|
||||
|
||||
result2 = extractor._extract_imported_filters({
|
||||
"native_filter_state": {
|
||||
"NATIVE-KEY": {
|
||||
"id": "OVERRIDE-ID",
|
||||
"filterState": {"value": "test_val"},
|
||||
"extraFormData": {},
|
||||
}
|
||||
}
|
||||
})
|
||||
assert result2[0]["filter_name"] == "OVERRIDE-ID"
|
||||
# #endregion Test.SupersetContextFiltersExtra
|
||||
@@ -0,0 +1,507 @@
|
||||
# #region Test.SupersetContextParsing [C:3] [TYPE Module] [SEMANTICS test, superset, context, parse, link]
|
||||
# @BRIEF Tests for _parsing.py — SupersetContextParsingMixin.parse_superset_link full branch coverage.
|
||||
# @RELATION BINDS_TO -> [SupersetContextParsingMixin]
|
||||
# @TEST_CONTRACT: [link:str] -> [SupersetParsedContext]
|
||||
# @TEST_SCENARIO: dataset_link -> resolves dataset_id and dataset_ref
|
||||
# @TEST_SCENARIO: dashboard_permalink -> resolves via get_dashboard_permalink_state
|
||||
# @TEST_SCENARIO: dashboard_native_filters -> resolves via get_dashboard_detail + native_filters_key
|
||||
# @TEST_SCENARIO: chart_link -> partial recovery
|
||||
# @TEST_SCENARIO: empty_link -> ValueError
|
||||
# @TEST_EDGE: missing_dataset_detail -> partial_recovery set True
|
||||
# @TEST_EDGE: permalink_non_dict_state -> logged, no crash
|
||||
# @TEST_EDGE: unsupported_link_shape -> ValueError
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
|
||||
|
||||
import os
|
||||
|
||||
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-testing")
|
||||
os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:")
|
||||
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:")
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.utils.superset_context_extractor._base import (
|
||||
SupersetContextExtractorBase,
|
||||
SupersetParsedContext,
|
||||
)
|
||||
from src.core.utils.superset_context_extractor._filters import (
|
||||
SupersetContextFiltersExtractMixin,
|
||||
)
|
||||
from src.core.utils.superset_context_extractor._parsing import (
|
||||
SupersetContextParsingMixin,
|
||||
)
|
||||
|
||||
|
||||
class _TestExtractor(
|
||||
SupersetContextFiltersExtractMixin,
|
||||
SupersetContextParsingMixin,
|
||||
SupersetContextExtractorBase,
|
||||
):
|
||||
"""Concrete test class matching the full extractor MRO for test isolation."""
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_env():
|
||||
"""Return env mock."""
|
||||
env = MagicMock()
|
||||
env.id = "test-env"
|
||||
env.url = "https://superset.example.com"
|
||||
return env
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def extractor(mock_env):
|
||||
"""Return _TestExtractor with client where:
|
||||
- Methods called WITH await use AsyncMock
|
||||
- Methods called WITHOUT await use MagicMock
|
||||
"""
|
||||
client = MagicMock()
|
||||
# AWAITED in source
|
||||
client.get_dashboard_detail = AsyncMock()
|
||||
client.get_chart = AsyncMock()
|
||||
# SYNC calls — MagicMock with sensible return values
|
||||
client.get_dashboard_permalink_state = MagicMock()
|
||||
client.extract_native_filters_from_key = MagicMock()
|
||||
# get_dataset_detail runs AFTER dataset_id is resolved — always set a default
|
||||
client.get_dataset_detail = MagicMock(
|
||||
return_value={"table_name": "users", "schema": "public"}
|
||||
)
|
||||
return _TestExtractor(environment=mock_env, client=client)
|
||||
|
||||
|
||||
def _set_dataset_detail(extractor, return_value=None):
|
||||
"""Helper: set sync get_dataset_detail mock."""
|
||||
if return_value is None:
|
||||
return_value = {"table_name": "users", "schema": "public"}
|
||||
extractor.client.get_dataset_detail = MagicMock(return_value=return_value)
|
||||
|
||||
|
||||
def _set_dashboard_detail(extractor, return_value=None):
|
||||
"""Helper: set async get_dashboard_detail mock."""
|
||||
if return_value is None:
|
||||
return_value = {"id": 5, "datasets": [{"id": 10}]}
|
||||
extractor.client.get_dashboard_detail = AsyncMock(return_value=return_value)
|
||||
|
||||
|
||||
def _set_permalink_state(extractor, return_value=None):
|
||||
"""Helper: set sync get_dashboard_permalink_state mock."""
|
||||
if return_value is None:
|
||||
return_value = {"state": {"dashboardId": 42}}
|
||||
extractor.client.get_dashboard_permalink_state = MagicMock(
|
||||
return_value=return_value
|
||||
)
|
||||
|
||||
|
||||
def _set_chart(extractor, return_value=None):
|
||||
"""Helper: set async get_chart mock."""
|
||||
if return_value is None:
|
||||
return_value = {"result": {"datasource_id": 15}}
|
||||
extractor.client.get_chart = AsyncMock(return_value=return_value)
|
||||
|
||||
|
||||
def _set_native_filters_key(extractor, return_value=None):
|
||||
"""Helper: set sync extract_native_filters_from_key mock."""
|
||||
if return_value is None:
|
||||
return_value = {"dataMask": {"NATIVE-1": {"filterState": {"value": "US"}}}}
|
||||
extractor.client.extract_native_filters_from_key = MagicMock(
|
||||
return_value=return_value
|
||||
)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Input validation
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestParseSupersetLinkValidation:
|
||||
"""parse_superset_link input guards."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_link_raises_value_error(self, extractor):
|
||||
"""Empty link string raises ValueError."""
|
||||
with pytest.raises(ValueError, match="must be non-empty"):
|
||||
await extractor.parse_superset_link("")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_none_link_raises_value_error(self, extractor):
|
||||
"""None link raises ValueError."""
|
||||
with pytest.raises(ValueError, match="must be non-empty"):
|
||||
await extractor.parse_superset_link(None)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_whitespace_link_raises_value_error(self, extractor):
|
||||
"""Whitespace-only link raises ValueError."""
|
||||
with pytest.raises(ValueError, match="must be non-empty"):
|
||||
await extractor.parse_superset_link(" ")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_http_scheme_raises_value_error(self, extractor):
|
||||
"""Non-http(s) scheme raises ValueError."""
|
||||
with pytest.raises(ValueError, match="absolute http"):
|
||||
await extractor.parse_superset_link("ftp://superset.example.com/dataset/5")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_netloc_raises_value_error(self, extractor):
|
||||
"""URL without netloc raises ValueError."""
|
||||
with pytest.raises(ValueError, match="absolute http"):
|
||||
await extractor.parse_superset_link("http:///dataset/5")
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Direct dataset link
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestParseSupersetLinkDataset:
|
||||
"""parse_superset_link — direct /dataset/N/ path."""
|
||||
|
||||
DATASET_URL = "https://superset.example.com/superset/dataset/5/"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_dataset_link(self, extractor):
|
||||
"""dataset/5/ resolves dataset_id=5 and resource_type='dataset'."""
|
||||
_set_dataset_detail(extractor)
|
||||
result = await extractor.parse_superset_link(self.DATASET_URL)
|
||||
assert result.dataset_id == 5
|
||||
assert result.resource_type == "dataset"
|
||||
assert result.dataset_ref == "public.users"
|
||||
assert result.chart_id is None
|
||||
assert result.dashboard_id is None
|
||||
assert result.partial_recovery is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dataset_link_without_schema(self, extractor):
|
||||
"""dataset detail without schema uses just table_name as ref."""
|
||||
_set_dataset_detail(extractor, {"table_name": "users", "schema": ""})
|
||||
result = await extractor.parse_superset_link(self.DATASET_URL)
|
||||
assert result.dataset_ref == "users"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dataset_detail_fetch_fails_partial_recovery(self, extractor):
|
||||
"""Dataset detail fetch failure sets partial_recovery=True."""
|
||||
extractor.client.get_dataset_detail = MagicMock(
|
||||
side_effect=Exception("Superset API timeout")
|
||||
)
|
||||
result = await extractor.parse_superset_link(self.DATASET_URL)
|
||||
assert result.dataset_id == 5
|
||||
assert result.partial_recovery is True
|
||||
assert "dataset_detail_lookup_failed" in result.unresolved_references
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Dashboard permalink
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestParseSupersetLinkDashboardPermalink:
|
||||
"""parse_superset_link — /dashboard/p/<key>/ path"""
|
||||
|
||||
PERMALINK_URL = "https://superset.example.com/superset/dashboard/p/abc123/"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_permalink_resolves_dashboard_with_data_mask(self, extractor):
|
||||
"""Permalink state with dataMask + dashboardId resolves fully."""
|
||||
_set_permalink_state(extractor, {
|
||||
"state": {
|
||||
"dashboardId": 42,
|
||||
"dataMask": {"NATIVE-1": {"filterState": {"value": "US"}}},
|
||||
}
|
||||
})
|
||||
_set_dashboard_detail(extractor)
|
||||
result = await extractor.parse_superset_link(self.PERMALINK_URL)
|
||||
assert result.dashboard_id == 42
|
||||
assert result.dataset_id == 10
|
||||
assert result.resource_type == "dashboard"
|
||||
assert "dataMask" in result.query_state
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_permalink_resolves_chart_recovery(self, extractor):
|
||||
"""Permalink without dashboardId but with sliceId recovers dataset from chart."""
|
||||
_set_permalink_state(extractor, {
|
||||
"state": {
|
||||
"slice_id": 77,
|
||||
"dataMask": {},
|
||||
}
|
||||
})
|
||||
_set_chart(extractor)
|
||||
result = await extractor.parse_superset_link(self.PERMALINK_URL)
|
||||
assert result.chart_id == 77
|
||||
assert result.dataset_id == 15
|
||||
# dataset_ref gets canonicalized by get_dataset_detail result
|
||||
assert result.dataset_ref == "public.users"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_permalink_chart_missing_datasource_id(self, extractor):
|
||||
"""Chart without datasource_id adds unresolvable marker."""
|
||||
_set_permalink_state(extractor, {
|
||||
"state": {
|
||||
"slice_id": 77,
|
||||
"dataMask": {},
|
||||
}
|
||||
})
|
||||
_set_chart(extractor, {"result": {}})
|
||||
result = await extractor.parse_superset_link(self.PERMALINK_URL)
|
||||
assert result.dataset_id is None
|
||||
assert "chart_dataset_binding_unresolved" in result.unresolved_references
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_permalink_chart_fetch_raises(self, extractor):
|
||||
"""Chart fetch failure adds unresolvable marker."""
|
||||
_set_permalink_state(extractor, {
|
||||
"state": {
|
||||
"slice_id": 77,
|
||||
"dataMask": {},
|
||||
}
|
||||
})
|
||||
extractor.client.get_chart = AsyncMock(side_effect=Exception("Chart API error"))
|
||||
result = await extractor.parse_superset_link(self.PERMALINK_URL)
|
||||
assert "chart_dataset_binding_unresolved" in result.unresolved_references
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_permalink_state_not_a_dict(self, extractor):
|
||||
"""Permalink state value is not a dict — logged, does not crash."""
|
||||
_set_permalink_state(extractor, {"state": "not_a_dict"})
|
||||
extractor.client.get_dashboard_detail = AsyncMock(
|
||||
return_value={"id": 42, "datasets": []}
|
||||
)
|
||||
result = await extractor.parse_superset_link(self.PERMALINK_URL)
|
||||
assert result.resource_type == "dashboard"
|
||||
assert result.partial_recovery is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_permalink_no_ids_found(self, extractor):
|
||||
"""Permalink state without dashboardId or sliceId results in partial."""
|
||||
_set_permalink_state(extractor, {
|
||||
"state": {
|
||||
"dataMask": {},
|
||||
}
|
||||
})
|
||||
result = await extractor.parse_superset_link(self.PERMALINK_URL)
|
||||
assert result.partial_recovery is True
|
||||
assert result.dataset_id is None
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Dashboard slug / numeric reference
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestParseSupersetLinkDashboardRef:
|
||||
"""parse_superset_link — /dashboard/<id-or-slug>/ path"""
|
||||
|
||||
DASH_URL = "https://superset.example.com/superset/dashboard/my-dash/"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dashboard_slug_resolves_datasets(self, extractor):
|
||||
"""Dashboard slug resolves dashboard detail and first dataset."""
|
||||
_set_dashboard_detail(extractor, {
|
||||
"id": 5,
|
||||
"datasets": [{"id": 10, "table_name": "users"}],
|
||||
})
|
||||
result = await extractor.parse_superset_link(self.DASH_URL)
|
||||
assert result.dashboard_id == 5
|
||||
assert result.dataset_id == 10
|
||||
# dataset_ref gets canonicalized by get_dataset_detail result
|
||||
assert result.dataset_ref == "public.users"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dashboard_multiple_datasets_partial(self, extractor):
|
||||
"""Dashboard with multiple datasets sets partial_recovery."""
|
||||
_set_dashboard_detail(extractor, {
|
||||
"id": 5,
|
||||
"datasets": [
|
||||
{"id": 10, "table_name": "users"},
|
||||
{"id": 11, "table_name": "orders"},
|
||||
],
|
||||
})
|
||||
result = await extractor.parse_superset_link(self.DASH_URL)
|
||||
assert result.partial_recovery is True
|
||||
assert "multiple_dashboard_datasets" in result.unresolved_references
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dashboard_no_datasets_partial(self, extractor):
|
||||
"""Dashboard with no datasets sets partial_recovery."""
|
||||
_set_dashboard_detail(extractor, {
|
||||
"id": 5,
|
||||
"datasets": [],
|
||||
})
|
||||
result = await extractor.parse_superset_link(self.DASH_URL)
|
||||
assert result.partial_recovery is True
|
||||
assert "dashboard_dataset_binding_missing" in result.unresolved_references
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dashboard_dataset_missing_id_partial(self, extractor):
|
||||
"""Dashboard dataset entry without id sets partial."""
|
||||
_set_dashboard_detail(extractor, {
|
||||
"id": 5,
|
||||
"datasets": [{"table_name": "users"}],
|
||||
})
|
||||
result = await extractor.parse_superset_link(self.DASH_URL)
|
||||
assert result.partial_recovery is True
|
||||
assert "dashboard_dataset_id_missing" in result.unresolved_references
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Dashboard with native_filters_key
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestParseSupersetLinkNativeFiltersKey:
|
||||
"""parse_superset_link — dashboard link with native_filters_key query param."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_filters_key_success(self, extractor):
|
||||
"""native_filters_key fetches filter state and merges dataMask."""
|
||||
url = "https://superset.example.com/superset/dashboard/5?native_filters_key=key123"
|
||||
_set_dashboard_detail(extractor, {
|
||||
"id": 5,
|
||||
"datasets": [{"id": 10}],
|
||||
})
|
||||
_set_native_filters_key(extractor)
|
||||
result = await extractor.parse_superset_link(url)
|
||||
assert result.dashboard_id == 5
|
||||
assert "native_filter_state" in result.query_state
|
||||
extractor.client.extract_native_filters_from_key.assert_called_once_with(5, "key123")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_filters_key_empty_datamask(self, extractor):
|
||||
"""native_filters_key with empty dataMask is logged, not merged."""
|
||||
url = "https://superset.example.com/superset/dashboard/5?native_filters_key=key123"
|
||||
_set_dashboard_detail(extractor, {
|
||||
"id": 5,
|
||||
"datasets": [{"id": 10}],
|
||||
})
|
||||
_set_native_filters_key(extractor, {"dataMask": {}})
|
||||
result = await extractor.parse_superset_link(url)
|
||||
assert "native_filter_state" not in result.query_state
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_filters_key_exception(self, extractor):
|
||||
"""native_filters_key failure is logged, does not crash."""
|
||||
url = "https://superset.example.com/superset/dashboard/5?native_filters_key=key123"
|
||||
_set_dashboard_detail(extractor, {
|
||||
"id": 5,
|
||||
"datasets": [{"id": 10}],
|
||||
})
|
||||
extractor.client.extract_native_filters_from_key = MagicMock(
|
||||
side_effect=Exception("API error")
|
||||
)
|
||||
result = await extractor.parse_superset_link(url)
|
||||
assert result.dashboard_id == 5
|
||||
assert result.dataset_id == 10
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Chart-only link
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestParseSupersetLinkChartOnly:
|
||||
"""parse_superset_link — /chart/N/ path without dashboard context."""
|
||||
|
||||
CHART_URL = "https://superset.example.com/superset/chart/42/"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chart_only_partial_recovery(self, extractor):
|
||||
"""Chart-only link returns partial_recovery=True."""
|
||||
result = await extractor.parse_superset_link(self.CHART_URL)
|
||||
assert result.chart_id == 42
|
||||
assert result.resource_type == "chart"
|
||||
assert result.partial_recovery is True
|
||||
assert "chart_dataset_binding_unresolved" in result.unresolved_references
|
||||
assert result.dataset_ref == "chart:42"
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Unsupported link shape
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestParseSupersetLinkUnsupported:
|
||||
"""parse_superset_link — unrecognized path shapes."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unsupported_path_raises_value_error(self, extractor):
|
||||
"""Link without dataset/dashboard/chart raises ValueError."""
|
||||
with pytest.raises(ValueError, match="Unsupported Superset link"):
|
||||
await extractor.parse_superset_link(
|
||||
"https://superset.example.com/superset/other/5/"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explore_path_without_known_prefix(self, extractor):
|
||||
"""/explore/ path without known prefixes raises ValueError."""
|
||||
with pytest.raises(ValueError, match="Unsupported Superset link"):
|
||||
await extractor.parse_superset_link(
|
||||
"https://superset.example.com/superset/explore/"
|
||||
)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# _recover_dataset_binding_from_dashboard
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestRecoverDatasetBindingFromDashboard:
|
||||
"""_recover_dataset_binding_from_dashboard — recover dataset from dashboard detail."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recovers_from_single_dataset(self, extractor):
|
||||
"""Single dataset returns resolved id."""
|
||||
_set_dashboard_detail(extractor, {"datasets": [{"id": 10}]})
|
||||
ds_id, refs = await extractor._recover_dataset_binding_from_dashboard(
|
||||
dashboard_id=5, dataset_ref=None, unresolved_references=[]
|
||||
)
|
||||
assert ds_id == 10
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recovers_from_multiple_datasets(self, extractor):
|
||||
"""Multiple datasets adds multiple_dashboard_datasets marker."""
|
||||
_set_dashboard_detail(extractor, {"datasets": [{"id": 10}, {"id": 11}]})
|
||||
ds_id, refs = await extractor._recover_dataset_binding_from_dashboard(
|
||||
dashboard_id=5, dataset_ref=None, unresolved_references=[]
|
||||
)
|
||||
assert ds_id == 10
|
||||
assert "multiple_dashboard_datasets" in refs
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_datasets_returns_none(self, extractor):
|
||||
"""No datasets returns None with marker."""
|
||||
_set_dashboard_detail(extractor, {"datasets": []})
|
||||
ds_id, refs = await extractor._recover_dataset_binding_from_dashboard(
|
||||
dashboard_id=5, dataset_ref=None, unresolved_references=[]
|
||||
)
|
||||
assert ds_id is None
|
||||
assert "dashboard_dataset_binding_missing" in refs
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dataset_missing_id_returns_none(self, extractor):
|
||||
"""Dataset without id returns None."""
|
||||
_set_dashboard_detail(extractor, {"datasets": [{"table_name": "users"}]})
|
||||
ds_id, refs = await extractor._recover_dataset_binding_from_dashboard(
|
||||
dashboard_id=5, dataset_ref=None, unresolved_references=[]
|
||||
)
|
||||
assert ds_id is None
|
||||
assert "dashboard_dataset_id_missing" in refs
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_existing_markers_not_duplicated(self, extractor):
|
||||
"""Existing markers are not duplicated."""
|
||||
_set_dashboard_detail(extractor, {"datasets": []})
|
||||
_, refs = await extractor._recover_dataset_binding_from_dashboard(
|
||||
dashboard_id=5,
|
||||
dataset_ref=None,
|
||||
unresolved_references=["dashboard_dataset_binding_missing"],
|
||||
)
|
||||
matches = [r for r in refs if r == "dashboard_dataset_binding_missing"]
|
||||
assert len(matches) == 1
|
||||
# #endregion Test.SupersetContextParsing
|
||||
225
backend/tests/test_core/test_superset_context_extractor_pii.py
Normal file
225
backend/tests/test_core/test_superset_context_extractor_pii.py
Normal file
@@ -0,0 +1,225 @@
|
||||
# #region Test.SupersetContextPII [C:2] [TYPE Module] [SEMANTICS test, superset, pii, mask, sanitize]
|
||||
# @BRIEF Tests for _pii.py — mask_pii_value, sanitize_imported_filter_for_assistant, _mask_sensitive_text.
|
||||
# @RELATION BINDS_TO -> [SupersetContextExtractorPII]
|
||||
# @TEST_CONTRACT: [value:Any] -> [tuple[Any, bool]]
|
||||
# @TEST_SCENARIO: email_mask -> replaces local part with ***
|
||||
# @TEST_SCENARIO: uuid_mask -> preserves last 12 chars
|
||||
# @TEST_SCENARIO: long_digit_mask -> preserves last 2 chars
|
||||
# @TEST_SCENARIO: mixed_identifier_mask -> preserves first+last 2 chars
|
||||
# @TEST_EDGE: nested_structure -> recurses into list/dict
|
||||
# @TEST_EDGE: non_string_value -> passthrough with masked=False
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.utils.superset_context_extractor._pii import (
|
||||
_mask_sensitive_text,
|
||||
mask_pii_value,
|
||||
sanitize_imported_filter_for_assistant,
|
||||
)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# _mask_sensitive_text
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestMaskSensitiveText:
|
||||
"""_mask_sensitive_text — redact PII substrings from text."""
|
||||
|
||||
def test_email_masked(self):
|
||||
"""Email local-part replaced with ***."""
|
||||
result, masked = _mask_sensitive_text("user@example.com")
|
||||
assert masked is True
|
||||
assert "@example.com" in result
|
||||
assert result.startswith("***")
|
||||
|
||||
def test_email_no_match(self):
|
||||
"""Text without email returns unchanged."""
|
||||
result, masked = _mask_sensitive_text("hello world")
|
||||
assert masked is False
|
||||
assert result == "hello world"
|
||||
|
||||
def test_uuid_masked(self):
|
||||
"""UUID replaced with asterisk pattern, last 12 chars preserved before long-digit pass."""
|
||||
uuid_str = "550e8400-e29b-41d4-a716-446655440000"
|
||||
result, masked = _mask_sensitive_text(uuid_str)
|
||||
assert masked is True
|
||||
# UUID pattern replaces with "********-****-****-****-" + token[-12:]
|
||||
# Then LONG_DIGIT_PATTERN hits the digit portion -> preserves last 2
|
||||
assert result.startswith("********-****-****-****-")
|
||||
assert result != uuid_str
|
||||
# The last part gets further masked by long-digit pattern
|
||||
assert result.count("*") >= 20
|
||||
|
||||
def test_long_digits_masked(self):
|
||||
"""Long digit string preserves last 2 chars."""
|
||||
result, masked = _mask_sensitive_text("1234567890")
|
||||
assert masked is True
|
||||
assert result.endswith("90")
|
||||
assert len(result) == len("1234567890")
|
||||
|
||||
def test_short_digits_not_masked(self):
|
||||
"""Short digit string (<6 chars) is not masked."""
|
||||
result, masked = _mask_sensitive_text("12345")
|
||||
assert masked is False
|
||||
assert result == "12345"
|
||||
|
||||
def test_mixed_identifier_masked(self):
|
||||
"""Mixed alphanumeric identifier masked."""
|
||||
result, masked = _mask_sensitive_text("abc123xyz789")
|
||||
assert masked is True
|
||||
# First 2 + *** + last 2
|
||||
assert "ab" in result and "89" in result
|
||||
|
||||
def test_mixed_identifier_short_uppercase(self):
|
||||
"""Short uppercase identifier (<5) not masked."""
|
||||
result, masked = _mask_sensitive_text("ABC12")
|
||||
assert masked is False
|
||||
|
||||
def test_email_and_uuid_both_masked(self):
|
||||
"""Multiple sensitive patterns in one string."""
|
||||
result, masked = _mask_sensitive_text(
|
||||
"Contact: user@test.com, token=550e8400-e29b-41d4-a716-446655440000"
|
||||
)
|
||||
assert masked is True
|
||||
assert "***@test.com" in result
|
||||
assert "********" in result # UUID masking present
|
||||
|
||||
def test_empty_string(self):
|
||||
"""Empty string returns unmasked."""
|
||||
result, masked = _mask_sensitive_text("")
|
||||
assert masked is False
|
||||
assert result == ""
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# mask_pii_value
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestMaskPiiValue:
|
||||
"""mask_pii_value — redact PII from nested structures."""
|
||||
|
||||
def test_string_with_email(self):
|
||||
"""String with email is masked."""
|
||||
result, masked = mask_pii_value("email: user@test.com")
|
||||
assert masked is True
|
||||
assert "***@test.com" in result
|
||||
|
||||
def test_non_string_passthrough(self):
|
||||
"""Non-string value returns unchanged, masked=False."""
|
||||
result, masked = mask_pii_value(42)
|
||||
assert masked is False
|
||||
assert result == 42
|
||||
|
||||
result, masked = mask_pii_value(None)
|
||||
assert masked is False
|
||||
assert result is None
|
||||
|
||||
result, masked = mask_pii_value(True)
|
||||
assert masked is False
|
||||
assert result is True
|
||||
|
||||
def test_list_with_sensitive_items(self):
|
||||
"""List with email items is masked."""
|
||||
result, masked = mask_pii_value(["user@test.com", "hello"])
|
||||
assert masked is True
|
||||
assert "***@test.com" in result[0]
|
||||
assert result[1] == "hello"
|
||||
|
||||
def test_list_no_sensitive_items(self):
|
||||
"""List without sensitive items returns unmasked."""
|
||||
result, masked = mask_pii_value(["hello", "world"])
|
||||
assert masked is False
|
||||
assert result == ["hello", "world"]
|
||||
|
||||
def test_dict_with_sensitive_value(self):
|
||||
"""Dict with email value is masked."""
|
||||
result, masked = mask_pii_value({"email": "user@test.com", "name": "John"})
|
||||
assert masked is True
|
||||
assert "***@test.com" in result["email"]
|
||||
assert result["name"] == "John"
|
||||
|
||||
def test_dict_no_sensitive_values(self):
|
||||
"""Dict without sensitive values returns unmasked."""
|
||||
result, masked = mask_pii_value({"a": 1, "b": "hello"})
|
||||
assert masked is False
|
||||
|
||||
def test_nested_dict_in_list(self):
|
||||
"""Nested structure is recursed."""
|
||||
result, masked = mask_pii_value(
|
||||
{"users": [{"email": "admin@test.com"}], "count": 5}
|
||||
)
|
||||
assert masked is True
|
||||
assert "***@test.com" in result["users"][0]["email"]
|
||||
|
||||
def test_empty_list(self):
|
||||
"""Empty list returns unmasked."""
|
||||
result, masked = mask_pii_value([])
|
||||
assert masked is False
|
||||
assert result == []
|
||||
|
||||
def test_empty_dict(self):
|
||||
"""Empty dict returns unmasked."""
|
||||
result, masked = mask_pii_value({})
|
||||
assert masked is False
|
||||
assert result == {}
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# sanitize_imported_filter_for_assistant
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestSanitizeImportedFilterForAssistant:
|
||||
"""sanitize_imported_filter_for_assistant — build assistant-safe filter payload."""
|
||||
|
||||
def test_sanitizes_raw_value_with_email(self):
|
||||
"""Raw value with email is masked and flag set."""
|
||||
result = sanitize_imported_filter_for_assistant(
|
||||
{"filter_name": "email_filter", "raw_value": "user@test.com"}
|
||||
)
|
||||
assert result["raw_value_masked"] is True
|
||||
assert "***@test.com" in result["raw_value"]
|
||||
|
||||
def test_passthrough_non_sensitive(self):
|
||||
"""Non-sensitive value passes through with mask=False."""
|
||||
result = sanitize_imported_filter_for_assistant(
|
||||
{"filter_name": "country", "raw_value": "US"}
|
||||
)
|
||||
assert result["raw_value_masked"] is False
|
||||
assert result["raw_value"] == "US"
|
||||
|
||||
def test_preserves_existing_raw_value_masked(self):
|
||||
"""Existing raw_value_masked flag is preserved."""
|
||||
result = sanitize_imported_filter_for_assistant(
|
||||
{
|
||||
"filter_name": "f1",
|
||||
"raw_value": "US",
|
||||
"raw_value_masked": True,
|
||||
}
|
||||
)
|
||||
assert result["raw_value_masked"] is True
|
||||
|
||||
def test_does_not_mutate_original(self):
|
||||
"""Original dict is not mutated (deepcopy)."""
|
||||
original = {"filter_name": "f1", "raw_value": "user@test.com"}
|
||||
result = sanitize_imported_filter_for_assistant(original)
|
||||
assert original["raw_value"] == "user@test.com"
|
||||
assert result["raw_value"] != original["raw_value"]
|
||||
|
||||
def test_none_raw_value(self):
|
||||
"""None raw_value handled gracefully."""
|
||||
result = sanitize_imported_filter_for_assistant(
|
||||
{"filter_name": "f1", "raw_value": None}
|
||||
)
|
||||
assert result["raw_value_masked"] is False
|
||||
assert result["raw_value"] is None
|
||||
# #endregion Test.SupersetContextPII
|
||||
@@ -0,0 +1,538 @@
|
||||
# #region Test.SupersetContextRecovery [C:3] [TYPE Module] [SEMANTICS test, superset, recovery, filter, dashboard]
|
||||
# @BRIEF Tests for _recovery.py — SupersetContextRecoveryMixin recover_imported_filters and _normalize_imported_filter_payload.
|
||||
# @RELATION BINDS_TO -> [SupersetContextRecoveryMixin]
|
||||
# @TEST_CONTRACT: [SupersetParsedContext] -> [List[Dict]]
|
||||
# @TEST_SCENARIO: dashboard_with_native_filters -> recovers from json_metadata
|
||||
# @TEST_SCENARIO: no_dashboard_id -> URL filters only
|
||||
# @TEST_SCENARIO: empty_recovered -> fallback entry
|
||||
# @TEST_EDGE: json_metadata_as_string -> parsed correctly
|
||||
# @TEST_EDGE: default_filters_as_string -> parsed correctly
|
||||
# @TEST_EDGE: metadata_fetch_exception -> fallback to URL filters
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-testing")
|
||||
os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:")
|
||||
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:")
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.utils.superset_context_extractor._base import SupersetParsedContext
|
||||
from src.core.utils.superset_context_extractor._recovery import (
|
||||
SupersetContextRecoveryMixin,
|
||||
)
|
||||
|
||||
|
||||
class _TestExtractor(SupersetContextRecoveryMixin):
|
||||
"""Concrete test class for the recovery mixin."""
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def extractor():
|
||||
ext = _TestExtractor()
|
||||
ext.client = MagicMock()
|
||||
return ext
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# _normalize_imported_filter_payload
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestNormalizeImportedFilterPayload:
|
||||
"""_normalize_imported_filter_payload — normalize one filter entry."""
|
||||
|
||||
def test_minimal_payload(self, extractor):
|
||||
"""Minimal payload uses defaults."""
|
||||
result = extractor._normalize_imported_filter_payload(
|
||||
{"filter_name": "country"},
|
||||
default_source="superset_url",
|
||||
default_note="From URL",
|
||||
)
|
||||
assert result["filter_name"] == "country"
|
||||
assert result["source"] == "superset_url"
|
||||
assert result["notes"] == "From URL"
|
||||
assert result["raw_value"] is None
|
||||
assert result["recovery_status"] == "partial"
|
||||
assert result["requires_confirmation"] is True
|
||||
assert result["confidence_state"] == "unresolved"
|
||||
|
||||
def test_full_payload(self, extractor):
|
||||
"""Full payload preserves values."""
|
||||
result = extractor._normalize_imported_filter_payload(
|
||||
{
|
||||
"filter_name": " country ",
|
||||
"display_name": "Country",
|
||||
"raw_value": "US",
|
||||
"source": "superset_permalink",
|
||||
"recovery_status": "recovered",
|
||||
"requires_confirmation": False,
|
||||
"notes": "From permalink",
|
||||
},
|
||||
default_source="superset_url",
|
||||
default_note="Fallback",
|
||||
)
|
||||
assert result["filter_name"] == "country" # stripped
|
||||
assert result["display_name"] == "Country"
|
||||
assert result["raw_value"] == "US"
|
||||
assert result["recovery_status"] == "recovered"
|
||||
assert result["requires_confirmation"] is False
|
||||
assert result["confidence_state"] == "imported"
|
||||
|
||||
def test_value_fallback(self, extractor):
|
||||
"""'value' key used when 'raw_value' absent."""
|
||||
result = extractor._normalize_imported_filter_payload(
|
||||
{"filter_name": "c", "value": "fallback"},
|
||||
default_source="url",
|
||||
default_note="note",
|
||||
)
|
||||
assert result["raw_value"] == "fallback"
|
||||
|
||||
def test_no_name_fallback(self, extractor):
|
||||
"""Missing filter_name uses 'unresolved_filter'."""
|
||||
result = extractor._normalize_imported_filter_payload(
|
||||
{},
|
||||
default_source="url",
|
||||
default_note="note",
|
||||
)
|
||||
assert result["filter_name"] == "unresolved_filter"
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# recover_imported_filters — no dashboard context
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestRecoverImportedFiltersNoDashboard:
|
||||
"""recover_imported_filters — when dashboard_id is None."""
|
||||
|
||||
def test_no_dashboard_id_url_filters_only(self, extractor):
|
||||
"""Filters from URL state are recovered without dashboard enrichment."""
|
||||
ctx = SupersetParsedContext(
|
||||
source_url="https://superset.example.com/superset/chart/42",
|
||||
dataset_ref="chart:42",
|
||||
chart_id=42,
|
||||
resource_type="chart",
|
||||
partial_recovery=True,
|
||||
unresolved_references=["chart_dataset_binding_unresolved"],
|
||||
imported_filters=[
|
||||
{
|
||||
"filter_name": "country",
|
||||
"raw_value": "US",
|
||||
"source": "superset_url",
|
||||
}
|
||||
],
|
||||
)
|
||||
result = extractor.recover_imported_filters(ctx)
|
||||
assert len(result) == 1
|
||||
assert result[0]["filter_name"] == "country"
|
||||
assert result[0]["raw_value"] == "US"
|
||||
|
||||
def test_empty_url_filters_returns_empty(self, extractor):
|
||||
"""No URL filters returns empty list when no dashboard."""
|
||||
ctx = SupersetParsedContext(
|
||||
source_url="https://superset.example.com/superset/chart/42",
|
||||
dataset_ref="chart:42",
|
||||
chart_id=42,
|
||||
resource_type="chart",
|
||||
)
|
||||
result = extractor.recover_imported_filters(ctx)
|
||||
assert result == []
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# recover_imported_filters — with dashboard context
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestRecoverImportedFiltersWithDashboard:
|
||||
"""recover_imported_filters — with dashboard_id, enriched from metadata."""
|
||||
|
||||
# ── Helpers ──────────────────────────────
|
||||
|
||||
def _make_dashboard_payload(
|
||||
self,
|
||||
native_filter_configuration=None,
|
||||
default_filters=None,
|
||||
) -> dict:
|
||||
meta = {}
|
||||
if native_filter_configuration is not None:
|
||||
meta["native_filter_configuration"] = native_filter_configuration
|
||||
if default_filters is not None:
|
||||
meta["default_filters"] = default_filters
|
||||
payload = {"json_metadata": json.dumps(meta) if meta else "{}"}
|
||||
return {"result": payload}
|
||||
|
||||
# ── Tests ────────────────────────────────
|
||||
|
||||
def test_dashboard_recovers_native_filters(self, extractor):
|
||||
"""Dashboard with native filter configuration recovers filter entries."""
|
||||
extractor.client.get_dashboard = MagicMock(
|
||||
return_value=self._make_dashboard_payload(
|
||||
native_filter_configuration=[
|
||||
{
|
||||
"name": "country_filter",
|
||||
"label": "Country",
|
||||
"id": "NATIVE-1",
|
||||
}
|
||||
],
|
||||
default_filters={"country_filter": "US"},
|
||||
)
|
||||
)
|
||||
ctx = SupersetParsedContext(
|
||||
source_url="https://superset.example.com/superset/dashboard/5",
|
||||
dataset_ref="dataset:10",
|
||||
dataset_id=10,
|
||||
dashboard_id=5,
|
||||
resource_type="dashboard",
|
||||
)
|
||||
result = extractor.recover_imported_filters(ctx)
|
||||
assert len(result) == 1
|
||||
assert result[0]["filter_name"] == "country_filter"
|
||||
assert result[0]["display_name"] == "Country"
|
||||
assert result[0]["raw_value"] == "US"
|
||||
assert result[0]["recovery_status"] == "recovered"
|
||||
|
||||
def test_native_filter_partial_no_default(self, extractor):
|
||||
"""Native filter without default value gets partial status."""
|
||||
extractor.client.get_dashboard = MagicMock(
|
||||
return_value=self._make_dashboard_payload(
|
||||
native_filter_configuration=[
|
||||
{"name": "year_filter", "label": "Year"}
|
||||
],
|
||||
)
|
||||
)
|
||||
ctx = SupersetParsedContext(
|
||||
source_url="https://superset.example.com/superset/dashboard/5",
|
||||
dataset_ref="dataset:10",
|
||||
dataset_id=10,
|
||||
dashboard_id=5,
|
||||
)
|
||||
result = extractor.recover_imported_filters(ctx)
|
||||
assert len(result) == 1
|
||||
assert result[0]["recovery_status"] == "partial"
|
||||
assert result[0]["requires_confirmation"] is True
|
||||
|
||||
def test_native_filters_merged_with_url_filters(self, extractor):
|
||||
"""URL-imported filters merge with dashboard metadata."""
|
||||
extractor.client.get_dashboard = MagicMock(
|
||||
return_value=self._make_dashboard_payload(
|
||||
native_filter_configuration=[
|
||||
{"name": "country_filter", "label": "Country", "id": "NATIVE-1"}
|
||||
],
|
||||
default_filters={"country_filter": "US"},
|
||||
)
|
||||
)
|
||||
ctx = SupersetParsedContext(
|
||||
source_url="https://superset.example.com/superset/dashboard/5",
|
||||
dataset_ref="dataset:10",
|
||||
dataset_id=10,
|
||||
dashboard_id=5,
|
||||
imported_filters=[
|
||||
{
|
||||
"filter_name": "NATIVE-1",
|
||||
"raw_value": None,
|
||||
"source": "superset_url",
|
||||
}
|
||||
],
|
||||
)
|
||||
result = extractor.recover_imported_filters(ctx)
|
||||
# Should be 1 entry after merge
|
||||
assert len(result) == 1
|
||||
# Filter name should be canonicalized from metadata
|
||||
assert result[0]["filter_name"] == "country_filter"
|
||||
|
||||
def test_default_filters_as_string_parsed(self, extractor):
|
||||
"""default_filters JSON string is parsed."""
|
||||
extractor.client.get_dashboard = MagicMock(
|
||||
return_value=self._make_dashboard_payload(
|
||||
native_filter_configuration=[
|
||||
{"name": "f1", "id": "N-1"}
|
||||
],
|
||||
default_filters='{"f1": "val1"}',
|
||||
)
|
||||
)
|
||||
ctx = SupersetParsedContext(
|
||||
source_url="https://superset.example.com/superset/dashboard/5",
|
||||
dataset_ref="dataset:10",
|
||||
dataset_id=10,
|
||||
dashboard_id=5,
|
||||
)
|
||||
result = extractor.recover_imported_filters(ctx)
|
||||
assert result[0]["raw_value"] == "val1"
|
||||
|
||||
def test_default_filters_invalid_json_logged(self, extractor):
|
||||
"""Invalid default_filters JSON is caught and logged."""
|
||||
extractor.client.get_dashboard = MagicMock(
|
||||
return_value=self._make_dashboard_payload(
|
||||
native_filter_configuration=[
|
||||
{"name": "f1"}
|
||||
],
|
||||
default_filters="{invalid}",
|
||||
)
|
||||
)
|
||||
ctx = SupersetParsedContext(
|
||||
source_url="https://superset.example.com/superset/dashboard/5",
|
||||
dataset_ref="dataset:10",
|
||||
dataset_id=10,
|
||||
dashboard_id=5,
|
||||
)
|
||||
result = extractor.recover_imported_filters(ctx)
|
||||
# Should not crash, filter has no default
|
||||
assert result[0]["raw_value"] is None
|
||||
|
||||
def test_get_dashboard_failure_falls_back(self, extractor):
|
||||
"""Dashboard fetch failure falls back to URL filters only."""
|
||||
extractor.client.get_dashboard = MagicMock(
|
||||
side_effect=Exception("Dashboard API error")
|
||||
)
|
||||
ctx = SupersetParsedContext(
|
||||
source_url="https://superset.example.com/superset/dashboard/5",
|
||||
dataset_ref="dataset:10",
|
||||
dataset_id=10,
|
||||
dashboard_id=5,
|
||||
imported_filters=[
|
||||
{
|
||||
"filter_name": "country",
|
||||
"raw_value": "US",
|
||||
"source": "superset_url",
|
||||
}
|
||||
],
|
||||
)
|
||||
result = extractor.recover_imported_filters(ctx)
|
||||
assert len(result) == 1
|
||||
assert result[0]["filter_name"] == "country"
|
||||
|
||||
def test_empty_recovered_filters_adds_fallback(self, extractor):
|
||||
"""No recovered filters adds a fallback entry."""
|
||||
extractor.client.get_dashboard = MagicMock(
|
||||
return_value=self._make_dashboard_payload(
|
||||
native_filter_configuration=[],
|
||||
)
|
||||
)
|
||||
ctx = SupersetParsedContext(
|
||||
source_url="https://superset.example.com/superset/dashboard/5",
|
||||
dataset_ref="dataset:10",
|
||||
dataset_id=10,
|
||||
dashboard_id=5,
|
||||
)
|
||||
result = extractor.recover_imported_filters(ctx)
|
||||
assert len(result) == 1
|
||||
assert result[0]["recovery_status"] == "partial"
|
||||
assert "dashboard_5_filters" in result[0]["filter_name"]
|
||||
|
||||
def test_non_dict_item_in_native_config_skipped(self, extractor):
|
||||
"""Non-dict items in native_filter_configuration are skipped."""
|
||||
extractor.client.get_dashboard = MagicMock(
|
||||
return_value=self._make_dashboard_payload(
|
||||
native_filter_configuration=[
|
||||
"not_a_dict",
|
||||
{"name": "valid_filter"},
|
||||
],
|
||||
)
|
||||
)
|
||||
ctx = SupersetParsedContext(
|
||||
source_url="https://superset.example.com/superset/dashboard/5",
|
||||
dataset_ref="dataset:10",
|
||||
dataset_id=10,
|
||||
dashboard_id=5,
|
||||
)
|
||||
result = extractor.recover_imported_filters(ctx)
|
||||
assert len(result) == 1
|
||||
assert result[0]["filter_name"] == "valid_filter"
|
||||
|
||||
def test_json_metadata_as_string_parsed(self, extractor):
|
||||
"""json_metadata as string is parsed (not dict)."""
|
||||
extractor.client.get_dashboard = MagicMock(
|
||||
return_value={
|
||||
"result": {
|
||||
"json_metadata": json.dumps(
|
||||
{
|
||||
"native_filter_configuration": [
|
||||
{"name": "f1"}
|
||||
]
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
)
|
||||
ctx = SupersetParsedContext(
|
||||
source_url="https://superset.example.com/superset/dashboard/5",
|
||||
dataset_ref="dataset:10",
|
||||
dataset_id=10,
|
||||
dashboard_id=5,
|
||||
)
|
||||
result = extractor.recover_imported_filters(ctx)
|
||||
assert len(result) == 1
|
||||
assert result[0]["filter_name"] == "f1"
|
||||
|
||||
def test_empty_filter_name_skipped(self, extractor):
|
||||
"""Filter with empty name in native configuration is skipped."""
|
||||
extractor.client.get_dashboard = MagicMock(
|
||||
return_value=self._make_dashboard_payload(
|
||||
native_filter_configuration=[
|
||||
{"name": ""},
|
||||
{"name": "valid_filter"},
|
||||
],
|
||||
)
|
||||
)
|
||||
ctx = SupersetParsedContext(
|
||||
source_url="https://superset.example.com/superset/dashboard/5",
|
||||
dataset_ref="dataset:10",
|
||||
dataset_id=10,
|
||||
dashboard_id=5,
|
||||
)
|
||||
result = extractor.recover_imported_filters(ctx)
|
||||
assert len(result) == 1
|
||||
assert result[0]["filter_name"] == "valid_filter"
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# merge_recovered_filter edge cases
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestRecoveredFilterMerge:
|
||||
"""Internal merge_recovered_filter behavior via recover_imported_filters."""
|
||||
|
||||
def test_url_filter_upgraded_by_metadata(self, extractor):
|
||||
"""URL filter gets display_name from metadata when missing."""
|
||||
extractor.client.get_dashboard = MagicMock(
|
||||
return_value={
|
||||
"result": {
|
||||
"json_metadata": json.dumps(
|
||||
{
|
||||
"native_filter_configuration": [
|
||||
{
|
||||
"name": "country_filter",
|
||||
"label": "Country",
|
||||
"id": "NATIVE-1",
|
||||
}
|
||||
],
|
||||
"default_filters": {"country_filter": "US"},
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
)
|
||||
ctx = SupersetParsedContext(
|
||||
source_url="https://superset.example.com/superset/dashboard/5",
|
||||
dataset_ref="dataset:10",
|
||||
dataset_id=10,
|
||||
dashboard_id=5,
|
||||
imported_filters=[
|
||||
{
|
||||
"filter_name": "NATIVE-1",
|
||||
"raw_value": None,
|
||||
"display_name": None,
|
||||
"source": "superset_url",
|
||||
}
|
||||
],
|
||||
)
|
||||
result = extractor.recover_imported_filters(ctx)
|
||||
assert result[0]["display_name"] == "Country"
|
||||
assert result[0]["filter_name"] == "country_filter"
|
||||
|
||||
def test_display_name_merged_from_metadata(self, extractor):
|
||||
"""Existing filter without display_name gets it from metadata merge (line 60)."""
|
||||
extractor.client.get_dashboard = MagicMock(
|
||||
return_value={
|
||||
"result": {
|
||||
"json_metadata": json.dumps(
|
||||
{
|
||||
"native_filter_configuration": [
|
||||
{"name": "country", "label": "Country Label"}
|
||||
],
|
||||
"default_filters": {"country": "US"},
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
)
|
||||
# URL filter with display_name=None, metadata has the label
|
||||
ctx = SupersetParsedContext(
|
||||
source_url="https://superset.example.com/superset/dashboard/5",
|
||||
dataset_ref="dataset:10",
|
||||
dataset_id=10,
|
||||
dashboard_id=5,
|
||||
imported_filters=[
|
||||
{
|
||||
"filter_name": "country",
|
||||
"raw_value": None,
|
||||
"display_name": None,
|
||||
"source": "superset_url",
|
||||
}
|
||||
],
|
||||
)
|
||||
result = extractor.recover_imported_filters(ctx)
|
||||
# Metadata filter with display_name should enrich the existing entry
|
||||
entry = next(f for f in result if f["filter_name"] == "country")
|
||||
assert entry["display_name"] == "Country Label"
|
||||
|
||||
def test_normalized_value_merged_from_metadata(self, extractor):
|
||||
"""Existing filter without normalized_value gets it from metadata merge (line 82)."""
|
||||
extractor.client.get_dashboard = MagicMock(
|
||||
return_value={
|
||||
"result": {
|
||||
"json_metadata": json.dumps(
|
||||
{
|
||||
"native_filter_configuration": [
|
||||
{"name": "status", "id": "N-1"}
|
||||
],
|
||||
"default_filters": {"status": "active"},
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
)
|
||||
ctx = SupersetParsedContext(
|
||||
source_url="https://superset.example.com/superset/dashboard/5",
|
||||
dataset_ref="dataset:10",
|
||||
dataset_id=10,
|
||||
dashboard_id=5,
|
||||
imported_filters=[
|
||||
{
|
||||
"filter_name": "N-1",
|
||||
"raw_value": None,
|
||||
"display_name": None,
|
||||
"source": "superset_url",
|
||||
}
|
||||
],
|
||||
)
|
||||
result = extractor.recover_imported_filters(ctx)
|
||||
# Metadata filter merges — after canonicalization "N-1" becomes "status"
|
||||
status = next(f for f in result if f["filter_name"] == "status")
|
||||
# The metadata filter should have been merged with URL entry
|
||||
assert status["display_name"] is not None
|
||||
|
||||
def test_json_metadata_non_dict_fallback(self, extractor):
|
||||
"""json_metadata that is not a dict is reset to empty (line 105)."""
|
||||
extractor.client.get_dashboard = MagicMock(
|
||||
return_value={
|
||||
"result": {
|
||||
"json_metadata": 42 # Not a dict or string
|
||||
}
|
||||
}
|
||||
)
|
||||
ctx = SupersetParsedContext(
|
||||
source_url="https://superset.example.com/superset/dashboard/5",
|
||||
dataset_ref="dataset:10",
|
||||
dataset_id=10,
|
||||
dashboard_id=5,
|
||||
)
|
||||
result = extractor.recover_imported_filters(ctx)
|
||||
# Should not crash, should add fallback since no native filters found
|
||||
assert len(result) == 1
|
||||
assert result[0]["recovery_status"] == "partial"
|
||||
# #endregion Test.SupersetContextRecovery
|
||||
@@ -0,0 +1,112 @@
|
||||
# #region Test.SupersetContextTemplatesExtra [C:2] [TYPE Module] [SEMANTICS test, superset, template, edge]
|
||||
# @BRIEF Extra tests for _templates.py — edge cases covering lines 45, 62, 80, 92, 148.
|
||||
# @RELATION BINDS_TO -> [SupersetContextTemplatesMixin]
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
|
||||
|
||||
import os
|
||||
|
||||
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-testing")
|
||||
os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:")
|
||||
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:")
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.utils.superset_context_extractor._templates import (
|
||||
SupersetContextTemplatesMixin,
|
||||
)
|
||||
|
||||
|
||||
class _TestExtractor(SupersetContextTemplatesMixin):
|
||||
"""Concrete test class for the mixin."""
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def extractor():
|
||||
return _TestExtractor()
|
||||
|
||||
|
||||
class TestDiscoverTemplateVariablesExtra:
|
||||
"""Extra edge cases for discover_template_variables."""
|
||||
|
||||
def test_empty_filter_values_variable_skipped(self, extractor):
|
||||
"""filter_values('') with empty name is skipped (line 45)."""
|
||||
result = extractor.discover_template_variables({
|
||||
"sql": "{{ filter_values('') }}"
|
||||
})
|
||||
assert result == []
|
||||
|
||||
def test_empty_url_param_variable_skipped(self, extractor):
|
||||
"""url_param('') with empty name is skipped (line 62)."""
|
||||
result = extractor.discover_template_variables({
|
||||
"sql": "{{ url_param('') }}"
|
||||
})
|
||||
assert result == []
|
||||
|
||||
def test_empty_jinja_expression_skipped(self, extractor):
|
||||
"""Empty Jinja expression {{ }} is skipped (line 80)."""
|
||||
result = extractor.discover_template_variables({
|
||||
"sql": "SELECT * FROM table WHERE {{ }}"
|
||||
})
|
||||
assert result == []
|
||||
|
||||
def test_jinja_keyword_not_identifier(self, extractor):
|
||||
"""Jinja expression with 'if' keyword returns no variable (line 92)."""
|
||||
result = extractor.discover_template_variables({
|
||||
"sql": "{% if True %} SELECT * FROM table {% endif %}"
|
||||
})
|
||||
# The {% %} block is not matched by {{ }} re pattern, so this is fine.
|
||||
# Test with {{ if }} instead:
|
||||
result = extractor.discover_template_variables({
|
||||
"sql": "SELECT * FROM table WHERE {{ if }}"
|
||||
})
|
||||
assert result == []
|
||||
|
||||
def test_non_dict_column_skipped_in_columns(self, extractor):
|
||||
"""Non-dict item in columns list is skipped (line 148)."""
|
||||
result = extractor._collect_query_bearing_expressions({
|
||||
"columns": [
|
||||
"string_column",
|
||||
{"sqlExpression": "UPPER(name)"},
|
||||
42,
|
||||
]
|
||||
})
|
||||
assert "UPPER(name)" in result
|
||||
# string items and non-dict int items should not appear
|
||||
assert "string_column" not in result # it's a str but append_expression checks type
|
||||
|
||||
def test_jinja_expression_with_dot_skip(self, extractor):
|
||||
"""Jinja expression using filter_values inside {{ }} is skipped."""
|
||||
result = extractor.discover_template_variables({
|
||||
"sql": "{{ filter_values('country') }}"
|
||||
})
|
||||
# filter_values inside {{ }} is already caught by the filter_values regex,
|
||||
# and the jinja_match also matches but is skipped due to token check.
|
||||
# Should get one entry from filter_values match.
|
||||
assert len(result) == 1
|
||||
assert result[0]["variable_kind"] == "native_filter"
|
||||
|
||||
def test_jinja_expression_with_get_filters_skipped(self, extractor):
|
||||
"""Jinja expression with get_filters is skipped (no extra entry)."""
|
||||
result = extractor.discover_template_variables({
|
||||
"sql": "{{ get_filters('country') }}"
|
||||
})
|
||||
# get_filters is not in the filter_values regex, but jinja_match captures it.
|
||||
# It has 'get_filters(' so the inner loop skips it.
|
||||
assert result == []
|
||||
|
||||
def test_url_param_with_default_value(self, extractor):
|
||||
"""url_param with default literal parsed correctly."""
|
||||
result = extractor.discover_template_variables({
|
||||
"sql": "{{ url_param('limit', '100') }}"
|
||||
})
|
||||
param = next(v for v in result if v["variable_name"] == "limit")
|
||||
assert param["is_required"] is False
|
||||
assert param["default_value"] == "100"
|
||||
# #endregion Test.SupersetContextTemplatesExtra
|
||||
Reference in New Issue
Block a user