feat(037): Phase 3 — US2 Superset-Native Execution (T011-T015)
- T011: 4 NO-SQL tests (reject SQL, scalar execution, error taxonomy, temporal filter) - T012: _chart_data.py — SupersetChartDataMixin with execute_chart_data() - T013: query_executor.py — execute_dashboard_query (no-SQL guard, kind mapping) - T014: Agent tools — inspect_dashboard_query_model + execute_dashboard_result - T015: Verified superset_execute_sql excluded from _SCENARIO_TOOL_ALLOWLIST 4/4 executor tests pass. ChartDataMixin registered in SupersetClient. Dashboard testing tools added to both allowlist + dashboard context affinity.
This commit is contained in:
@@ -23,6 +23,8 @@ _SCENARIO_TOOL_ALLOWLIST: frozenset = frozenset({
|
||||
"run_llm_validation",
|
||||
"run_llm_documentation",
|
||||
"show_capabilities",
|
||||
"inspect_dashboard_query_model",
|
||||
"execute_dashboard_result",
|
||||
})
|
||||
"""Tools allowed in dashboard-testing scenario mode.
|
||||
superset_execute_sql, superset_format_sql, superset_create_dataset, and any
|
||||
@@ -39,6 +41,8 @@ _CONTEXT_TOOL_AFFINITY: dict[str, set[str]] = {
|
||||
"execute_migration",
|
||||
"create_branch",
|
||||
"commit_changes",
|
||||
"inspect_dashboard_query_model",
|
||||
"execute_dashboard_result",
|
||||
},
|
||||
"dataset": {
|
||||
"superset_list_databases",
|
||||
|
||||
@@ -1257,6 +1257,106 @@ async def superset_format_sql(environment_id: str, sql: str) -> str:
|
||||
# #endregion AgentChat.Tools.SupersetFormatSql
|
||||
|
||||
|
||||
# ── 037 Dashboard Testing Tools ──────────────────────────────────
|
||||
|
||||
# #region AgentChat.Tools.InspectDashboardQueryModelInput [C:1] [TYPE Class] [SEMANTICS agent-chat,tools,schema,dashboard-testing]
|
||||
class InspectDashboardQueryModelInput(BaseModel):
|
||||
environment_id: str = Field(..., description="Superset environment ID")
|
||||
dashboard_id: int = Field(..., description="Superset dashboard ID")
|
||||
# #endregion AgentChat.Tools.InspectDashboardQueryModelInput
|
||||
|
||||
|
||||
# #region AgentChat.Tools.InspectDashboardQueryModel [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,dashboard-testing,inspect]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Inspect a dashboard's query model — returns charts, datasets, metrics, native filters.
|
||||
# @PRE User authenticated, dashboard readable.
|
||||
# @POST Returns deterministic JSON snapshot of the dashboard's query structure.
|
||||
@tool(args_schema=InspectDashboardQueryModelInput)
|
||||
async def inspect_dashboard_query_model(environment_id: str, dashboard_id: int) -> str:
|
||||
"""Inspect a dashboard's query model — charts, datasets, metrics, native filters."""
|
||||
logger.reason("Inspect dashboard query model",
|
||||
payload={"environment_id": environment_id, "dashboard_id": dashboard_id},
|
||||
extra={"src": "AgentChat.Tools.InspectDashboardQueryModel"})
|
||||
resp = await _get("/api/dashboard-testing/query-model",
|
||||
params={"environment_id": environment_id, "dashboard_id": str(dashboard_id)})
|
||||
if resp.status_code != 200:
|
||||
logger.explore("Query model inspection failed",
|
||||
payload={"status": resp.status_code, "dashboard_id": dashboard_id},
|
||||
error=resp.text[:200],
|
||||
extra={"src": "AgentChat.Tools.InspectDashboardQueryModel"})
|
||||
return f"Error {resp.status_code}: {resp.text[:500]}"
|
||||
logger.reflect("Query model inspected",
|
||||
payload={"dashboard_id": dashboard_id},
|
||||
extra={"src": "AgentChat.Tools.InspectDashboardQueryModel"})
|
||||
return _trim_response(resp.text)
|
||||
# #endregion AgentChat.Tools.InspectDashboardQueryModel
|
||||
|
||||
|
||||
# #region AgentChat.Tools.ExecuteDashboardResultInput [C:1] [TYPE Class] [SEMANTICS agent-chat,tools,schema,dashboard-testing]
|
||||
class ExecuteDashboardResultInput(BaseModel):
|
||||
environment_id: str = Field(..., description="Superset environment ID")
|
||||
dashboard_id: int = Field(..., description="Superset dashboard ID")
|
||||
chart_id: int | None = Field(None, description="Chart ID to execute (required for chart queries)")
|
||||
dataset_id: int | None = Field(None, description="Dataset ID (alternative to chart_id)")
|
||||
result_key: str = Field(..., description="Metric key to extract from result")
|
||||
normalized_filters_json: str = Field(default="", description="JSON string of normalized filter context (from normalize_filters)")
|
||||
# #endregion AgentChat.Tools.ExecuteDashboardResultInput
|
||||
|
||||
|
||||
# #region AgentChat.Tools.ExecuteDashboardResult [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,dashboard-testing,execute]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Execute a Superset-native chart/dataset query and return normalized result.
|
||||
# @PRE User authenticated, dashboard/chart accessible.
|
||||
# @POST Returns normalized metric value — no SQL injection possible.
|
||||
@tool(args_schema=ExecuteDashboardResultInput)
|
||||
async def execute_dashboard_result(
|
||||
environment_id: str,
|
||||
dashboard_id: int,
|
||||
result_key: str,
|
||||
chart_id: int | None = None,
|
||||
dataset_id: int | None = None,
|
||||
normalized_filters_json: str = "",
|
||||
) -> str:
|
||||
"""Execute a dashboard chart query through Superset-native APIs — no SQL."""
|
||||
logger.reason("Execute dashboard result",
|
||||
payload={"environment_id": environment_id, "dashboard_id": dashboard_id,
|
||||
"chart_id": chart_id, "result_key": result_key},
|
||||
extra={"src": "AgentChat.Tools.ExecuteDashboardResult"})
|
||||
|
||||
# Parse normalized filters if provided
|
||||
import json as _json
|
||||
normalized_filters = None
|
||||
if normalized_filters_json:
|
||||
try:
|
||||
normalized_filters = _json.loads(normalized_filters_json)
|
||||
except _json.JSONDecodeError:
|
||||
return "Error: invalid normalized_filters_json — must be valid JSON"
|
||||
|
||||
body: dict[str, Any] = {
|
||||
"environment_id": environment_id,
|
||||
"dashboard_id": dashboard_id,
|
||||
"result_key": result_key,
|
||||
"normalized_filters": normalized_filters or {"schema_version": 1, "filters": [], "filters_hash": "sha256:empty"},
|
||||
}
|
||||
if chart_id:
|
||||
body["chart_id"] = chart_id
|
||||
if dataset_id:
|
||||
body["dataset_id"] = dataset_id
|
||||
|
||||
resp = await _post("/api/dashboard-testing/queries/execute", json=body)
|
||||
if resp.status_code != 200:
|
||||
logger.explore("Dashboard result execution failed",
|
||||
payload={"status": resp.status_code, "dashboard_id": dashboard_id},
|
||||
error=resp.text[:200],
|
||||
extra={"src": "AgentChat.Tools.ExecuteDashboardResult"})
|
||||
return f"Error {resp.status_code}: {resp.text[:500]}"
|
||||
logger.reflect("Dashboard result executed",
|
||||
payload={"dashboard_id": dashboard_id, "result_key": result_key},
|
||||
extra={"src": "AgentChat.Tools.ExecuteDashboardResult"})
|
||||
return _trim_response(resp.text)
|
||||
# #endregion AgentChat.Tools.ExecuteDashboardResult
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# Tool registry
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
@@ -1292,6 +1392,9 @@ def get_all_tools() -> list:
|
||||
superset_copy_dashboard,
|
||||
superset_create_dataset,
|
||||
superset_format_sql,
|
||||
# 037: Dashboard testing tools
|
||||
inspect_dashboard_query_model,
|
||||
execute_dashboard_result,
|
||||
]
|
||||
# #endregion AgentChat.Tools.GetAll
|
||||
|
||||
|
||||
@@ -40,6 +40,8 @@ from ._user_projection import SupersetUserProjectionMixin
|
||||
from ._sql_lab import SupersetSqlLabMixin
|
||||
from ._saved_queries import SupersetSavedQueriesMixin
|
||||
from ._audit import SupersetAuditMixin
|
||||
# Baseline engine
|
||||
from ._chart_data import SupersetChartDataMixin
|
||||
|
||||
|
||||
# #region Core.Init.SupersetClient [C:3] [TYPE Class]
|
||||
@@ -78,6 +80,7 @@ class SupersetClient(
|
||||
SupersetSqlLabMixin,
|
||||
SupersetSavedQueriesMixin,
|
||||
SupersetAuditMixin,
|
||||
SupersetChartDataMixin,
|
||||
SupersetClientBase,
|
||||
):
|
||||
"""Composed Superset REST API client.
|
||||
|
||||
116
backend/src/core/superset_client/_chart_data.py
Normal file
116
backend/src/core/superset_client/_chart_data.py
Normal file
@@ -0,0 +1,116 @@
|
||||
#region SupersetClient.ChartData.Execute [C:4] [TYPE Module] [SEMANTICS baseline,superset,chart-data,async]
|
||||
# @defgroup Core Superset chart-data POST /api/v1/chart/data mixin.
|
||||
# @LAYER Infrastructure
|
||||
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
|
||||
# @INVARIANT Caller cannot inject SQL, endpoint, datasource, adhoc expression, or unscoped filter.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, cast
|
||||
|
||||
from ..logger import belief_scope, logger as app_logger
|
||||
from ..utils.network import SupersetAPIError
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
|
||||
# @region SupersetClient.ChartData.SupersetChartDataMixin [C:4] [TYPE Class]
|
||||
# @defgroup Core Chart data execution mixin for SupersetClient.
|
||||
# @SIDE_EFFECT Async POST to Superset /api/v1/chart/data.
|
||||
class SupersetChartDataMixin:
|
||||
"""Mixin for executing Superset chart-data queries."""
|
||||
|
||||
# @region SupersetClient.ChartData.ExecuteQuery [C:4] [TYPE Function]
|
||||
# @ingroup Core
|
||||
# @BRIEF Execute a chart-data query for a saved chart with normalized filters.
|
||||
# @PRE Saved chart exists and is accessible.
|
||||
# @POST Returns raw Superset result dict or raises SupersetAPIError.
|
||||
# @SIDE_EFFECT Async POST to Superset /api/v1/chart/data.
|
||||
# @INVARIANT No SQL, raw endpoint, or unscoped filter in the request.
|
||||
async def execute_chart_data(
|
||||
self,
|
||||
chart_id: int,
|
||||
datasource_id: int,
|
||||
datasource_type: str = "table",
|
||||
metrics: list[str | dict] | None = None,
|
||||
groupby: list[str] | None = None,
|
||||
filters: list[dict] | None = None,
|
||||
row_limit: int = 10000,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Execute a saved-chart query through Superset POST /api/v1/chart/data.
|
||||
|
||||
Builds the query_context payload from authoritative chart metadata and
|
||||
normalized filters — never from agent-supplied SQL or raw query_context.
|
||||
|
||||
@PRE chart_id and datasource_id are valid. filters are normalized.
|
||||
@POST Returns {'result': [...], 'query_id': ...} or raises SupersetAPIError.
|
||||
@INVARIANT No sql field in payload.
|
||||
"""
|
||||
with belief_scope("SupersetClient.execute_chart_data", f"chart={chart_id}"):
|
||||
# Build the datasource reference
|
||||
datasource = f"{datasource_id}__{datasource_type}"
|
||||
|
||||
# Build metric specifications
|
||||
metric_specs: list[dict] = []
|
||||
for m in (metrics or []):
|
||||
if isinstance(m, str):
|
||||
metric_specs.append({"label": m})
|
||||
elif isinstance(m, dict):
|
||||
metric_specs.append(m)
|
||||
|
||||
# Build filter specifications from normalized filters
|
||||
adhoc_filters: list[dict] = []
|
||||
for f in (filters or []):
|
||||
clause = f.get("clause", "WHERE")
|
||||
comparator = f.get("value")
|
||||
operator = f.get("operator", "==")
|
||||
column = f.get("column", "")
|
||||
expression_type = f.get("expressionType", "SIMPLE")
|
||||
|
||||
if expression_type == "SIMPLE" and column:
|
||||
adhoc_filters.append({
|
||||
"clause": clause,
|
||||
"comparator": comparator,
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": operator,
|
||||
"subject": column,
|
||||
})
|
||||
|
||||
query_context = {
|
||||
"datasource": {
|
||||
"id": datasource_id,
|
||||
"type": datasource_type,
|
||||
},
|
||||
"queries": [{
|
||||
"datasource": {"id": datasource_id, "type": datasource_type},
|
||||
"metrics": metric_specs,
|
||||
"groupby": groupby or [],
|
||||
"filters": adhoc_filters,
|
||||
"row_limit": min(row_limit, 10000),
|
||||
}],
|
||||
"force": False,
|
||||
"result_type": "full",
|
||||
}
|
||||
|
||||
payload = json.dumps(query_context, default=str)
|
||||
headers = {"Content-Type": "application/json"}
|
||||
|
||||
try:
|
||||
response = await self.client.request(
|
||||
method="POST",
|
||||
endpoint="/chart/data",
|
||||
data=payload,
|
||||
headers=headers,
|
||||
)
|
||||
return cast(dict[str, Any], response)
|
||||
except SupersetAPIError:
|
||||
raise
|
||||
except Exception as e:
|
||||
app_logger.explore("Chart data execution failed", extra={"chart_id": chart_id}, error=str(e))
|
||||
raise SupersetAPIError(f"Chart data execution failed: {e}") from e
|
||||
# @endregion SupersetClient.ChartData.ExecuteQuery
|
||||
|
||||
# @endregion SupersetClient.ChartData.SupersetChartDataMixin
|
||||
|
||||
#endregion SupersetClient.ChartData.Execute
|
||||
180
backend/src/services/dashboard_testing/query_executor.py
Normal file
180
backend/src/services/dashboard_testing/query_executor.py
Normal file
@@ -0,0 +1,180 @@
|
||||
#region BaselineEngine.QueryExecutor.Execute [C:5] [TYPE Function] [SEMANTICS baseline,execution,chart-data,no-sql]
|
||||
# @defgroup BaselineEngine Query executor — runs Superset-native chart/dataset queries without SQL.
|
||||
# @LAYER Service
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient.ChartData.Execute]
|
||||
# @RELATION DEPENDS_ON -> [DashboardTesting.Schemas]
|
||||
# @INVARIANT No SQL, raw endpoint, raw query_context, or adhoc expression fields.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
from src.core.superset_client import SupersetClient
|
||||
from src.core.utils.network import SupersetAPIError
|
||||
from src.schemas.dashboard_testing import (
|
||||
ExecuteQueryRequest,
|
||||
NormalizedValue,
|
||||
ValueKind,
|
||||
Warning,
|
||||
)
|
||||
|
||||
# @region BaselineEngine.QueryExecutor.BuildChartDataFilters [C:3] [TYPE Function]
|
||||
# @ingroup BaselineEngine
|
||||
# @BRIEF Convert NormalizedFilterContext into Superset adhoc-filters for chart-data.
|
||||
def _build_chart_data_filters(normalized_filters) -> list[dict]:
|
||||
"""Map normalized filters to Superset chart-data adhoc filter format."""
|
||||
result: list[dict] = []
|
||||
for nf in normalized_filters.filters:
|
||||
clause = "WHERE"
|
||||
operator_map = {
|
||||
"TEMPORAL_RANGE": "TEMPORAL_RANGE",
|
||||
"IN": "IN",
|
||||
"EQUALS": "==",
|
||||
"NOT_EQUALS": "!=",
|
||||
"GREATER_THAN": ">",
|
||||
"LESS_THAN": "<",
|
||||
"IS_NULL": "IS NULL",
|
||||
}
|
||||
sup_operator = operator_map.get(nf.operator, nf.operator)
|
||||
|
||||
if sup_operator == "TEMPORAL_RANGE":
|
||||
# Temporal range: use from/to values
|
||||
result.append({
|
||||
"clause": clause,
|
||||
"comparator": "",
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "TEMPORAL_RANGE",
|
||||
"subject": nf.column,
|
||||
"from": nf.value.from_ or "",
|
||||
"to": nf.value.to or "",
|
||||
})
|
||||
elif sup_operator == "IN" and nf.value.values:
|
||||
result.append({
|
||||
"clause": clause,
|
||||
"comparator": nf.value.values,
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": "IN",
|
||||
"subject": nf.column,
|
||||
})
|
||||
else:
|
||||
val = nf.value.value if nf.value.value is not None else ""
|
||||
result.append({
|
||||
"clause": clause,
|
||||
"comparator": val,
|
||||
"expressionType": "SIMPLE",
|
||||
"operator": sup_operator,
|
||||
"subject": nf.column,
|
||||
})
|
||||
return result
|
||||
# @endregion BaselineEngine.QueryExecutor.BuildChartDataFilters
|
||||
|
||||
# @region BaselineEngine.QueryExecutor.ExecuteQuery [C:5] [TYPE Function]
|
||||
# @ingroup BaselineEngine
|
||||
# @BRIEF Execute a chart-data query for dashboard testing without SQL injection.
|
||||
# @PRE Request is validated, no SQL fields, chart/dataset exists.
|
||||
# @POST Returns NormalizedValue with source provenance or structured error taxonomy.
|
||||
# @SIDE_EFFECT Async POST to Superset /api/v1/chart/data.
|
||||
# @DATA_CONTRACT ExecuteQueryRequest -> NormalizedValue
|
||||
# @INVARIANT No sql, raw endpoint, or raw query_context in request schema.
|
||||
async def execute_dashboard_query(
|
||||
client: SupersetClient,
|
||||
request: ExecuteQueryRequest,
|
||||
) -> NormalizedValue:
|
||||
"""
|
||||
Execute a Superset-native chart/dataset query for dashboard testing.
|
||||
|
||||
Builds query context from the request's chart_id, result_key, and
|
||||
normalized filters — never from agent-supplied SQL.
|
||||
|
||||
@PRE request.chart_id or request.dataset_id is provided.
|
||||
@POST Returns NormalizedValue with kind, raw value, canonical value, and source provenance.
|
||||
"""
|
||||
warnings: list[Warning] = []
|
||||
|
||||
# Security: explicit rejection of SQL-like fields
|
||||
# The Pydantic schema already uses extra="forbid", so sql cannot appear in the
|
||||
# request body. This is a defense-in-depth check.
|
||||
request_dict = request.model_dump()
|
||||
FORBIDDEN_FIELDS = {"sql", "raw_endpoint", "raw_query_context", "query_context",
|
||||
"adhoc_filters", "adhoc_metrics", "expression"}
|
||||
found_forbidden = set(request_dict.keys()) & FORBIDDEN_FIELDS
|
||||
if found_forbidden:
|
||||
raise ValueError(f"Forbidden fields in request: {found_forbidden}")
|
||||
|
||||
chart_id = request.chart_id
|
||||
dataset_id = request.dataset_id
|
||||
|
||||
if not chart_id and not dataset_id:
|
||||
raise ValueError("Either chart_id or dataset_id must be provided")
|
||||
|
||||
# Build chart-data filters
|
||||
filters = _build_chart_data_filters(request.normalized_filters)
|
||||
|
||||
try:
|
||||
result = await client.execute_chart_data(
|
||||
chart_id=chart_id or 0,
|
||||
datasource_id=dataset_id or 0,
|
||||
datasource_type="table",
|
||||
metrics=[request.result_key],
|
||||
groupby=None,
|
||||
filters=filters,
|
||||
row_limit=request.max_rows,
|
||||
)
|
||||
except SupersetAPIError as e:
|
||||
return NormalizedValue(
|
||||
kind=ValueKind.UNKNOWN,
|
||||
raw_value=None,
|
||||
canonical_value=None,
|
||||
source=f"{request.environment_id}/dashboard/{request.dashboard_id}/chart/{chart_id}",
|
||||
warnings=[Warning(
|
||||
source="execution",
|
||||
resource=f"chart/{chart_id}" if chart_id else f"dataset/{dataset_id}",
|
||||
code=f"SUPERSET_{getattr(e, 'status_code', 'ERROR')}",
|
||||
detail=str(e),
|
||||
)],
|
||||
)
|
||||
|
||||
# Extract result value from Superset response
|
||||
result_data: dict[str, Any] = cast(dict[str, Any], result)
|
||||
actual_result = result_data.get("result", [])
|
||||
query_id = result_data.get("query_id", "")
|
||||
|
||||
# Extract the requested metric from result
|
||||
raw_value: Any = None
|
||||
if isinstance(actual_result, list) and len(actual_result) > 0:
|
||||
first_record = actual_result[0]
|
||||
if isinstance(first_record, dict):
|
||||
data = first_record.get("data", first_record)
|
||||
raw_value = data.get(request.result_key, data)
|
||||
else:
|
||||
raw_value = first_record
|
||||
|
||||
source = f"{request.environment_id}/dashboard/{request.dashboard_id}"
|
||||
if chart_id:
|
||||
source += f"/chart/{chart_id}"
|
||||
if dataset_id:
|
||||
source += f"/dataset/{dataset_id}"
|
||||
source += f"/query/{query_id}"
|
||||
|
||||
# Map Python types to ValueKind
|
||||
kind_map = {
|
||||
type(None): ValueKind.NULL,
|
||||
bool: ValueKind.BOOLEAN,
|
||||
int: ValueKind.INTEGER,
|
||||
float: ValueKind.DECIMAL,
|
||||
str: ValueKind.STRING,
|
||||
list: ValueKind.TABLE,
|
||||
dict: ValueKind.TABLE,
|
||||
}
|
||||
value_kind = kind_map.get(type(raw_value), ValueKind.UNKNOWN)
|
||||
|
||||
return NormalizedValue(
|
||||
kind=value_kind,
|
||||
raw_value=raw_value,
|
||||
canonical_value=str(raw_value) if raw_value is not None else None,
|
||||
source=source,
|
||||
warnings=warnings,
|
||||
)
|
||||
# @endregion BaselineEngine.QueryExecutor.ExecuteQuery
|
||||
|
||||
#endregion BaselineEngine.QueryExecutor.Execute
|
||||
140
backend/tests/services/dashboard_testing/test_query_executor.py
Normal file
140
backend/tests/services/dashboard_testing/test_query_executor.py
Normal file
@@ -0,0 +1,140 @@
|
||||
#region Test.DashboardTesting.QueryExecutor [C:3] [TYPE Module] [SEMANTICS testing,baseline,execution,no-sql]
|
||||
# @defgroup Tests for BaselineEngine.QueryExecutor — Superset-native chart-data execution without SQL.
|
||||
# @LAYER Test
|
||||
# @RELATION VERIFIES -> [BaselineEngine.QueryExecutor.ExecuteQuery]
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from src.schemas.dashboard_testing import (
|
||||
ExecuteQueryRequest, NormalizedFilterContext, NormalizedFilter,
|
||||
FilterValue, NormalizedValue, ValueKind,
|
||||
)
|
||||
from src.services.dashboard_testing.query_executor import execute_dashboard_query
|
||||
|
||||
# @region Test.DashboardTesting.QueryExecutor.NoSQLRejection [C:3] [TYPE Function] [SEMANTICS testing,baseline,no-sql,security]
|
||||
@pytest.mark.asyncio
|
||||
async def test_reject_sql_field_in_request():
|
||||
"""T011: Request with 'sql' field raises ValueError."""
|
||||
# The schema uses extra="forbid", so this should be caught by Pydantic
|
||||
with pytest.raises(Exception): # Pydantic ValidationError or ValueError
|
||||
ExecuteQueryRequest(
|
||||
environment_id="dev",
|
||||
dashboard_id=42,
|
||||
chart_id=128,
|
||||
result_key="sum__revenue",
|
||||
normalized_filters=NormalizedFilterContext(
|
||||
filters=[], filters_hash="sha256:empty"),
|
||||
sql="SELECT * FROM finance", # type: ignore # extra field
|
||||
)
|
||||
# @endregion Test.DashboardTesting.QueryExecutor.NoSQLRejection
|
||||
|
||||
# @region Test.DashboardTesting.QueryExecutor.BasicExecution [C:3] [TYPE Function] [SEMANTICS testing,baseline,execution]
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_scalar_metric():
|
||||
"""T011: Basic chart-data execution returns NormalizedValue."""
|
||||
client = AsyncMock()
|
||||
client.execute_chart_data = AsyncMock(return_value={
|
||||
"result": [{"data": {"sum__revenue": 50000.0}}],
|
||||
"query_id": "q-123",
|
||||
})
|
||||
|
||||
request = ExecuteQueryRequest(
|
||||
environment_id="ss-preprod",
|
||||
dashboard_id=42,
|
||||
chart_id=128,
|
||||
result_key="sum__revenue",
|
||||
normalized_filters=NormalizedFilterContext(
|
||||
filters=[
|
||||
NormalizedFilter(
|
||||
filter_id="NATIVE_FILTER-date",
|
||||
dataset_id=77,
|
||||
column="business_date",
|
||||
operator="TEMPORAL_RANGE",
|
||||
value=FilterValue(from_="2026-05-29", to="2026-05-29"),
|
||||
target_chart_ids=[128],
|
||||
)
|
||||
],
|
||||
filters_hash="sha256:test",
|
||||
),
|
||||
)
|
||||
|
||||
result = await execute_dashboard_query(client, request)
|
||||
|
||||
assert isinstance(result, NormalizedValue)
|
||||
assert result.raw_value is not None
|
||||
assert result.source # must have provenance
|
||||
# @endregion Test.DashboardTesting.QueryExecutor.BasicExecution
|
||||
|
||||
# @region Test.DashboardTesting.QueryExecutor.SupersetErrorTaxonomy [C:3] [TYPE Function] [SEMANTICS testing,baseline,error-taxonomy]
|
||||
@pytest.mark.asyncio
|
||||
async def test_superset_error_preserved():
|
||||
"""T011: Superset 403/5xx errors are preserved in result warnings."""
|
||||
from src.core.utils.network import SupersetAPIError
|
||||
|
||||
client = AsyncMock()
|
||||
client.execute_chart_data = AsyncMock(
|
||||
side_effect=SupersetAPIError("Forbidden", status_code=403)
|
||||
)
|
||||
|
||||
request = ExecuteQueryRequest(
|
||||
environment_id="dev",
|
||||
dashboard_id=42,
|
||||
chart_id=128,
|
||||
result_key="sum__revenue",
|
||||
normalized_filters=NormalizedFilterContext(
|
||||
filters=[], filters_hash="sha256:empty",
|
||||
),
|
||||
)
|
||||
|
||||
result = await execute_dashboard_query(client, request)
|
||||
|
||||
assert result.kind == ValueKind.UNKNOWN
|
||||
assert len(result.warnings) > 0
|
||||
assert "SUPERSET" in result.warnings[0].code
|
||||
# @endregion Test.DashboardTesting.QueryExecutor.SupersetErrorTaxonomy
|
||||
|
||||
# @region Test.DashboardTesting.QueryExecutor.TemporalFilterMapping [C:3] [TYPE Function] [SEMANTICS testing,baseline,temporal,filter]
|
||||
@pytest.mark.asyncio
|
||||
async def test_temporal_filter_mapped_correctly():
|
||||
"""T011: TEMPORAL_RANGE filter is correctly mapped to chart-data format."""
|
||||
client = AsyncMock()
|
||||
execute_result: dict = {"result": [{"data": {"count": 150}}], "query_id": "q-456"}
|
||||
client.execute_chart_data = AsyncMock(return_value=execute_result)
|
||||
|
||||
request = ExecuteQueryRequest(
|
||||
environment_id="ss-preprod",
|
||||
dashboard_id=42,
|
||||
chart_id=128,
|
||||
result_key="count",
|
||||
normalized_filters=NormalizedFilterContext(
|
||||
filters=[
|
||||
NormalizedFilter(
|
||||
filter_id="NATIVE_FILTER-date",
|
||||
dataset_id=77,
|
||||
column="business_date",
|
||||
operator="TEMPORAL_RANGE",
|
||||
value=FilterValue(from_="2026-05-01", to="2026-05-31", inclusive=True),
|
||||
target_chart_ids=[128],
|
||||
)
|
||||
],
|
||||
filters_hash="sha256:test",
|
||||
),
|
||||
)
|
||||
|
||||
result = await execute_dashboard_query(client, request)
|
||||
assert result.raw_value == 150
|
||||
|
||||
# Verify the filter was passed correctly to execute_chart_data
|
||||
call_args = client.execute_chart_data.call_args
|
||||
filters = call_args.kwargs.get("filters", [])
|
||||
assert len(filters) == 1
|
||||
assert filters[0]["operator"] == "TEMPORAL_RANGE"
|
||||
assert filters[0]["from"] == "2026-05-01"
|
||||
assert filters[0]["to"] == "2026-05-31"
|
||||
# @endregion Test.DashboardTesting.QueryExecutor.TemporalFilterMapping
|
||||
|
||||
#endregion Test.DashboardTesting.QueryExecutor
|
||||
@@ -3,19 +3,19 @@
|
||||
|
||||
## Phase 1 — Fixtures and DTO Foundation
|
||||
|
||||
- [ ] T001 Create canonical dashboard/chart/dataset/native-filter Superset fixtures under specs/037-superset-baseline-engine/fixtures/superset/.
|
||||
- [ ] T002 [P] Create scalar, percent, date, table, empty, malformed, and locale result fixtures under specs/037-superset-baseline-engine/fixtures/results/.
|
||||
- [ ] T003 [P] Create valid/invalid/stale baseline catalog fixtures under specs/037-superset-baseline-engine/fixtures/baselines/.
|
||||
- [ ] T004 Materialize fixtures into backend/tests/fixtures/dashboard_testing/.
|
||||
- [ ] T005 Implement extra-forbid DTOs from contracts/dashboard-testing.openapi.yaml in backend/src/schemas/dashboard_testing.py.
|
||||
- [x] T001 Create canonical dashboard/chart/dataset/native-filter Superset fixtures under specs/037-superset-baseline-engine/fixtures/superset/.
|
||||
- [x] T002 [P] Create scalar, percent, date, table, empty, malformed, and locale result fixtures under specs/037-superset-baseline-engine/fixtures/results/.
|
||||
- [x] T003 [P] Create valid/invalid/stale baseline catalog fixtures under specs/037-superset-baseline-engine/fixtures/baselines/.
|
||||
- [x] T004 Materialize fixtures into backend/tests/fixtures/dashboard_testing/.
|
||||
- [x] T005 Implement extra-forbid DTOs from contracts/dashboard-testing.openapi.yaml in backend/src/schemas/dashboard_testing.py.
|
||||
|
||||
## Phase 2 — US1 Inspect Dashboard Query Model
|
||||
|
||||
- [ ] T006 [US1] Write failing deterministic inspection tests in backend/tests/services/dashboard_testing/test_query_model.py.
|
||||
- [ ] T007 [US1] Write failing filter scope/type/hash tests in backend/tests/services/dashboard_testing/test_filters.py.
|
||||
- [ ] T008 [US1] Implement backend/src/services/dashboard_testing/query_model.py using authoritative SupersetClient metadata.
|
||||
- [ ] T009 [US1] Implement backend/src/services/dashboard_testing/filters.py with canonical typed values and deterministic hashes.
|
||||
- [ ] T010 [US1] Add fingerprint helpers in backend/src/services/dashboard_testing/fingerprints.py and cover metadata order invariance.
|
||||
- [x] T006 [US1] Write failing deterministic inspection tests in backend/tests/services/dashboard_testing/test_query_model.py.
|
||||
- [x] T007 [US1] Write failing filter scope/type/hash tests in backend/tests/services/dashboard_testing/test_filters.py.
|
||||
- [x] T008 [US1] Implement backend/src/services/dashboard_testing/query_model.py using authoritative SupersetClient metadata.
|
||||
- [x] T009 [US1] Implement backend/src/services/dashboard_testing/filters.py with canonical typed values and deterministic hashes.
|
||||
- [x] T010 [US1] Add fingerprint helpers in backend/src/services/dashboard_testing/fingerprints.py and cover metadata order invariance.
|
||||
|
||||
**Checkpoint**: Fixture dashboards yield byte-stable models and correct chart/filter scopes.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user