busya
a32ca0631b
- Authoritative candidate capture with server-issued artifacts and raw-byte
immutability hashing (source_response_hash server-owned)
- Closed-period lifecycle: request-hash bound approvals, persisted closure
immutability violations, byte-for-byte catalog stability on reclosure
- Verification runs: persisted VerificationRun model + FK migration,
publish gate (block_publish), scheduled observability runs (02:00 UTC)
- FR-013 baseline inheritance: prior_release_id migration, plan_inheritance/
execute_inheritance classification and re-extraction, API endpoints
- Visual executor bound to release-deployment environment; caller mismatch
rejected; visual SSIM/reconciliation modules
- Query execution decomposed: envelope/model/executor split, no direct SQL
- AgentRun approvals extracted to submodule; evidence adapter; _utils
- Dashboard testing service decomposed into 30+ modules (all <400 LOC)
- Five Feature-037 agent tools with permission guards (tools_037.py)
- API readiness endpoint; Alembic env/migrations; test fixture repos
- Specs 036/037 contracts, openapi.yaml, schema.json, tasks/traceability
updated; semantic index rebuilt with 0 parse warnings
- Fix ADR-0003 parser ambiguity: remove [DEF🆔ADR] prose example
- Add axiom-mcp-agent-feedback.md: agent findings for MCP rework plan
- Tests: 298 service + 1464 API + 45 agent passing; ruff clean
407 lines
16 KiB
Python
407 lines
16 KiB
Python
# #region Test.DashboardTesting.QueryExecutor [C:3] [TYPE Module] [SEMANTICS testing,baseline,execution,no-sql]
|
|
# @defgroup Tests for BaselineEngine.QueryExecutor — Superset-native chart-data execution without SQL.
|
|
# @LAYER Test
|
|
# @RELATION VERIFIES -> [BaselineEngine.QueryExecutor.ExecuteQuery]
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import pytest
|
|
from unittest.mock import AsyncMock
|
|
|
|
from src.core.superset_client._chart_data import ChartDataResponse
|
|
from src.schemas.dashboard_testing import (
|
|
ChartQueryModel,
|
|
ColumnInfo,
|
|
DashboardQueryModel,
|
|
DatasetQueryModel,
|
|
ExecuteQueryRequest,
|
|
FilterTarget,
|
|
FilterValue,
|
|
MetricDescriptor,
|
|
NativeFilterModel,
|
|
NormalizedFilter,
|
|
NormalizedFilterContext,
|
|
NormalizedValue,
|
|
ValueKind,
|
|
)
|
|
from src.services.dashboard_testing.query_executor import execute_dashboard_query
|
|
|
|
# ── Helpers ────────────────────────────────────────────────────────
|
|
|
|
|
|
def _chart_data_response(result_dict: dict) -> ChartDataResponse:
|
|
"""Build a ChartDataResponse from a result dict (simulates httpx raw_response)."""
|
|
raw = json.dumps(result_dict, sort_keys=True, default=str).encode("utf-8")
|
|
import hashlib
|
|
return ChartDataResponse(
|
|
parsed=result_dict,
|
|
raw_bytes=raw,
|
|
source_response_hash=hashlib.sha256(raw).hexdigest(),
|
|
)
|
|
|
|
|
|
# #region Test.DashboardTesting.QueryExecutor.NoSQLRejection [C:3] [TYPE Function] [SEMANTICS testing,baseline,no-sql,security]
|
|
@pytest.mark.asyncio
|
|
async def test_reject_sql_field_in_request():
|
|
"""T011: Request with 'sql' field raises ValueError."""
|
|
# The schema uses extra="forbid", so this should be caught by Pydantic
|
|
with pytest.raises(ValueError): # Pydantic ValidationError or ValueError
|
|
ExecuteQueryRequest(
|
|
environment_id="dev",
|
|
dashboard_id=42,
|
|
chart_id=128,
|
|
result_key="sum__revenue",
|
|
normalized_filters=NormalizedFilterContext(
|
|
filters=[], filters_hash="sha256:empty"),
|
|
sql="SELECT * FROM finance", # type: ignore # extra field
|
|
)
|
|
# #endregion Test.DashboardTesting.QueryExecutor.NoSQLRejection
|
|
|
|
# #region Test.DashboardTesting.QueryExecutor.BasicExecution [C:3] [TYPE Function] [SEMANTICS testing,baseline,execution]
|
|
@pytest.mark.asyncio
|
|
async def test_execute_scalar_metric():
|
|
"""T011: Basic chart-data execution returns NormalizedValue."""
|
|
client = AsyncMock()
|
|
client.execute_chart_data_raw = AsyncMock(return_value=_chart_data_response({
|
|
"result": [{"data": {"sum__revenue": 50000.0}}],
|
|
"query_id": "q-123",
|
|
}))
|
|
|
|
request = ExecuteQueryRequest(
|
|
environment_id="ss-preprod",
|
|
dashboard_id=42,
|
|
chart_id=128,
|
|
result_key="sum__revenue",
|
|
normalized_filters=NormalizedFilterContext(
|
|
filters=[
|
|
NormalizedFilter(
|
|
filter_id="NATIVE_FILTER-date",
|
|
dataset_id=77,
|
|
column="business_date",
|
|
operator="TEMPORAL_RANGE",
|
|
value=FilterValue(from_="2026-05-29", to="2026-05-29"),
|
|
target_chart_ids=[128],
|
|
)
|
|
],
|
|
filters_hash="sha256:test",
|
|
),
|
|
)
|
|
|
|
result = await execute_dashboard_query(client, request)
|
|
|
|
assert isinstance(result, NormalizedValue)
|
|
assert result.raw_value is not None
|
|
assert result.source # must have provenance
|
|
# #endregion Test.DashboardTesting.QueryExecutor.BasicExecution
|
|
|
|
# #region Test.DashboardTesting.QueryExecutor.SupersetErrorTaxonomy [C:3] [TYPE Function] [SEMANTICS testing,baseline,error-taxonomy]
|
|
@pytest.mark.asyncio
|
|
async def test_superset_error_preserved():
|
|
"""T011: Superset 403/5xx errors are preserved in result warnings."""
|
|
from src.core.utils.network import SupersetAPIError
|
|
|
|
client = AsyncMock()
|
|
client.execute_chart_data_raw = AsyncMock(
|
|
side_effect=SupersetAPIError("Forbidden", status_code=403)
|
|
)
|
|
|
|
request = ExecuteQueryRequest(
|
|
environment_id="dev",
|
|
dashboard_id=42,
|
|
chart_id=128,
|
|
result_key="sum__revenue",
|
|
normalized_filters=NormalizedFilterContext(
|
|
filters=[], filters_hash="sha256:empty",
|
|
),
|
|
)
|
|
|
|
result = await execute_dashboard_query(client, request)
|
|
|
|
assert result.kind == ValueKind.UNKNOWN
|
|
assert len(result.warnings) > 0
|
|
assert "SUPERSET" in result.warnings[0].code
|
|
# #endregion Test.DashboardTesting.QueryExecutor.SupersetErrorTaxonomy
|
|
|
|
# #region Test.DashboardTesting.QueryExecutor.TemporalFilterMapping [C:3] [TYPE Function] [SEMANTICS testing,baseline,temporal,filter]
|
|
@pytest.mark.asyncio
|
|
async def test_temporal_filter_mapped_correctly():
|
|
"""T011: TEMPORAL_RANGE filter is correctly mapped to chart-data format."""
|
|
client = AsyncMock()
|
|
execute_result: dict = {"result": [{"data": {"count": 150}}], "query_id": "q-456"}
|
|
client.execute_chart_data_raw = AsyncMock(return_value=_chart_data_response(execute_result))
|
|
|
|
request = ExecuteQueryRequest(
|
|
environment_id="ss-preprod",
|
|
dashboard_id=42,
|
|
chart_id=128,
|
|
result_key="count",
|
|
normalized_filters=NormalizedFilterContext(
|
|
filters=[
|
|
NormalizedFilter(
|
|
filter_id="NATIVE_FILTER-date",
|
|
dataset_id=77,
|
|
column="business_date",
|
|
operator="TEMPORAL_RANGE",
|
|
value=FilterValue(from_="2026-05-01", to="2026-05-31", inclusive=True),
|
|
target_chart_ids=[128],
|
|
)
|
|
],
|
|
filters_hash="sha256:test",
|
|
),
|
|
)
|
|
|
|
result = await execute_dashboard_query(client, request)
|
|
assert result.raw_value == 150
|
|
|
|
# Verify the filter was passed correctly to execute_chart_data_raw
|
|
call_args = client.execute_chart_data_raw.call_args
|
|
filters = call_args.kwargs.get("filters", [])
|
|
assert len(filters) == 1
|
|
assert filters[0]["operator"] == "TEMPORAL_RANGE"
|
|
assert filters[0]["from"] == "2026-05-01"
|
|
assert filters[0]["to"] == "2026-05-31"
|
|
# #endregion Test.DashboardTesting.QueryExecutor.TemporalFilterMapping
|
|
|
|
# ── Authoritative model test helpers ─────────────────────────────────
|
|
|
|
def _make_basic_query_model(
|
|
chart_ids: list[int] | None = None,
|
|
filter_targets: dict[str, list[int]] | None = None,
|
|
fingerprint: str = "sha256:test_fingerprint",
|
|
) -> DashboardQueryModel:
|
|
"""Build a minimal query model for executor tests."""
|
|
if chart_ids is None:
|
|
chart_ids = [128, 129]
|
|
|
|
charts = [
|
|
ChartQueryModel(
|
|
chart_id=cid,
|
|
slice_name=f"Chart {cid}",
|
|
viz_type="bar",
|
|
dataset_id=77,
|
|
dataset_name="public.finance_transactions",
|
|
metrics=[MetricDescriptor(metric_name="sum__revenue", label="SUM(revenue)", expression_type="SIMPLE")],
|
|
applied_filter_ids=[fid for fid, targets in (filter_targets or {}).items() if cid in targets],
|
|
excluded_filter_ids=[],
|
|
)
|
|
for cid in chart_ids
|
|
]
|
|
|
|
if filter_targets is None:
|
|
filter_targets = {"NATIVE_FILTER-date": [128, 129], "NATIVE_FILTER-region": [128]}
|
|
|
|
native_filters = [
|
|
NativeFilterModel(
|
|
filter_id=fid,
|
|
filter_type="NATIVE_FILTER",
|
|
name=fid.replace("NATIVE_FILTER-", "").replace("-", " ").title(),
|
|
column="business_date" if "date" in fid else "business_region",
|
|
dataset_id=77,
|
|
type="DATE" if "date" in fid else "STRING",
|
|
targets=[FilterTarget(chart_id=cid, dataset_id=77) for cid in targets],
|
|
)
|
|
for fid, targets in filter_targets.items()
|
|
]
|
|
|
|
return DashboardQueryModel(
|
|
environment_id="ss-preprod",
|
|
dashboard_id=42,
|
|
title="FI-0080 Finance Overview",
|
|
charts=charts,
|
|
datasets=[
|
|
DatasetQueryModel(
|
|
dataset_id=77,
|
|
dataset_name="public.finance_transactions",
|
|
columns=[
|
|
ColumnInfo(column_name="business_date", type="DATE", groupby=True, filterable=True),
|
|
ColumnInfo(column_name="business_region", type="STRING", groupby=True, filterable=True),
|
|
],
|
|
)
|
|
],
|
|
native_filters=native_filters,
|
|
query_model_fingerprint=fingerprint,
|
|
)
|
|
|
|
# ── Authoritative model tests ────────────────────────────────────────
|
|
|
|
# #region Test.DashboardTesting.QueryExecutor.AuthBasicExecution [C:3] [TYPE Function] [SEMANTICS testing,baseline,execution,authoritative]
|
|
@pytest.mark.asyncio
|
|
async def test_execute_with_authoritative_model():
|
|
"""T037: Execute with authoritative DashboardQueryModel — passes validation, returns value."""
|
|
client = AsyncMock()
|
|
client.execute_chart_data_raw = AsyncMock(return_value=_chart_data_response({
|
|
"result": [{"data": {"sum__revenue": 50000.0}}],
|
|
"query_id": "q-123",
|
|
}))
|
|
|
|
query_model = _make_basic_query_model()
|
|
request = ExecuteQueryRequest(
|
|
environment_id="ss-preprod",
|
|
dashboard_id=42,
|
|
chart_id=128,
|
|
result_key="sum__revenue",
|
|
normalized_filters=NormalizedFilterContext(
|
|
filters=[
|
|
NormalizedFilter(
|
|
filter_id="NATIVE_FILTER-date",
|
|
dataset_id=77,
|
|
column="business_date",
|
|
operator="TEMPORAL_RANGE",
|
|
value=FilterValue(from_="2026-05-29", to="2026-05-29"),
|
|
target_chart_ids=[128],
|
|
)
|
|
],
|
|
filters_hash="sha256:test",
|
|
),
|
|
query_model_fingerprint="sha256:test_fingerprint",
|
|
)
|
|
|
|
result = await execute_dashboard_query(client, request, query_model)
|
|
assert isinstance(result, NormalizedValue)
|
|
assert result.raw_value == 50000.0
|
|
# #endregion Test.DashboardTesting.QueryExecutor.AuthBasicExecution
|
|
|
|
# #region Test.DashboardTesting.QueryExecutor.AuthRejectChartNotInDashboard [C:3] [TYPE Function] [SEMANTICS testing,baseline,execution,authoritative,rejection]
|
|
@pytest.mark.asyncio
|
|
async def test_reject_chart_not_in_dashboard():
|
|
"""T037: Chart not in authoritative model raises ValueError."""
|
|
client = AsyncMock()
|
|
query_model = _make_basic_query_model(chart_ids=[128])
|
|
request = ExecuteQueryRequest(
|
|
environment_id="ss-preprod",
|
|
dashboard_id=42,
|
|
chart_id=999,
|
|
result_key="sum__revenue",
|
|
normalized_filters=NormalizedFilterContext(
|
|
filters=[], filters_hash="sha256:empty",
|
|
),
|
|
)
|
|
with pytest.raises(ValueError, match="not found in dashboard"):
|
|
await execute_dashboard_query(client, request, query_model)
|
|
# #endregion Test.DashboardTesting.QueryExecutor.AuthRejectChartNotInDashboard
|
|
|
|
# #region Test.DashboardTesting.QueryExecutor.AuthRejectDatasetNotInDashboard [C:3] [TYPE Function] [SEMANTICS testing,baseline,execution,authoritative,rejection]
|
|
@pytest.mark.asyncio
|
|
async def test_reject_dataset_not_in_dashboard():
|
|
"""T037: Dataset not in authoritative model raises ValueError."""
|
|
client = AsyncMock()
|
|
query_model = _make_basic_query_model(chart_ids=[128])
|
|
request = ExecuteQueryRequest(
|
|
environment_id="ss-preprod",
|
|
dashboard_id=42,
|
|
dataset_id=999,
|
|
result_key="sum__revenue",
|
|
normalized_filters=NormalizedFilterContext(
|
|
filters=[], filters_hash="sha256:empty",
|
|
),
|
|
)
|
|
with pytest.raises(ValueError, match="not found in dashboard"):
|
|
await execute_dashboard_query(client, request, query_model)
|
|
# #endregion Test.DashboardTesting.QueryExecutor.AuthRejectDatasetNotInDashboard
|
|
|
|
# #region Test.DashboardTesting.QueryExecutor.AuthRejectFingerprintMismatch [C:3] [TYPE Function] [SEMANTICS testing,baseline,execution,authoritative,fingerprint]
|
|
@pytest.mark.asyncio
|
|
async def test_reject_fingerprint_mismatch():
|
|
"""T037: query_model_fingerprint mismatch raises ValueError."""
|
|
client = AsyncMock()
|
|
query_model = _make_basic_query_model(fingerprint="sha256:authoritative_fp")
|
|
request = ExecuteQueryRequest(
|
|
environment_id="ss-preprod",
|
|
dashboard_id=42,
|
|
chart_id=128,
|
|
result_key="sum__revenue",
|
|
normalized_filters=NormalizedFilterContext(
|
|
filters=[], filters_hash="sha256:empty",
|
|
),
|
|
query_model_fingerprint="sha256:wrong_fingerprint",
|
|
)
|
|
with pytest.raises(ValueError, match="fingerprint mismatch"):
|
|
await execute_dashboard_query(client, request, query_model)
|
|
# #endregion Test.DashboardTesting.QueryExecutor.AuthRejectFingerprintMismatch
|
|
|
|
# #region Test.DashboardTesting.QueryExecutor.AuthFilterScoping [C:3] [TYPE Function] [SEMANTICS testing,baseline,execution,authoritative,filter-scope]
|
|
@pytest.mark.asyncio
|
|
async def test_filter_scoping_only_targeted_chart():
|
|
"""T037: Only filters targeting the requested chart are passed to execute_chart_data_raw."""
|
|
client = AsyncMock()
|
|
client.execute_chart_data_raw = AsyncMock(return_value=_chart_data_response({
|
|
"result": [{"data": {"sum__revenue": 50000.0}}],
|
|
"query_id": "q-123",
|
|
}))
|
|
|
|
query_model = _make_basic_query_model(
|
|
chart_ids=[128, 129],
|
|
filter_targets={
|
|
"NATIVE_FILTER-date": [128, 129], # scoped to both
|
|
"NATIVE_FILTER-region": [128], # scoped only to chart 128
|
|
},
|
|
)
|
|
|
|
request = ExecuteQueryRequest(
|
|
environment_id="ss-preprod",
|
|
dashboard_id=42,
|
|
chart_id=129, # querying chart 129
|
|
result_key="sum__revenue",
|
|
normalized_filters=NormalizedFilterContext(
|
|
filters=[
|
|
NormalizedFilter(
|
|
filter_id="NATIVE_FILTER-date",
|
|
dataset_id=77, column="business_date",
|
|
operator="TEMPORAL_RANGE",
|
|
value=FilterValue(from_="2026-05-29", to="2026-05-29"),
|
|
target_chart_ids=[128, 129],
|
|
),
|
|
NormalizedFilter(
|
|
filter_id="NATIVE_FILTER-region",
|
|
dataset_id=77, column="business_region",
|
|
operator="IN",
|
|
value=FilterValue(values=["West"]),
|
|
target_chart_ids=[128],
|
|
),
|
|
],
|
|
filters_hash="sha256:test",
|
|
),
|
|
query_model_fingerprint="sha256:test_fingerprint",
|
|
)
|
|
|
|
result = await execute_dashboard_query(client, request, query_model)
|
|
assert isinstance(result, NormalizedValue)
|
|
|
|
# verify only the date filter (scoped to 129) was passed; region filter excluded
|
|
call_args = client.execute_chart_data_raw.call_args
|
|
passed_filters = call_args.kwargs.get("filters", [])
|
|
assert len(passed_filters) == 1
|
|
assert passed_filters[0]["operator"] == "TEMPORAL_RANGE"
|
|
# #endregion Test.DashboardTesting.QueryExecutor.AuthFilterScoping
|
|
|
|
# #region Test.DashboardTesting.QueryExecutor.AuthUnknownMetricWarning [C:3] [TYPE Function] [SEMANTICS testing,baseline,execution,authoritative,metric-warning]
|
|
@pytest.mark.asyncio
|
|
async def test_unknown_metric_produces_warning():
|
|
"""T037: result_key not a known metric produces Warning but does not block execution."""
|
|
client = AsyncMock()
|
|
client.execute_chart_data_raw = AsyncMock(return_value=_chart_data_response({
|
|
"result": [{"data": {"unknown_key": 100}}],
|
|
"query_id": "q-456",
|
|
}))
|
|
|
|
query_model = _make_basic_query_model(chart_ids=[128])
|
|
request = ExecuteQueryRequest(
|
|
environment_id="ss-preprod",
|
|
dashboard_id=42,
|
|
chart_id=128,
|
|
result_key="unknown_key", # not in chart's known metrics
|
|
normalized_filters=NormalizedFilterContext(
|
|
filters=[], filters_hash="sha256:empty",
|
|
),
|
|
query_model_fingerprint="sha256:test_fingerprint",
|
|
)
|
|
|
|
result = await execute_dashboard_query(client, request, query_model)
|
|
assert isinstance(result, NormalizedValue)
|
|
# Execution proceeds but a warning is emitted
|
|
assert any(w.code == "UNKNOWN_METRIC" for w in result.warnings)
|
|
# #endregion Test.DashboardTesting.QueryExecutor.AuthUnknownMetricWarning
|
|
|
|
# #endregion Test.DashboardTesting.QueryExecutor
|