Files
ss-tools/backend/tests/api/test_dashboard_testing_openapi.py

498 lines
26 KiB
Python

# #region Test.Api.DashboardTesting.OpenAPI [C:3] [TYPE Module] [SEMANTICS testing,api,dashboard-testing,openapi,validation,alignment]
# @defgroup OpenAPI validation and alignment tests for dashboard-testing API.
# @LAYER Test
# @RELATION VERIFIES -> [Api.DashboardTesting]
# @TEST_EDGE: all_routes_documented -> Every route in the router appears in OpenAPI paths.
# @TEST_EDGE: approval_fields_aligned -> OpenAPI approval-gate body requires agent_run_id + release_version + release_commit_hash.
# @TEST_EDGE: status_codes_aligned -> 201 for creation endpoints, 422 for validation errors.
# @TEST_EDGE: no_undocumented_routes -> No route diverges from the OpenAPI spec.
from __future__ import annotations
import pytest
import re as _re
from typing import ClassVar
from fastapi.testclient import TestClient
from src.app import app
from src.dependencies import get_current_user
from src.models.auth import Role, User
# ── Fixtures ──
@pytest.fixture
def openapi_schema():
"""Extract the OpenAPI schema from the FastAPI app."""
return app.openapi()
@pytest.fixture
def client():
"""TestClient for API calls."""
admin_role = Role(id="admin-role-openapi", name="Admin", is_admin=True)
mock_user = User(id="test-user-openapi", username="tester", email="tester@test.com")
mock_user.roles = [admin_role]
app.dependency_overrides[get_current_user] = lambda: mock_user
yield TestClient(app)
app.dependency_overrides.pop(get_current_user, None)
# ── Shared helpers ──
def _normalize_path(p: str) -> str:
"""Convert {camelCase} to {snake_case} in path params for comparison."""
def _to_snake(m: _re.Match) -> str:
name = m.group(1)
snake = _re.sub(r'(?<!^)(?=[A-Z])', '_', name).lower()
return '{' + snake + '}'
return _re.sub(r'\{(\w+)\}', _to_snake, p)
def _resolve_ref(schema: dict, openapi_schema: dict) -> dict:
"""Resolve a $ref pointer in a schema."""
ref = schema.get("$ref", "")
if ref:
components = openapi_schema.get("components", {})
ref_name = ref.split("/")[-1]
return components.get("schemas", {}).get(ref_name, {})
return schema
# #region Test.Api.DashboardTesting.OpenAPI.Alignment [C:2] [TYPE Class] [SEMANTICS test,api,openapi,alignment]
class TestOpenAPIAlignment:
"""OpenAPI schema alignment — routes, fields, and status codes match the implementation."""
_EXPECTED_PATHS: ClassVar = {
"/api/dashboard-testing/query-model": {"get"},
"/api/dashboard-testing/filters/normalize": {"post"},
"/api/dashboard-testing/queries/execute": {"post"},
"/api/dashboard-testing/comparisons": {"post"},
"/api/dashboard-testing/baselines": {"get"},
"/api/dashboard-testing/baseline-candidates": {"post"},
"/api/dashboard-testing/baseline-candidates/capture": {"post"},
"/api/dashboard-testing/baseline-candidates/{candidate_id}/approval-gate": {"post"},
"/api/dashboard-testing/baseline-candidates/{candidate_id}/approval-gate/{gate_id}/decide": {"post"},
"/api/dashboard-testing/baseline-candidates/{candidate_id}/approval-gate/{gate_id}/consume": {"post"},
"/api/dashboard-testing/structure-diff": {"post"},
"/api/dashboard-testing/structure-snapshot/capture": {"post"},
"/api/dashboard-testing/structure-snapshot/diff": {"post"},
"/api/dashboard-testing/verification-runs": {"post"},
"/api/dashboard-testing/verification/history": {"get"},
"/api/dashboard-testing/verification/{run_id}": {"get"},
"/api/dashboard-testing/inheritance/plan": {"post"},
"/api/dashboard-testing/inheritance/execute": {"post"},
"/api/dashboard-testing/scenarios/compile": {"post"},
"/api/dashboard-testing/scenarios/validate": {"post"},
"/api/dashboard-testing/scenarios": {"get", "post"},
"/api/dashboard-testing/scenarios/{scenario_id}": {"get"},
"/api/dashboard-testing/scenarios/{scenario_id}/revisions": {"get", "post"},
"/api/dashboard-testing/scenarios/{scenario_id}/revisions/{revision_id}": {"get"},
"/api/dashboard-testing/scenarios/{scenario_id}/revisions/{rev_a}/diff/{rev_b}": {"get"},
"/api/dashboard-testing/scenarios/{scenario_id}/transition": {"post"},
"/api/dashboard-testing/scenarios/{scenario_id}/clone": {"post"},
"/api/dashboard-testing/scenarios/{scenario_id}/archive": {"post"},
"/api/dashboard-testing/scenarios/{scenario_id}/restore": {"post"},
"/api/dashboard-testing/scenarios/{scenario_id}/health": {"get"},
"/api/dashboard-testing/scenarios/{scenario_id}/edit": {"get"},
"/api/dashboard-testing/scenarios/{scenario_id}/edits/apply": {"post"},
"/api/dashboard-testing/scenarios/{scenario_id}/edits/agent-propose": {"post"},
"/api/dashboard-testing/scenarios/{scenario_id}/edits/proposals/{proposal_id}/save": {"post"},
"/api/dashboard-testing/scenarios/{scenario_id}/edits/save": {"post"},
"/api/dashboard-testing/scenarios/{scenario_id}/revalidate": {"post"},
"/api/dashboard-testing/scenarios/{scenario_id}/capture": {"post"},
"/api/dashboard-testing/scenarios/{scenario_id}/disposition": {"post"},
"/api/dashboard-testing/scenarios/{scenario_id}/draft-pack": {"post"},
"/api/dashboard-testing/scenarios/{scenario_id}/resolve": {"post"},
"/api/dashboard-testing/scenarios/{scenario_id}/vlm": {"post"},
}
# #region Test.Api.DashboardTesting.OpenAPI.Alignment.TestAllRoutesDocumented [C:2] [TYPE Function]
# @BRIEF All dashboard-testing routes appear in the OpenAPI paths.
# @TEST_EDGE: all_routes_documented -> Every route in the router appears in OpenAPI paths.
def test_all_routes_documented(self, openapi_schema):
"""T037: Every dashboard-testing route appears in the OpenAPI schema."""
paths = openapi_schema.get("paths", {})
for path, expected_methods in self._EXPECTED_PATHS.items():
assert path in paths, f"Path {path} missing from OpenAPI schema"
path_item = paths[path]
for method in expected_methods:
assert method in path_item, (
f"Method {method.upper()} missing for path {path} "
f"in OpenAPI schema. Available: {list(path_item.keys())}"
)
# #endregion Test.Api.DashboardTesting.OpenAPI.Alignment.TestAllRoutesDocumented
# #region Test.Api.DashboardTesting.OpenAPI.Alignment.TestNoUndocumentedRoutes [C:2] [TYPE Function]
# @BRIEF No undocumented dashboard-testing routes exist in OpenAPI schema.
# @TEST_EDGE: no_undocumented_routes -> Only expected routes are present.
def test_no_undocumented_dashboard_routes(self, openapi_schema):
"""T037: Fail if any unexpected dashboard-testing route is in the schema."""
paths = openapi_schema.get("paths", {})
dashboard_paths = {k: v for k, v in paths.items() if k.startswith("/api/dashboard-testing")}
for path in dashboard_paths:
assert path in self._EXPECTED_PATHS, (
f"Unexpected path {path} found in OpenAPI schema"
)
# #endregion Test.Api.DashboardTesting.OpenAPI.Alignment.TestNoUndocumentedRoutes
# #region Test.Api.DashboardTesting.OpenAPI.Alignment.TestApprovalGateRequestFields [C:2] [TYPE Function]
# @BRIEF OpenAPI approval-gate request body requires agent_run_id, release_version, release_commit_hash.
# @TEST_EDGE: approval_fields_aligned -> Required fields match the ApprovalGateRequest schema.
def test_approval_gate_required_fields(self, openapi_schema):
"""T037: Approval-gate body requires agent_run_id, release_version, release_commit_hash."""
components = openapi_schema.get("components", {})
schemas = components.get("schemas", {})
gate_schema = schemas.get("ApprovalGateRequest", {})
required = gate_schema.get("required", [])
assert "agent_run_id" in required, (
f"agent_run_id must be required in ApprovalGateRequest. "
f"Required fields: {required}"
)
assert "release_version" in required, (
f"release_version must be required in ApprovalGateRequest. "
f"Required fields: {required}"
)
assert "release_commit_hash" in required, (
f"release_commit_hash must be required in ApprovalGateRequest. "
f"Required fields: {required}"
)
# #endregion Test.Api.DashboardTesting.OpenAPI.Alignment.TestApprovalGateRequestFields
# #region Test.Api.DashboardTesting.OpenAPI.Alignment.TestSnapshotCaptureResponseFields [C:2] [TYPE Function]
# @BRIEF SnapshotCaptureResponse includes new provenance fields: release_id, release_commit_hash, repository_id.
# @TEST_EDGE: snapshot_capture_response_fields -> Release-bound capture response has all required fields.
def test_snapshot_capture_response_fields(self, openapi_schema):
"""SnapshotCaptureResponse includes release_id, release_commit_hash, repository_id, etc."""
components = openapi_schema.get("components", {})
schemas = components.get("schemas", {})
cap_schema = schemas.get("SnapshotCaptureResponse", {})
props = cap_schema.get("properties", {})
# Verify new provenance fields exist in properties (they have defaults so not required)
for field in ("release_id", "release_commit_hash", "repository_id", "repository_key", "dashboard_key",
"charts_count", "filters_count", "datasets_count", "query_model_fingerprint", "warnings"):
assert field in props, (
f"Field '{field}' missing from SnapshotCaptureResponse properties. "
f"Properties: {list(props.keys())}"
)
# Verify release_commit_hash exists (pattern not required on response)
assert "release_commit_hash" in props, (
"release_commit_hash missing from SnapshotCaptureResponse properties"
)
# Verify existing required fields still present
required = cap_schema.get("required", [])
for field in ("snapshot_path", "environment_id", "dashboard_id", "release_version", "repository_key", "dashboard_key"):
assert field in required, (
f"Field '{field}' must be required in SnapshotCaptureResponse. "
f"Required: {required}"
)
# #endregion Test.Api.DashboardTesting.OpenAPI.Alignment.TestSnapshotCaptureResponseFields
# #region Test.Api.DashboardTesting.OpenAPI.Alignment.TestStructDiffResponseSchema [C:2] [TYPE Function]
# @BRIEF Structure-diff response schema is documented with all required fields.
def test_structure_diff_response_schema(self, openapi_schema):
"""T037: Structure-diff response has release_from, release_to, changes, summary."""
path = "/api/dashboard-testing/structure-diff"
path_item = openapi_schema["paths"].get(path, {})
post_op = path_item.get("post", {})
responses = post_op.get("responses", {})
ok_resp = responses.get("200", {})
content = ok_resp.get("content", {})
json_schema = content.get("application/json", {}).get("schema", {})
# Use $ref if present
ref = json_schema.get("$ref", "")
if ref:
# Resolve ref
components = openapi_schema.get("components", {})
schemas = components.get("schemas", {})
ref_name = ref.split("/")[-1]
json_schema = schemas.get(ref_name, {})
props = json_schema.get("properties", {})
for field in ("release_from", "release_to", "changes", "summary"):
assert field in props, (
f"Field '{field}' missing from StructureDiff response schema. "
f"Properties: {list(props.keys())}"
)
# #endregion Test.Api.DashboardTesting.OpenAPI.Alignment.TestStructDiffResponseSchema
# #region Test.Api.DashboardTesting.OpenAPI.Alignment.TestVerificationRun201 [C:2] [TYPE Function]
# @BRIEF Verify the OpenAPI schema includes 201 for verification-runs.
def test_verification_runs_201_response(self, openapi_schema):
"""T037: Verification-runs endpoint is documented with 201 response."""
path = "/api/dashboard-testing/verification-runs"
path_item = openapi_schema["paths"].get(path, {})
post_op = path_item.get("post", {})
responses = post_op.get("responses", {})
assert "201" in responses, (
f"201 response missing for verification-runs. "
f"Available responses: {list(responses.keys())}"
)
# #endregion Test.Api.DashboardTesting.OpenAPI.Alignment.TestVerificationRun201
# #region Test.Api.DashboardTesting.OpenAPI.Alignment.TestVerificationRunRequest [C:2] [TYPE Function]
# @BRIEF Verification-runs request body has required fields: repository_id, trigger, environment_id, categories.
def test_verification_runs_request_schema(self, openapi_schema):
"""T037: Verification-run request body documents all required fields."""
path = "/api/dashboard-testing/verification-runs"
path_item = openapi_schema["paths"].get(path, {})
post_op = path_item.get("post", {})
request_body = post_op.get("requestBody", {})
content = request_body.get("content", {})
json_schema = content.get("application/json", {}).get("schema", {})
# Follow $ref if present
ref = json_schema.get("$ref", "")
if ref:
components = openapi_schema.get("components", {})
schemas = components.get("schemas", {})
ref_name = ref.split("/")[-1]
json_schema = schemas.get(ref_name, {})
required = json_schema.get("required", [])
for field in ("repository_id", "trigger", "environment_id", "categories"):
assert field in required, (
f"Field '{field}' must be required in verification-run request. "
f"Required fields: {required}"
)
# #endregion Test.Api.DashboardTesting.OpenAPI.Alignment.TestVerificationRunRequest
# #endregion Test.Api.DashboardTesting.OpenAPI.Alignment
# #region Test.Api.DashboardTesting.StructureSnapshotRoutes [C:3] [TYPE Class] [SEMANTICS test,api,structure-snapshot,route,provenance]
class TestStructureSnapshotRoutes:
"""API-level TestClient regression tests for structure-snapshot capture/diff routes.
Verifies that the capture route correctly resolves Environment from the
release/deployment chain and calls get_superset_client(env). These tests
mock the DB layer to isolate route logic.
"""
# #region Test.Api.DashboardTesting.StructureSnapshotRoutes.TestCaptureRouteReachesService [C:3] [TYPE Function]
# @BRIEF Capture route resolves environment, creates a client, and reaches capture service.
# @TEST_EDGE: capture_route_env_resolution -> Route resolves effective_env_id, creates client, and passes to service.
def test_capture_route_resolves_env_and_reaches_capture_service(self, client, monkeypatch):
"""T037: A valid TestClient request resolves env, builds a client, and reaches capture."""
from unittest.mock import AsyncMock, MagicMock
from src.api.routes.dashboard_testing import structure_snapshot
from src.core.config_models import Environment
from src.core.database import get_db
from src.schemas.dashboard_testing import SnapshotCaptureResponse
release = MagicMock(deployment_id="deployment-1")
deployment = MagicMock(environment_id="env-preprod")
release_query = MagicMock()
release_query.filter.return_value.first.return_value = release
deployment_query = MagicMock()
deployment_query.filter.return_value.first.return_value = deployment
db = MagicMock()
db.query.side_effect = [release_query, deployment_query]
resolved_env = Environment(
id="env-preprod",
name="Preprod",
url="https://superset.preprod.test",
username="tester",
password="secret",
)
config_manager = MagicMock()
config_manager.get_environment.return_value = resolved_env
mock_client = MagicMock()
get_client = AsyncMock(return_value=mock_client)
capture_response = SnapshotCaptureResponse(
snapshot_path="/tmp/snapshots/test.json",
environment_id="env-preprod",
dashboard_id=42,
release_version="v1.0.0",
release_id="release-1",
release_commit_hash="a" * 40,
repository_id="repo-1",
repository_key="repo-1",
dashboard_key="dash_42",
)
capture_service = AsyncMock(return_value=capture_response)
monkeypatch.setattr(structure_snapshot, "get_config_manager", lambda: config_manager)
monkeypatch.setattr(structure_snapshot, "get_superset_client", get_client)
monkeypatch.setattr(structure_snapshot, "capture_release_snapshot", capture_service)
app.dependency_overrides[get_db] = lambda: db
try:
response = client.post(
"/api/dashboard-testing/structure-snapshot/capture?environment_id=env-preprod",
json={"release_id": "release-1"},
)
finally:
app.dependency_overrides.pop(get_db, None)
assert response.status_code == 201, response.text
assert response.json()["release_id"] == "release-1"
config_manager.get_environment.assert_called_once_with("env-preprod")
get_client.assert_awaited_once_with(resolved_env)
capture_service.assert_awaited_once()
assert capture_service.await_args.kwargs["client"] is mock_client
assert capture_service.await_args.kwargs["environment_id"] == "env-preprod"
assert capture_service.await_args.kwargs["db"] is db
# #endregion Test.Api.DashboardTesting.StructureSnapshotRoutes.TestCaptureRouteReachesService
# #region Test.Api.DashboardTesting.StructureSnapshotRoutes.TestCaptureRouteRequiresEnvParameter [C:2] [TYPE Function]
# @BRIEF Capture route REQUIRES environment_id query parameter (not optional).
# The caller's environment_id is cross-checked against the release's
# authoritative deployment environment. Mismatch returns 422 before
# any Superset call or write.
# @TEST_EDGE: capture_env_required -> environment_id is required on capture endpoint.
def test_capture_route_requires_env_parameter_in_openapi(self, openapi_schema):
"""T037: Capture endpoint REQUIRES environment_id parameter in OpenAPI spec."""
path = "/api/dashboard-testing/structure-snapshot/capture"
path_item = openapi_schema.get("paths", {}).get(path, {})
post_op = path_item.get("post", {})
params = post_op.get("parameters", [])
env_param = next((p for p in params if p.get("name") == "environment_id"), None)
assert env_param is not None, (
f"environment_id parameter missing from capture endpoint. "
f"Parameters: {[p.get('name') for p in params]}"
)
assert env_param.get("required") is True, (
"environment_id MUST be required (caller must cross-check against deployment env). "
f"Got required={env_param.get('required')}"
)
assert env_param.get("schema", {}).get("type") == "string", (
"environment_id schema type must be string"
)
# Verify the OpenAPI spec for capture path exists and has correct method
assert "post" in path_item, "Capture endpoint must be POST"
assert "201" in post_op.get("responses", {}), "Capture endpoint must have 201 response"
assert "422" in post_op.get("responses", {}), "Capture endpoint must have 422 response"
# #endregion Test.Api.DashboardTesting.StructureSnapshotRoutes.TestCaptureRouteRequiresEnvParameter
# #region Test.Api.DashboardTesting.StructureSnapshotRoutes.TestCaptureRoute201Status [C:2] [TYPE Function]
# @BRIEF Capture route is documented with 201 status in OpenAPI.
def test_capture_route_201_status(self, openapi_schema):
"""T037: Capture endpoint has 201 response in OpenAPI spec."""
path = "/api/dashboard-testing/structure-snapshot/capture"
assert path in openapi_schema.get("paths", {}), "Capture path missing from OpenAPI"
post_op = openapi_schema["paths"][path]["post"]
assert "201" in post_op.get("responses", {}), "201 response missing for capture endpoint"
# #endregion Test.Api.DashboardTesting.StructureSnapshotRoutes.TestCaptureRoute201Status
# #region Test.Api.DashboardTesting.StructureSnapshotRoutes.TestCaptureEnvMismatch [C:3] [TYPE Function]
# @BRIEF Capture with environment_id that doesn't match deployment returns 422.
# @TEST_EDGE: capture_env_mismatch -> 422 before Superset call if env_id doesn't match deployment.
def test_capture_rejects_env_mismatch(self, client, monkeypatch):
"""T037: Capture with wrong environment_id returns 422 before Superset call."""
from unittest.mock import MagicMock
from src.api.routes.dashboard_testing import structure_snapshot
from src.core.database import get_db
release = MagicMock(deployment_id="deployment-1")
deployment = MagicMock(environment_id="env-production")
release_query = MagicMock()
release_query.filter.return_value.first.return_value = release
deployment_query = MagicMock()
deployment_query.filter.return_value.first.return_value = deployment
db = MagicMock()
db.query.side_effect = [release_query, deployment_query]
# Track whether get_superset_client was called — it MUST NOT be
get_client_called = False
async def _fail_if_called(*_args, **_kwargs):
nonlocal get_client_called
get_client_called = True
raise AssertionError("get_superset_client should NOT be called on env mismatch")
monkeypatch.setattr(structure_snapshot, "get_superset_client", _fail_if_called)
config_manager = MagicMock()
config_manager.get_environment.return_value = None # Should not reach this
monkeypatch.setattr(structure_snapshot, "get_config_manager", lambda: config_manager)
app.dependency_overrides[get_db] = lambda: db
try:
response = client.post(
"/api/dashboard-testing/structure-snapshot/capture?environment_id=env-staging",
json={"release_id": "release-1"},
)
finally:
app.dependency_overrides.pop(get_db, None)
assert response.status_code == 422, (
f"Expected 422 for env mismatch, got {response.status_code}: {response.text}"
)
detail = response.json().get("detail", "").lower()
assert "mismatch" in detail, f"Response should mention mismatch: {detail}"
assert get_client_called is False, "get_superset_client MUST NOT be called on env mismatch"
# #endregion Test.Api.DashboardTesting.StructureSnapshotRoutes.TestCaptureEnvMismatch
# #region Test.Api.DashboardTesting.StructureSnapshotRoutes.TestDiffRoute200Status [C:2] [TYPE Function]
# @BRIEF Diff route is documented with 200 status in OpenAPI.
def test_diff_route_200_status(self, openapi_schema):
"""T037: Diff endpoint has 200 response in OpenAPI spec."""
path = "/api/dashboard-testing/structure-snapshot/diff"
assert path in openapi_schema.get("paths", {}), "Diff path missing from OpenAPI"
post_op = openapi_schema["paths"][path]["post"]
assert "200" in post_op.get("responses", {}), "200 response missing for diff endpoint"
# #endregion Test.Api.DashboardTesting.StructureSnapshotRoutes.TestDiffRoute200Status
# #endregion Test.Api.DashboardTesting.StructureSnapshotRoutes
# #region Test.Api.DashboardTesting.OpenAPI.SemverCommitHash [C:2] [TYPE Class] [SEMANTICS test,api,openapi,semver,commit-hash]
class TestSemverCommitHashConstraints:
"""OpenAPI schema includes SemVer and 40/64 hex constraints on approval/consume endpoints."""
# #region Test.Api.DashboardTesting.OpenAPI.SemverCommitHash.TestConsumeSemverValidation [C:2] [TYPE Function]
# @BRIEF Consume endpoint rejects invalid SemVer release_version with 422.
def test_consume_rejects_invalid_semver(self, client):
"""T037: Consume with non-SemVer release_version returns 422."""
resp = client.post(
"/api/dashboard-testing/baseline-candidates/00000000-0000-0000-0000-000000000000"
"/approval-gate/00000000-0000-0000-0000-000000000000/consume"
"?release_version=not-semver&release_commit_hash=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b",
)
assert resp.status_code == 422, f"Expected 422, got {resp.status_code}: {resp.text}"
# #endregion Test.Api.DashboardTesting.OpenAPI.SemverCommitHash.TestConsumeSemverValidation
# #region Test.Api.DashboardTesting.OpenAPI.SemverCommitHash.TestConsumeInvalidCommitHash [C:2] [TYPE Function]
# @BRIEF Consume endpoint rejects invalid commit hash (not 40/64 hex) with 422.
def test_consume_rejects_invalid_commit_hash(self, client):
"""T037: Consume with invalid commit hash returns 422."""
resp = client.post(
"/api/dashboard-testing/baseline-candidates/00000000-0000-0000-0000-000000000000"
"/approval-gate/00000000-0000-0000-0000-000000000000/consume"
"?release_version=v1.0.0&release_commit_hash=invalid",
)
assert resp.status_code == 422, f"Expected 422, got {resp.status_code}: {resp.text}"
# #endregion Test.Api.DashboardTesting.OpenAPI.SemverCommitHash.TestConsumeInvalidCommitHash
# #endregion Test.Api.DashboardTesting.OpenAPI.SemverCommitHash
# #endregion Test.Api.DashboardTesting.OpenAPI