# #region Test.Integration.MaintenanceLifecycleE2E [C:5] [TYPE Module] [SEMANTICS test,integration,maintenance,e2e,superset,testcontainers] # @BRIEF Full maintenance lifecycle E2E tests: seed PG table, register in Superset, create dashboard, # exercise start_maintenance/end_maintenance orchestrators, verify postconditions in BOTH DB and Superset. # @RELATION BINDS_TO -> [Services.Orchestrators.MaintenanceOrchestrators] # @RELATION BINDS_TO -> [Core.DashboardsWrite.SupersetDashboardsWriteMixin] # @RELATION BINDS_TO -> [Services.DashboardScanner.MaintenanceDashboardScanner] # @RELATION BINDS_TO -> [Models.Maintenance.MaintenanceModels] # @RELATION DEPENDS_ON -> [Test.Conftest.IntegrationTestConftest] # # @TEST_CONTRACT MaintenanceLifecycle -> # { # scenarios: [ # "start creates exactly one banner and updates dashboard layout", # "repeated start call on already-processed event is idempotent", # "end removes banner and transitions event to COMPLETED", # "end restores dashboard layout after banner removal" # ] # } # @TEST_EDGE: repeated_start_idempotent -> start_maintenance on ACTIVE event returns current status # @TEST_EDGE: missing_superset_dataset -> start_maintenance returns "no_match" gracefully # @TEST_EDGE: end_maintenance_after_removed -> idempotent end call returns "already_completed" # # @TEST_INVARIANT event_lifecycle -> VERIFIED_BY: [test_start_creates_banner_and_updates_layout, test_end_removes_banner_and_restores_state] # @TEST_INVARIANT idempotent_start -> VERIFIED_BY: [Test.Integration.TestRepeatedStartIsIdempotent] # @TEST_INVARIANT banner_uniqueness -> VERIFIED_BY: [Test.Integration.TestStartCreatesBannerAndUpdatesLayout] # # @PRE Docker daemon running; testcontainers PostgreSQL + Superset containers can start. # @POST All test resources cleaned up (PG tables, Superset databases/datasets/dashboards/charts). # @SIDE_EFFECT Creates and removes real Superset and PostgreSQL resources. # @RATIONALE # Previous integration tests only verify ORM models in isolation (test_maintenance_integration.py). # This test drives the actual production orchestrator code path (start_maintenance / end_maintenance) # with a real Superset instance and real PostgreSQL, verifying postconditions in both systems. # # The narrowest real production entrypoint is the orchestrator functions, not the API routes # (which require TaskManager + auth). Calling start_maintenance() directly exercises: # - find_affected_dashboards() — scans real Superset datasets # - ensure_banner_chart() — creates real markdown chart in Superset # - update_dashboard_layout() — modifies real dashboard position_json # - _process_states_for_end() — removes chart and restores layout # # @REJECTED # Calling the API route directly rejected — requires TaskManager auth, JWT/permission setup, # and async TaskManager processing which adds ~30s per test without additional coverage. # Mocking SupersetClient rejected — would test the orchestrator's SQL logic but not the # actual Superset API integration. from contextlib import suppress from datetime import UTC, datetime, timedelta import json import pytest from uuid import uuid4 from sqlalchemy import text from src.models.maintenance import ( MaintenanceDashboardBanner, MaintenanceDashboardBannerStatus, MaintenanceDashboardState, MaintenanceDashboardStateStatus, MaintenanceEvent, MaintenanceEventStatus, MaintenanceSettings, ) from src.services.maintenance._orchestrators import end_maintenance, start_maintenance # #region Test.Integration.MaintenanceLifecycleE2E.Test [C:4] [TYPE Class] # @BRIEF E2E tests for maintenance lifecycle with real Superset + PostgreSQL. class TestMaintenanceLifecycleE2E: """Full maintenance lifecycle: seed, start, verify, end, verify.""" # #region Test.Integration.TestStartCreatesBannerAndUpdatesLayout [C:3] [TYPE Function] # @BRIEF Seed PG table, register in Superset, create dashboard, start maintenance — verify banner + layout. # @TEST_SCENARIO: start_creates_banner -> MaintenanceEvent transitions to ACTIVE, # MaintenanceDashboardBanner created with ACTIVE status, # MaintenanceDashboardState created with ACTIVE status, # Superset dashboard layout includes MARKDOWN-banner- element. @pytest.mark.integration @pytest.mark.asyncio async def test_start_creates_banner_and_updates_layout( # noqa: C901 self, superset_client, superset_db_url, pg_engine, db_session, ): suffix = uuid4().hex[:10] table_name = f"maint_e2e_start_{suffix}" dashboard_title = f"Maintenance E2E Start {suffix}" database_id = None dataset_id = None chart_id = None dashboard_id = None event_id = None banner_id = None # ── Step 1: Seed physical PostgreSQL table ── with pg_engine.begin() as conn: conn.execute( text(f'CREATE TABLE public."{table_name}" (id INTEGER PRIMARY KEY, label VARCHAR(64))') ) conn.execute( text(f"""INSERT INTO public."{table_name}" VALUES (1, 'alpha'), (2, 'beta'), (3, 'gamma')""") ) try: # ── Step 2: Register database in Superset ── from urllib.parse import urlparse parsed = urlparse(superset_db_url) database_uri = ( f"postgresql://test:test@{parsed.hostname}:{parsed.port}/test_translate" ) db_resp = await superset_client.create_database( database_name=f"Maint E2E Source {suffix}", sqlalchemy_uri=database_uri, ) database_id = int( db_resp.get("id") or db_resp.get("result", {}).get("id", 0) ) # ── Step 3: Create dataset ── ds_resp = await superset_client.create_dataset( table_name=table_name, database=database_id, schema_name="public", ) dataset_id = int( ds_resp.get("id") or ds_resp.get("result", {}).get("id", 0) ) # ── Step 4: Create dashboard ── dash_resp = await superset_client.create_dashboard( dashboard_title=dashboard_title, slug=f"maint-e2e-start-{suffix}", published=True, ) dashboard_id = int( dash_resp.get("id") or dash_resp.get("result", {}).get("id", 0) ) # ── Step 5: Create chart and add to dashboard ── chart_resp = await superset_client.client.request( method="POST", endpoint="/chart/", data={ "dashboards": [dashboard_id], "datasource_id": dataset_id, "datasource_type": "table", "slice_name": f"Maint E2E Chart {suffix}", "viz_type": "table", "params": json.dumps({ "datasource": f"{dataset_id}__table", "viz_type": "table", "all_columns": ["id", "label"], "row_limit": 100, }), }, ) chart_id = int( chart_resp.get("id") or chart_resp.get("result", {}).get("id", 0) ) await superset_client.client.request( method="PUT", endpoint=f"/dashboard/{dashboard_id}", data={ "position_json": json.dumps({ f"CHART-{chart_id}": { "id": f"CHART-{chart_id}", "type": "CHART", "meta": {"chartId": chart_id, "width": 12, "height": 50}, } }) }, ) # ── Step 6: Ensure MaintenanceSettings exist ── settings = db_session.query(MaintenanceSettings).filter( MaintenanceSettings.id == "default" ).first() if not settings: settings = MaintenanceSettings( id="default", target_environment_id="test_env", banner_template="Maintenance: {message} ({start_time} - {end_time})", ) db_session.add(settings) db_session.commit() # ── Step 7: Create PENDING MaintenanceEvent ── now = datetime.now(UTC) event = MaintenanceEvent( tables=[f"public.{table_name}"], start_time=now, end_time=now + timedelta(hours=2), message="Scheduled maintenance - E2E test", status=MaintenanceEventStatus.PENDING, environment_id="test_env", ) db_session.add(event) db_session.commit() event_id = event.id # ── Step 8: Call start_maintenance orchestrator ── result = await start_maintenance(event_id, db_session, superset_client) db_session.commit() # ── Verify postconditions ── # 8a: Result indicates success assert result["status"] in ("active", "partial"), f"Expected active/partial, got {result['status']}" assert result["affected_dashboards"] >= 1, f"Expected >=1 dashboard, got {result['affected_dashboards']}" # 8b: Event transitions to ACTIVE db_session.expire_all() updated_event = db_session.query(MaintenanceEvent).filter( MaintenanceEvent.id == event_id ).first() assert updated_event is not None assert updated_event.status == MaintenanceEventStatus.ACTIVE, \ f"Expected ACTIVE, got {updated_event.status}" # 8c: DashboardBanner created with ACTIVE status banners = db_session.query(MaintenanceDashboardBanner).filter( MaintenanceDashboardBanner.environment_id == "test_env", MaintenanceDashboardBanner.status == MaintenanceDashboardBannerStatus.ACTIVE, ).all() assert len(banners) >= 1, "Expected at least one active banner" banner = banners[0] banner_id = banner.id assert banner.dashboard_id == dashboard_id, f"Expected dashboard_id={dashboard_id}, got {banner.dashboard_id}" assert banner.chart_id is not None, "Expected chart_id to be set" # 8d: DashboardState created with ACTIVE status states = db_session.query(MaintenanceDashboardState).filter( MaintenanceDashboardState.event_id == event_id, MaintenanceDashboardState.dashboard_id == dashboard_id, ).all() assert len(states) >= 1, "Expected at least one dashboard state" assert states[0].status == MaintenanceDashboardStateStatus.ACTIVE, \ f"Expected ACTIVE, got {states[0].status}" # 8e: Superset dashboard layout includes MARKDOWN-banner element dashboard_detail = await superset_client.client.request( method="GET", endpoint=f"/dashboard/{dashboard_id}", ) result_data = dashboard_detail.get("result", dashboard_detail) position_json = json.loads(result_data.get("position_json", "{}")) markdown_keys = [k for k in position_json if "MARKDOWN-banner" in k] assert len(markdown_keys) >= 1, \ f"Expected MARKDOWN-banner key in layout, got keys: {list(position_json.keys())[:10]}" # The chart_id is encoded in the key name: MARKDOWN-banner-{chart_id} chart_ids_in_layout = set() for k in position_json: if k.startswith("MARKDOWN-banner-"): try: cid = int(k.split("-")[-1]) chart_ids_in_layout.add(cid) except (ValueError, IndexError): pass assert banner.chart_id in chart_ids_in_layout, \ f"Layout MARKDOWN references chart_ids {chart_ids_in_layout}, expected {banner.chart_id}" # 8f: Only ONE active banner for this (env, dashboard) pair active_count = db_session.query(MaintenanceDashboardBanner).filter( MaintenanceDashboardBanner.environment_id == "test_env", MaintenanceDashboardBanner.dashboard_id == dashboard_id, MaintenanceDashboardBanner.status == MaintenanceDashboardBannerStatus.ACTIVE, ).count() assert active_count == 1, f"Expected exactly 1 active banner, got {active_count}" finally: # ── Cleanup ── if event_id: with suppress(Exception): # End maintenance if still active await end_maintenance(event_id, db_session, superset_client) db_session.commit() if banner_id: with suppress(Exception): b = db_session.query(MaintenanceDashboardBanner).filter( MaintenanceDashboardBanner.id == banner_id ).first() if b: b.status = MaintenanceDashboardBannerStatus.REMOVED db_session.commit() if chart_id: with suppress(Exception): await superset_client.delete_chart(chart_id) if dashboard_id: with suppress(Exception): await superset_client.delete_dashboard(dashboard_id) if dataset_id: with suppress(Exception): await superset_client.delete_dataset(dataset_id) if database_id: with suppress(Exception): await superset_client.delete_database(database_id) with pg_engine.begin() as conn: conn.execute(text(f'DROP TABLE IF EXISTS public."{table_name}"')) # #endregion Test.Integration.TestStartCreatesBannerAndUpdatesLayout # #region Test.Integration.TestRepeatedStartIsIdempotent [C:3] [TYPE Function] # @BRIEF Call start_maintenance on an already-ACTIVE event — verify it returns current status. # @TEST_EDGE: repeated_start_idempotent -> VERIFIED_BY: test_repeated_start_is_idempotent @pytest.mark.integration @pytest.mark.asyncio async def test_repeated_start_is_idempotent( self, superset_client, superset_db_url, pg_engine, db_session, ): suffix = uuid4().hex[:10] table_name = f"maint_e2e_idem_{suffix}" dashboard_title = f"Maintenance E2E Idemp {suffix}" # ── Setup: seed table, create Superset resources, create PENDING event, start once ── with pg_engine.begin() as conn: conn.execute( text(f'CREATE TABLE public."{table_name}" (id INTEGER PRIMARY KEY, label VARCHAR(64))') ) conn.execute(text(f"""INSERT INTO public."{table_name}" VALUES (1, 'x')""")) database_id = dataset_id = chart_id = dashboard_id = event_id = None try: from urllib.parse import urlparse parsed = urlparse(superset_db_url) database_uri = f"postgresql://test:test@{parsed.hostname}:{parsed.port}/test_translate" db_resp = await superset_client.create_database( database_name=f"Maint Idem {suffix}", sqlalchemy_uri=database_uri, ) database_id = int(db_resp.get("id") or db_resp.get("result", {}).get("id", 0)) ds_resp = await superset_client.create_dataset( table_name=table_name, database=database_id, schema_name="public", ) dataset_id = int(ds_resp.get("id") or ds_resp.get("result", {}).get("id", 0)) dash_resp = await superset_client.create_dashboard( dashboard_title=dashboard_title, slug=f"maint-idem-{suffix}", published=True, ) dashboard_id = int(dash_resp.get("id") or dash_resp.get("result", {}).get("id", 0)) chart_resp = await superset_client.client.request( method="POST", endpoint="/chart/", data={ "dashboards": [dashboard_id], "datasource_id": dataset_id, "datasource_type": "table", "slice_name": f"Maint Idem Chart {suffix}", "viz_type": "table", "params": json.dumps({"datasource": f"{dataset_id}__table", "viz_type": "table", "row_limit": 100}), }, ) chart_id = int(chart_resp.get("id") or chart_resp.get("result", {}).get("id", 0)) await superset_client.client.request( method="PUT", endpoint=f"/dashboard/{dashboard_id}", data={ "position_json": json.dumps({ f"CHART-{chart_id}": { "id": f"CHART-{chart_id}", "type": "CHART", "meta": {"chartId": chart_id, "width": 12, "height": 50}, } }) }, ) # Create PENDING event now = datetime.now(UTC) event = MaintenanceEvent( tables=[f"public.{table_name}"], start_time=now, end_time=now + timedelta(hours=2), status=MaintenanceEventStatus.PENDING, environment_id="test_env", ) db_session.add(event) db_session.commit() event_id = event.id # First call: start result1 = await start_maintenance(event_id, db_session, superset_client) db_session.commit() assert result1["status"] in ("active", "partial"), f"First start failed: {result1}" # Second call: start again — should be idempotent result2 = await start_maintenance(event_id, db_session, superset_client) db_session.commit() # Verify idempotency: returns current status without re-creating assert result2["status"] in ("active", "partial"), \ f"Idempotent start returned unexpected status: {result2['status']}" # Verify still exactly ONE active banner active_count = db_session.query(MaintenanceDashboardBanner).filter( MaintenanceDashboardBanner.environment_id == "test_env", MaintenanceDashboardBanner.dashboard_id == dashboard_id, MaintenanceDashboardBanner.status == MaintenanceDashboardBannerStatus.ACTIVE, ).count() assert active_count == 1, \ f"Expected exactly 1 active banner after idempotent start, got {active_count}" # Verify event status unchanged (still ACTIVE) db_session.expire_all() ev = db_session.query(MaintenanceEvent).filter(MaintenanceEvent.id == event_id).first() assert ev is not None assert ev.status == MaintenanceEventStatus.ACTIVE, \ f"Expected ACTIVE after idempotent start, got {ev.status}" finally: if event_id: with suppress(Exception): await end_maintenance(event_id, db_session, superset_client) db_session.commit() for cid in ([chart_id] if chart_id else []): with suppress(Exception): await superset_client.delete_chart(cid) for did in ([dashboard_id] if dashboard_id else []): with suppress(Exception): await superset_client.delete_dashboard(did) for dsid in ([dataset_id] if dataset_id else []): with suppress(Exception): await superset_client.delete_dataset(dsid) for dbid in ([database_id] if database_id else []): with suppress(Exception): await superset_client.delete_database(dbid) with pg_engine.begin() as conn: conn.execute(text(f'DROP TABLE IF EXISTS public."{table_name}"')) # #endregion Test.Integration.TestRepeatedStartIsIdempotent # #region Test.Integration.TestEndRemovesBannerAndRestoresState [C:3] [TYPE Function] # @BRIEF Full lifecycle: start then end — verify COMPLETED event, REMOVED banner, restored state. # @TEST_SCENARIO: end_removes_banner -> Event transitions to COMPLETED, # Banner transitions to REMOVED, DashboardState transitions to REMOVED, # MARKDOWN-banner element removed from Superset dashboard layout. @pytest.mark.integration @pytest.mark.asyncio async def test_end_removes_banner_and_restores_state( self, superset_client, superset_db_url, pg_engine, db_session, ): suffix = uuid4().hex[:10] table_name = f"maint_e2e_end_{suffix}" with pg_engine.begin() as conn: conn.execute( text(f'CREATE TABLE public."{table_name}" (id INTEGER PRIMARY KEY, label VARCHAR(64))') ) conn.execute(text(f"""INSERT INTO public."{table_name}" VALUES (1, 'end-test')""")) database_id = dataset_id = chart_id = dashboard_id = event_id = None try: from urllib.parse import urlparse parsed = urlparse(superset_db_url) database_uri = f"postgresql://test:test@{parsed.hostname}:{parsed.port}/test_translate" db_resp = await superset_client.create_database( database_name=f"Maint End {suffix}", sqlalchemy_uri=database_uri, ) database_id = int(db_resp.get("id") or db_resp.get("result", {}).get("id", 0)) ds_resp = await superset_client.create_dataset( table_name=table_name, database=database_id, schema_name="public", ) dataset_id = int(ds_resp.get("id") or ds_resp.get("result", {}).get("id", 0)) dash_resp = await superset_client.create_dashboard( dashboard_title=f"Maintenance E2E End {suffix}", slug=f"maint-end-{suffix}", published=True, ) dashboard_id = int(dash_resp.get("id") or dash_resp.get("result", {}).get("id", 0)) chart_resp = await superset_client.client.request( method="POST", endpoint="/chart/", data={ "dashboards": [dashboard_id], "datasource_id": dataset_id, "datasource_type": "table", "slice_name": f"Maint End Chart {suffix}", "viz_type": "table", "params": json.dumps({"datasource": f"{dataset_id}__table", "viz_type": "table", "row_limit": 100}), }, ) chart_id = int(chart_resp.get("id") or chart_resp.get("result", {}).get("id", 0)) await superset_client.client.request( method="PUT", endpoint=f"/dashboard/{dashboard_id}", data={ "position_json": json.dumps({ f"CHART-{chart_id}": { "id": f"CHART-{chart_id}", "type": "CHART", "meta": {"chartId": chart_id, "width": 12, "height": 50}, } }) }, ) now = datetime.now(UTC) event = MaintenanceEvent( tables=[f"public.{table_name}"], start_time=now, end_time=now + timedelta(hours=2), status=MaintenanceEventStatus.PENDING, environment_id="test_env", ) db_session.add(event) db_session.commit() event_id = event.id # ── Start ── start_result = await start_maintenance(event_id, db_session, superset_client) db_session.commit() assert start_result["status"] in ("active", "partial"), f"Start failed: {start_result}" assert start_result["affected_dashboards"] >= 1 # Capture banner_id before end banner = db_session.query(MaintenanceDashboardBanner).filter( MaintenanceDashboardBanner.environment_id == "test_env", MaintenanceDashboardBanner.status == MaintenanceDashboardBannerStatus.ACTIVE, ).first() assert banner is not None banner_id = banner.id # Verify banner chart exists in Superset if banner.chart_id: try: chart_data = await superset_client.get_chart(banner.chart_id) assert chart_data is not None, "Banner chart not found in Superset" except Exception as e: pytest.fail(f"Banner chart {banner.chart_id} not found in Superset: {e}") # ── End ── end_result = await end_maintenance(event_id, db_session, superset_client) db_session.commit() assert end_result["status"] == "completed", f"End failed: {end_result}" assert end_result["removed_from"] >= 1, f"Expected >=1 removed, got {end_result}" # ── Verify postconditions ── # Event is COMPLETED db_session.expire_all() ev = db_session.query(MaintenanceEvent).filter(MaintenanceEvent.id == event_id).first() assert ev is not None assert ev.status == MaintenanceEventStatus.COMPLETED, \ f"Expected COMPLETED, got {ev.status}" # Banner is REMOVED b = db_session.query(MaintenanceDashboardBanner).filter( MaintenanceDashboardBanner.id == banner_id ).first() assert b is not None assert b.status == MaintenanceDashboardBannerStatus.REMOVED, \ f"Expected REMOVED banner, got {b.status}" # DashboardState is REMOVED states = db_session.query(MaintenanceDashboardState).filter( MaintenanceDashboardState.event_id == event_id, ).all() assert len(states) >= 1 assert all( s.status == MaintenanceDashboardStateStatus.REMOVED for s in states ), f"Not all states are REMOVED: {[s.status for s in states]}" # MARKDOWN-banner element removed from dashboard layout dash_detail = await superset_client.client.request( method="GET", endpoint=f"/dashboard/{dashboard_id}", ) result_data = dash_detail.get("result", dash_detail) position_json = json.loads(result_data.get("position_json", "{}")) markdown_keys = [k for k in position_json if "MARKDOWN-banner" in k] assert len(markdown_keys) == 0, \ f"Expected no MARKDOWN-banner keys after end, got {markdown_keys}" finally: with suppress(Exception): await end_maintenance(event_id, db_session, superset_client) if event_id else None db_session.commit() for cid in ([chart_id] if chart_id else []): with suppress(Exception): await superset_client.delete_chart(cid) for did in ([dashboard_id] if dashboard_id else []): with suppress(Exception): await superset_client.delete_dashboard(did) for dsid in ([dataset_id] if dataset_id else []): with suppress(Exception): await superset_client.delete_dataset(dsid) for dbid in ([database_id] if database_id else []): with suppress(Exception): await superset_client.delete_database(dbid) with pg_engine.begin() as conn: conn.execute(text(f'DROP TABLE IF EXISTS public."{table_name}"')) # #endregion Test.Integration.TestEndRemovesBannerAndRestoresState # #endregion Test.Integration.MaintenanceLifecycleE2E.Test # #endregion Test.Integration.MaintenanceLifecycleE2E