From ffa4d6a85bea65cc57f824056873cd30cfbd8983 Mon Sep 17 00:00:00 2001 From: busya Date: Fri, 21 Aug 2026 16:15:40 +0300 Subject: [PATCH] feat(scenarios): implement execution engine contracts --- AGENTS.md | 150 +++ ...3a4_add_investigation_evidence_snapshot.py | 28 + ...5b6_add_scenario_live_execution_binding.py | 41 + ...dd_scenario_artifact_attempt_projection.py | 50 + ...c7d8_add_scenario_cancel_drain_deadline.py | 43 + .../dashboard_testing/scenario_analytics.py | 15 +- .../dashboard_testing/scenario_automation.py | 153 ++- .../routes/dashboard_testing/scenario_runs.py | 73 +- backend/src/app.py | 34 + backend/src/core/config_models.py | 20 + backend/src/core/scheduler.py | 252 ++++- backend/src/dependencies.py | 20 + backend/src/models/scenario_artifact.py | 11 +- backend/src/models/scenario_investigation.py | 4 + backend/src/models/scenario_run.py | 13 + .../dashboard_testing/analytics/flakiness.py | 18 + .../analytics/investigation.py | 245 ++++- .../dashboard_testing/analytics/recurring.py | 31 +- .../dashboard_testing/automation/notify.py | 28 + .../dashboard_testing/automation/trigger.py | 104 +- .../dashboard_testing/execution/approval.py | 23 +- .../dashboard_testing/execution/artifacts.py | 143 ++- .../dashboard_testing/execution/dispatch.py | 19 +- .../execution/environment_policy.py | 52 + .../execution/executor_registry.py | 27 +- .../dashboard_testing/execution/executors.py | 362 +++++++ .../dashboard_testing/execution/lifecycle.py | 513 +++++++++- .../execution/live_adapter.py | 92 ++ .../execution/live_binding.py | 260 +++++ .../execution/live_composition.py | 252 +++++ .../dashboard_testing/execution/result.py | 14 +- .../dashboard_testing/execution/runner.py | 949 +++++++++++++++++- .../execution/runner_plan.py | 61 +- .../dashboard_testing/execution/worker.py | 36 +- .../dashboard_testing/registry/staleness.py | 29 + .../scenario/templates/__init__.py | 106 ++ .../tests/api/test_scenario_analytics_api.py | 6 +- .../tests/api/test_scenario_automation_api.py | 124 ++- backend/tests/api/test_scenario_runs_api.py | 332 +++++- .../fixtures/scenario_execution/graph.json | 11 +- .../dashboard_testing/registry/conftest.py | 27 + .../registry/test_live_execution_binding.py | 567 +++++++++++ .../registry/test_scenario_auto_queue.py | 30 + .../test_scenario_automation_trigger.py | 53 +- .../registry/test_scenario_cancel_timeout.py | 437 ++++++++ .../registry/test_scenario_crash_recovery.py | 221 ++++ .../registry/test_scenario_dispatch.py | 25 +- .../registry/test_scenario_executors.py | 472 +++++++++ .../registry/test_scenario_investigation.py | 48 +- .../registry/test_scenario_lifecycle.py | 134 ++- .../registry/test_scenario_manual_run_only.py | 175 ++++ .../registry/test_scenario_queued_dispatch.py | 218 ++++ .../registry/test_scenario_resume.py | 9 +- .../registry/test_scenario_retry_closure.py | 203 ++++ .../registry/test_scenario_runner.py | 220 +++- .../registry/test_scenario_runner_plan.py | 13 +- .../registry/test_scenario_runner_walker.py | 244 +++++ .../test_scenario_scheduler_callbacks.py | 197 ++++ .../test_scenario_terminal_signals.py | 160 +++ .../registry/test_scenario_worker.py | 83 +- .../registry/test_staleness.py | 18 +- ...enario_live_execution_binding_migration.py | 69 ++ .../tests/test_translate_scheduler_audit.py | 2 +- .../test_translate_scheduler_execution.py | 31 +- .../tests/test_translate_scheduler_guard.py | 33 +- .../src/lib/models/RunMonitorModel.svelte.ts | 13 +- .../models/__tests__/RunMonitorModel.test.ts | 4 +- .../data-model.md | 2 +- .../checklists/requirements.md | 15 +- specs/042-dashboard-scenario-registry/plan.md | 5 +- .../prototype/manifest.md | 5 +- .../quickstart.md | 3 + specs/042-dashboard-scenario-registry/spec.md | 16 +- .../042-dashboard-scenario-registry/tasks.md | 21 +- .../traceability.md | 3 + .../checklists/requirements.md | 13 +- specs/043-dashboard-scenario-editor/plan.md | 5 +- .../prototype/manifest.md | 5 +- .../quickstart.md | 3 + specs/043-dashboard-scenario-editor/spec.md | 15 +- specs/043-dashboard-scenario-editor/tasks.md | 15 +- .../traceability.md | 3 + .../SESSION_STATE.md | 117 +++ .../checklists/requirements.md | 7 +- .../contracts/modules.md | 13 +- .../data-model.md | 26 +- .../044-dashboard-scenario-execution/plan.md | 5 +- .../prototype/index.html | 115 ++- .../prototype/manifest.md | 26 +- .../prototype/validate_static.py | 98 ++ .../quickstart.md | 32 +- .../044-dashboard-scenario-execution/spec.md | 108 +- .../044-dashboard-scenario-execution/tasks.md | 88 +- .../traceability.md | 23 +- .../checklists/requirements.md | 5 +- specs/045-dashboard-run-monitor/plan.md | 5 +- .../prototype/manifest.md | 5 +- specs/045-dashboard-run-monitor/quickstart.md | 3 + specs/045-dashboard-run-monitor/spec.md | 16 +- specs/045-dashboard-run-monitor/tasks.md | 17 +- .../045-dashboard-run-monitor/traceability.md | 3 + .../checklists/requirements.md | 7 +- .../046-dashboard-scenario-automation/plan.md | 5 +- .../prototype/manifest.md | 5 +- .../quickstart.md | 3 + .../046-dashboard-scenario-automation/spec.md | 31 +- .../tasks.md | 49 +- .../traceability.md | 14 +- .../checklists/requirements.md | 9 +- .../047-dashboard-scenario-analytics/plan.md | 5 +- .../prototype/manifest.md | 5 +- .../quickstart.md | 3 + .../047-dashboard-scenario-analytics/spec.md | 21 +- .../047-dashboard-scenario-analytics/tasks.md | 37 +- .../traceability.md | 12 +- specs/WORKSTATE-043-047.md | 176 ++-- specs/prototype-ui.css | 13 + 117 files changed, 9156 insertions(+), 530 deletions(-) create mode 100644 backend/alembic/versions/b9c0d1e2f3a4_add_investigation_evidence_snapshot.py create mode 100644 backend/alembic/versions/c1d2e3f4a5b6_add_scenario_live_execution_binding.py create mode 100644 backend/alembic/versions/d2e3f4a5b6c7_add_scenario_artifact_attempt_projection.py create mode 100644 backend/alembic/versions/e3f4a5b6c7d8_add_scenario_cancel_drain_deadline.py create mode 100644 backend/src/services/dashboard_testing/execution/environment_policy.py create mode 100644 backend/src/services/dashboard_testing/execution/executors.py create mode 100644 backend/src/services/dashboard_testing/execution/live_adapter.py create mode 100644 backend/src/services/dashboard_testing/execution/live_binding.py create mode 100644 backend/src/services/dashboard_testing/execution/live_composition.py create mode 100644 backend/tests/services/dashboard_testing/registry/test_live_execution_binding.py create mode 100644 backend/tests/services/dashboard_testing/registry/test_scenario_auto_queue.py create mode 100644 backend/tests/services/dashboard_testing/registry/test_scenario_cancel_timeout.py create mode 100644 backend/tests/services/dashboard_testing/registry/test_scenario_crash_recovery.py create mode 100644 backend/tests/services/dashboard_testing/registry/test_scenario_executors.py create mode 100644 backend/tests/services/dashboard_testing/registry/test_scenario_manual_run_only.py create mode 100644 backend/tests/services/dashboard_testing/registry/test_scenario_queued_dispatch.py create mode 100644 backend/tests/services/dashboard_testing/registry/test_scenario_retry_closure.py create mode 100644 backend/tests/services/dashboard_testing/registry/test_scenario_runner_walker.py create mode 100644 backend/tests/services/dashboard_testing/registry/test_scenario_scheduler_callbacks.py create mode 100644 backend/tests/services/dashboard_testing/registry/test_scenario_terminal_signals.py create mode 100644 backend/tests/test_scenario_live_execution_binding_migration.py create mode 100644 specs/044-dashboard-scenario-execution/SESSION_STATE.md create mode 100644 specs/044-dashboard-scenario-execution/prototype/validate_static.py diff --git a/AGENTS.md b/AGENTS.md index 07f0a8c5a..642589b80 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,6 +4,156 @@ - Рабочий виртуальный окружение (`.venv`) находится внутри `backend/`: `backend/.venv`. +## Тестовый стенд и запуск + +### Прямой локальный стенд: `run.sh` + +Основной способ запуска тестового стенда для разработки — из корня репозитория: + +```bash +./run.sh --skip-install +``` + +`run.sh` запускает три процесса и завершает их по `Ctrl+C`: + +| Сервис | Порт по умолчанию | URL | +|---|---:|---| +| Backend / FastAPI | `8000` | `http://127.0.0.1:8000` | +| Frontend / Vite | `5173` | `http://127.0.0.1:5173` | +| Gradio agent | `7860` | `http://127.0.0.1:7860` | + +После запуска backend готовность можно проверить через `http://127.0.0.1:8000/api/ready`, +а API-документация доступна на `http://127.0.0.1:8000/docs`. `run.sh` сам не ждет frontend +или agent healthcheck после запуска, поэтому первое открытие UI может потребовать несколько секунд. + +При старте скрипт: + +1. Проверяет `python3 >= 3.9` и `npm`. +2. Загружает `backend/.env` до database preflight. +3. Берет URL БД в порядке `DATABASE_URL`, `POSTGRES_URL`, затем локальный PostgreSQL: + `postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools`. +4. Для недоступного локального PostgreSQL пытается выполнить `docker compose up -d db` и + ждет доступность порта до 20 секунд. +5. Генерирует и сохраняет отсутствующие или некорректные `ENCRYPTION_KEY` и + `AUTH_SECRET_KEY` в `backend/.env`. +6. Перед запуском Uvicorn выполняет `alembic upgrade head`; существующую схему без + `alembic_version` скрипт пытается `alembic stamp head`. +7. Загружает `backend/.env` также в backend и agent. `SERVICE_JWT`, если не задан, получает + случайное значение на текущий запуск; при запуске сервисов в отдельных терминалах его + нужно задать одинаковым явно. + +Опции и переменные `run.sh`: + +```bash +./run.sh --help +./run.sh --skip-install +DEV_MODE=true ./run.sh --skip-install +BACKEND_PORT=8001 FRONTEND_PORT=5174 AGENT_PORT=7861 ./run.sh --skip-install +``` + +- `DEV_MODE=true` включает `uvicorn --reload --reload-dir src` и watchfiles для agent. +- Без `--skip-install` скрипт создает `backend/.venv`, устанавливает `backend/requirements.txt`, + `shared` и frontend dependencies. +- `AGENT_CONFIRM_TOOLS` по умолчанию `true`. +- Backend и agent читают LLM-конфигурацию через `/api/agent/llm-config`; провайдеры обычно + настраиваются в Admin -> LLM Settings. + +### Docker Compose стенд + +Для полного контейнерного стенда использовать `build.sh`, а не смешивать его с прямым +`run.sh`: + +```bash +./build.sh up current +./build.sh status +./build.sh logs current +./build.sh down current +``` + +Профили `build.sh`: + +- `current` (по умолчанию): `docker-compose.yml`, project `superset-tools-current`; +- `master`: `docker-compose.yml`, project `superset-tools-master`; +- `enterprise-clean`: `docker-compose.enterprise-clean.yml`, внешний PostgreSQL и + корпоративные сертификаты. + +Для профилей `current`/`master` переменные берутся из `.env.current`/`.env.master`. +Ключевые host ports профиля `current`: PostgreSQL `5433`, backend `8101`, frontend `8100`, +agent `7860`; для `master`: PostgreSQL `5432`, backend `8001`, frontend `8000`, agent `7860`. +Секреты `AUTH_SECRET_KEY`, `ENCRYPTION_KEY` и `SERVICE_JWT` должны быть заданы явно в +Docker-профиле. Не использовать публичные значения из example-файлов. + +### Изолированный Playwright E2E стенд + +E2E запускается отдельным compose-файлом и не должен использовать тот же проект/порты, что +и текущий локальный стенд: + +```bash +docker compose --env-file .env.e2e -f docker-compose.e2e.yml up -d --build +cd frontend && npm run test:e2e +docker compose --env-file .env.e2e -f docker-compose.e2e.yml down -v +``` + +Профиль по умолчанию использует PostgreSQL `5435`, backend `8103`, frontend `8102` и +Playwright runner на образе `mcr.microsoft.com/playwright:v1.52.0-noble`. Для E2E нужны +`AUTH_SECRET_KEY`, `E2E_USERNAME`, `E2E_PASSWORD`; `GITEA_TOKEN` нужен только тестам, +которые обращаются к Gitea. + +## Тестовые команды + +Перед backend-командами: + +```bash +cd backend +source .venv/bin/activate +``` + +Основные проверки: + +```bash +python -m pytest -v # unit/service tests; integration skipped +python -m pytest -v --run-integration # включая Docker/Testcontainers integration +python -m ruff check . +python -m compileall -q src +alembic heads +alembic upgrade head +``` + +Для спеки 044: + +```bash +python -m pytest -q \ + tests/services/dashboard_testing/registry/test_scenario_*.py \ + tests/api/test_scenario_runs_api.py \ + tests/api/test_scenario_automation_api.py \ + tests/api/test_scenario_analytics_api.py +python -m ruff check \ + src/services/dashboard_testing/execution \ + src/api/routes/dashboard_testing/scenario_runs.py +cd .. +source backend/.venv/bin/activate +python specs/044-dashboard-scenario-execution/prototype/validate_static.py +``` + +Frontend: + +```bash +cd frontend +npm run test -- --run +npm run lint +npm run build +``` + +Важные ограничения: + +- Integration tests пропускаются без `--run-integration`. +- Реальные `alembic check/upgrade` требуют доступный PostgreSQL и корректный + `DATABASE_URL`; SQLite не заменяет проверку production migration chain. +- `run.sh` может автоматически создать локальный `backend/.env` и секреты; не коммитить + этот файл и не переносить его секреты в Docker/E2E конфигурацию. +- Если используется `docker compose`, переменная `SERVICE_JWT` обязательна для backend и + agent; Compose намеренно завершается без нее. + ## AXIOM doc-gen Генерация документации (Doxygen/JSDoc) из семантических контрактов выполняется diff --git a/backend/alembic/versions/b9c0d1e2f3a4_add_investigation_evidence_snapshot.py b/backend/alembic/versions/b9c0d1e2f3a4_add_investigation_evidence_snapshot.py new file mode 100644 index 000000000..b4284af7e --- /dev/null +++ b/backend/alembic/versions/b9c0d1e2f3a4_add_investigation_evidence_snapshot.py @@ -0,0 +1,28 @@ +# #region Alembic.ScenarioInvestigationEvidence [C:2] [TYPE Module] [SEMANTICS alembic,scenario,investigation,evidence] +"""add immutable evidence snapshots to scenario investigation projections""" +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "b9c0d1e2f3a4" +down_revision: str | Sequence[str] | None = "a8b9c0d1e2f3" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column("scenario_investigation_queue", sa.Column("evidence_snapshot", sa.JSON(), nullable=False, server_default=sa.text("'{}'"))) + op.add_column("scenario_investigation_cases", sa.Column("evidence_snapshot", sa.JSON(), nullable=False, server_default=sa.text("'{}'"))) + op.add_column("scenario_investigation_cases", sa.Column("linked_run_ids", sa.JSON(), nullable=False, server_default=sa.text("'[]'"))) + op.add_column("scenario_investigation_cases", sa.Column("owner_id", sa.String(length=128), nullable=True)) + op.create_index("ix_scenario_investigation_cases_owner_id", "scenario_investigation_cases", ["owner_id"]) + + +def downgrade() -> None: + op.drop_index("ix_scenario_investigation_cases_owner_id", table_name="scenario_investigation_cases") + op.drop_column("scenario_investigation_cases", "owner_id") + op.drop_column("scenario_investigation_cases", "linked_run_ids") + op.drop_column("scenario_investigation_cases", "evidence_snapshot") + op.drop_column("scenario_investigation_queue", "evidence_snapshot") +# #endregion Alembic.ScenarioInvestigationEvidence diff --git a/backend/alembic/versions/c1d2e3f4a5b6_add_scenario_live_execution_binding.py b/backend/alembic/versions/c1d2e3f4a5b6_add_scenario_live_execution_binding.py new file mode 100644 index 000000000..552813a75 --- /dev/null +++ b/backend/alembic/versions/c1d2e3f4a5b6_add_scenario_live_execution_binding.py @@ -0,0 +1,41 @@ +# #region Alembic.ScenarioLiveExecutionBinding [C:3] [TYPE Module] [SEMANTICS alembic,migration,scenario,execution,live-binding] +# @ingroup Alembic +# @BRIEF Add nullable immutable live-execution binding identity fields to durable scenario runs. +# @RELATION DEPENDS_ON -> [Models.ScenarioExecution.Run] +# @INVARIANT Existing runs remain valid with no binding and therefore fail closed as unavailable live I/O. +# @RATIONALE The new fields are nullable and additive so historical runs preserve their original lifecycle evidence. +# @REJECTED Backfilling binding authority from environment_id was rejected — an environment label is not a pinned RLS/principal/model capability. +"""add nullable ScenarioRun live execution binding identity snapshot (044)""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "c1d2e3f4a5b6" +down_revision: str | Sequence[str] | None = "b9c0d1e2f3a4" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +# #region Alembic.ScenarioLiveExecutionBinding.Upgrade [C:4] [TYPE Function] [SEMANTICS alembic,migration,scenario,live-binding,additive] +# @BRIEF Add nullable persisted live-binding identity fields without deriving legacy authority. +# @POST Existing scenario runs retain valid null binding fields; a lookup index exists for binding_ref. +# @SIDE_EFFECT Schema mutation: two nullable columns and one index on scenario_runs. +def upgrade() -> None: + op.add_column("scenario_runs", sa.Column("live_execution_binding_ref", sa.String(128), nullable=True)) + op.add_column("scenario_runs", sa.Column("live_execution_binding_snapshot", sa.JSON(), nullable=True)) + op.create_index("ix_scenario_runs_live_execution_binding_ref", "scenario_runs", ["live_execution_binding_ref"]) +# #endregion Alembic.ScenarioLiveExecutionBinding.Upgrade + + +# #region Alembic.ScenarioLiveExecutionBinding.Downgrade [C:3] [TYPE Function] [SEMANTICS alembic,migration,scenario,live-binding,rollback] +# @BRIEF Remove only the additive live-binding fields in reverse dependency order. +# @SIDE_EFFECT Schema mutation: binding-ref index and nullable columns are removed. +def downgrade() -> None: + op.drop_index("ix_scenario_runs_live_execution_binding_ref", table_name="scenario_runs") + op.drop_column("scenario_runs", "live_execution_binding_snapshot") + op.drop_column("scenario_runs", "live_execution_binding_ref") +# #endregion Alembic.ScenarioLiveExecutionBinding.Downgrade +# #endregion Alembic.ScenarioLiveExecutionBinding diff --git a/backend/alembic/versions/d2e3f4a5b6c7_add_scenario_artifact_attempt_projection.py b/backend/alembic/versions/d2e3f4a5b6c7_add_scenario_artifact_attempt_projection.py new file mode 100644 index 000000000..686657d72 --- /dev/null +++ b/backend/alembic/versions/d2e3f4a5b6c7_add_scenario_artifact_attempt_projection.py @@ -0,0 +1,50 @@ +# #region Alembic.ScenarioArtifactAttemptProjection [C:3] [TYPE Module] [SEMANTICS alembic,scenario,artifact,retry,provenance] +# @ingroup Alembic +# @BRIEF Add additive retry-attempt projection fields without rewriting historical artifact rows. +# @RELATION DEPENDS_ON -> [Models.ScenarioExecution.Artifact] +# @INVARIANT Existing evidence defaults to active and remains inspectable; retries retire only +# future step-bound projections, never content/digest audit data. +# @REJECTED Deleting or backfilling artifact content for retry provenance was rejected — legacy +# rows remain valid historical evidence with nullable step/attempt linkage. +"""add active ScenarioArtifact attempt projection for retry closure (044)""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "d2e3f4a5b6c7" +down_revision: str | Sequence[str] | None = "c1d2e3f4a5b6" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +# #region Alembic.ScenarioArtifactAttemptProjection.Upgrade [C:4] [TYPE Function] [SEMANTICS alembic,scenario,artifact,retry,additive] +# @BRIEF Add nullable producer linkage and an active projection flag for durable retry evidence. +# @POST Legacy rows are active=true; no historical SHA/ref is changed or removed. +# @SIDE_EFFECT Schema mutation: two nullable provenance columns, active projection flag, invalidation timestamp, and lookup indexes. +def upgrade() -> None: + op.add_column("scenario_artifacts", sa.Column("logical_step_id", sa.String(128), nullable=True)) + op.add_column("scenario_artifacts", sa.Column("attempt", sa.Integer(), nullable=True)) + op.add_column( + "scenario_artifacts", + sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true()), + ) + op.add_column("scenario_artifacts", sa.Column("invalidated_at", sa.DateTime(), nullable=True)) + op.create_index("ix_scenario_artifacts_logical_step_id", "scenario_artifacts", ["logical_step_id"]) + op.create_index("ix_scenario_artifacts_is_active", "scenario_artifacts", ["is_active"]) +# #endregion Alembic.ScenarioArtifactAttemptProjection.Upgrade + + +# #region Alembic.ScenarioArtifactAttemptProjection.Downgrade [C:3] [TYPE Function] [SEMANTICS alembic,scenario,artifact,retry,rollback] +# @BRIEF Remove only additive projection metadata; retained evidence rows and digests are untouched. +def downgrade() -> None: + op.drop_index("ix_scenario_artifacts_is_active", table_name="scenario_artifacts") + op.drop_index("ix_scenario_artifacts_logical_step_id", table_name="scenario_artifacts") + op.drop_column("scenario_artifacts", "invalidated_at") + op.drop_column("scenario_artifacts", "is_active") + op.drop_column("scenario_artifacts", "attempt") + op.drop_column("scenario_artifacts", "logical_step_id") +# #endregion Alembic.ScenarioArtifactAttemptProjection.Downgrade +# #endregion Alembic.ScenarioArtifactAttemptProjection diff --git a/backend/alembic/versions/e3f4a5b6c7d8_add_scenario_cancel_drain_deadline.py b/backend/alembic/versions/e3f4a5b6c7d8_add_scenario_cancel_drain_deadline.py new file mode 100644 index 000000000..a9ff05406 --- /dev/null +++ b/backend/alembic/versions/e3f4a5b6c7d8_add_scenario_cancel_drain_deadline.py @@ -0,0 +1,43 @@ +# #region Alembic.ScenarioCancelDrainDeadline [C:3] [TYPE Module] [SEMANTICS alembic,scenario,execution,cancel,drain] +# @ingroup Alembic +# @BRIEF Add nullable persisted cancellation timing so a worker/scheduler can finish a bounded drain after restart. +# @RELATION DEPENDS_ON -> [Models.ScenarioExecution.Run] +# @INVARIANT Existing runs retain null timing fields; only a cancel request pins a deadline. +# @REJECTED Deriving a deadline from mutable target snapshot or an in-memory process timer was rejected. +"""add durable ScenarioRun cancellation drain timing (044)""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "e3f4a5b6c7d8" +down_revision: str | Sequence[str] | None = "d2e3f4a5b6c7" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +# #region Alembic.ScenarioCancelDrainDeadline.Upgrade [C:4] [TYPE Function] [SEMANTICS alembic,scenario,execution,cancel,additive] +# @BRIEF Add nullable request/deadline timestamps and a deadline lookup index. +# @POST Historical ScenarioRuns preserve null timing until explicitly cancelled. +# @SIDE_EFFECT Schema mutation: two nullable lifecycle timestamps and one index. +def upgrade() -> None: + op.add_column("scenario_runs", sa.Column("cancel_requested_at", sa.DateTime(), nullable=True)) + op.add_column("scenario_runs", sa.Column("cancel_drain_deadline_at", sa.DateTime(), nullable=True)) + op.create_index( + "ix_scenario_runs_cancel_drain_deadline_at", + "scenario_runs", + ["cancel_drain_deadline_at"], + ) +# #endregion Alembic.ScenarioCancelDrainDeadline.Upgrade + + +# #region Alembic.ScenarioCancelDrainDeadline.Downgrade [C:3] [TYPE Function] [SEMANTICS alembic,scenario,execution,cancel,rollback] +# @BRIEF Remove only additive timing metadata; no run/step/evidence history is altered. +def downgrade() -> None: + op.drop_index("ix_scenario_runs_cancel_drain_deadline_at", table_name="scenario_runs") + op.drop_column("scenario_runs", "cancel_drain_deadline_at") + op.drop_column("scenario_runs", "cancel_requested_at") +# #endregion Alembic.ScenarioCancelDrainDeadline.Downgrade +# #endregion Alembic.ScenarioCancelDrainDeadline diff --git a/backend/src/api/routes/dashboard_testing/scenario_analytics.py b/backend/src/api/routes/dashboard_testing/scenario_analytics.py index 41d9fa97c..ddfdd004b 100644 --- a/backend/src/api/routes/dashboard_testing/scenario_analytics.py +++ b/backend/src/api/routes/dashboard_testing/scenario_analytics.py @@ -13,7 +13,8 @@ from pydantic import BaseModel from src.dependencies import get_current_user, get_db, has_permission from src.models.scenario_investigation import InvestigationCase, InvestigationQueueItem from src.services.dashboard_testing.analytics.aggregates import case_detail, recurring_failures, scenario_trends -from src.services.dashboard_testing.analytics.investigation import open_case, set_disposition +from src.services.dashboard_testing.analytics.flakiness import ScenarioAnalyticsClient +from src.services.dashboard_testing.analytics.investigation import can_access_case, open_case, set_disposition from src.services.dashboard_testing.registry.health import get_health_badge router = APIRouter(prefix="/api/scenario-analytics", tags=["scenario-analytics"]) @@ -25,6 +26,8 @@ _TRIAGE = Depends(has_permission("scenario:result", "TRIAGE")) class DispositionRequest(BaseModel): disposition: str expected_version: int + verification_evidence: dict[str, Any] | None = None + rationale: str = "" # #region Api.ScenarioAnalytics.Routes.Serialize [C:2] [TYPE Function] [SEMANTICS scenario,analytics,serialize,projection] @@ -50,6 +53,7 @@ def _case_dict(case: InvestigationCase) -> dict[str, Any]: "status": case.status, "disposition": case.disposition, "decision_version": case.decision_version, + "owner_id": case.owner_id, "created_at": case.created_at.isoformat() if case.created_at else None, } # #endregion Api.ScenarioAnalytics.Routes.Serialize @@ -74,11 +78,13 @@ def open_queue_case(fingerprint: str, scenario_id: str, db=_DB, current_user=_US raise HTTPException(status_code=409, detail={"code": "OPEN_CASE_CONFLICT", "detail": str(exc)}) from exc @router.get("/cases/{case_id}") -def get_case(case_id: str, db=_DB, _view=_VIEW): +def get_case(case_id: str, db=_DB, current_user=_USER, _view=_VIEW): result = case_detail(db, case_id) if result is None: raise HTTPException(status_code=404, detail={"code": "CASE_NOT_FOUND"}) case, actions = result["case"], result["actions"] + if not can_access_case(case, actor_id=str(current_user.id), triage=False): + raise HTTPException(status_code=403, detail={"code": "CASE_ACL_DENIED"}) return {"case": _case_dict(case), "actions": actions} @router.post("/cases/{case_id}/disposition") @@ -87,7 +93,7 @@ def disposition(case_id: str, body: DispositionRequest, db=_DB, current_user=_US if existing is None: raise HTTPException(status_code=404, detail={"code": "CASE_NOT_FOUND"}) try: - result = set_disposition(db, case_id, disposition=body.disposition, expected_version=body.expected_version, actor_id=str(current_user.id)) + result = set_disposition(db, case_id, disposition=body.disposition, expected_version=body.expected_version, actor_id=str(current_user.id), verification_evidence=body.verification_evidence, rationale=body.rationale) db.commit() return _case_dict(result) except ValueError as exc: @@ -96,7 +102,8 @@ def disposition(case_id: str, body: DispositionRequest, db=_DB, current_user=_US @router.get("/scenarios/{scenario_id}/health") def health(scenario_id: str, db=_DB, _view=_VIEW): - result = get_health_badge(db, scenario_id) + client = ScenarioAnalyticsClient(db) + result = get_health_badge(db, scenario_id, analytics_client=client) if result is None: raise HTTPException(status_code=404, detail={"code": "SCENARIO_NOT_FOUND"}) return result diff --git a/backend/src/api/routes/dashboard_testing/scenario_automation.py b/backend/src/api/routes/dashboard_testing/scenario_automation.py index f0f965172..d64d1c4fd 100644 --- a/backend/src/api/routes/dashboard_testing/scenario_automation.py +++ b/backend/src/api/routes/dashboard_testing/scenario_automation.py @@ -5,9 +5,13 @@ # @RELATION DEPENDS_ON -> [ScenarioAutomation.Schedule] # @RELATION DEPENDS_ON -> [ScenarioAutomation.Metrics.Aggregate] # @RELATION DEPENDS_ON -> [ScenarioExecution.Runner.Start] +# @RELATION DEPENDS_ON -> [ScenarioExecution.Runner.QueuedDispatch] +# @RELATION DEPENDS_ON -> [ScenarioExecution.EnvironmentPolicy.Resolve] # @INVARIANT Mutations require scenario:automation MANAGE; direct triggers require TRIGGER -# (+ PROD for PROD-classified environments). PROD classification is server-owned +# (+ automation PROD and scenario RUN_PROD for PROD-classified environments). PROD classification is server-owned # (ConfigManager stage/is_production) — never derived from the environment name. +# @INVARIANT 046 routes pass a server-owned automation origin to 044. A revision with a human +# step is rejected as manual-run-only before a ScenarioRun, gate, or notification exists. # @REJECTED A parallel scheduler was rejected — this surface persists configuration; APScheduler # job registration stays server-owned via the 037 framework. # @REJECTED str(environment_id).startswith("prod") PROD classification was rejected — a client-name @@ -19,10 +23,14 @@ from typing import Any from fastapi import APIRouter, Depends, Header, HTTPException, status from pydantic import BaseModel, Field -from src.dependencies import get_config_manager, get_current_user, get_db, has_permission +from src.dependencies import get_config_manager, get_current_user, get_db, get_scheduler_service, has_permission from src.models.scenario_automation import AutomationPolicy, ScenarioNotificationEvent, ScenarioSchedule, ScenarioTriggerRule from src.services.dashboard_testing.automation.metrics import automation_metrics from src.services.dashboard_testing.automation.retention import tier_limits +from src.services.dashboard_testing.automation.trigger import dispatch_trigger_event +from src.services.dashboard_testing.execution.environment_policy import ( + resolve_environment_execution_policy, +) from src.services.dashboard_testing.execution.runner import start_run router = APIRouter(prefix="/api/scenario-automation", tags=["scenario-automation"]) @@ -47,6 +55,8 @@ class ScheduleRequest(BaseModel): timezone: str = "UTC" policy_id: str | None = None missed_execution_policy: str = "skip" + max_instances: int = Field(default=1, ge=1) + misfire_grace_time: int = Field(default=300, ge=0) enabled: bool = True # #endregion Api.ScenarioAutomation.ScheduleRequest @@ -88,6 +98,16 @@ class ApiTriggerRequest(BaseModel): # #endregion Api.ScenarioAutomation.ApiTriggerRequest +# #region Api.ScenarioAutomation.EventDispatchRequest [C:1] [TYPE Class] [SEMANTICS scenario,automation,api,trigger,event] +# @ingroup Api +# @BRIEF Server-owned typed event envelope accepted by the 046 trigger dispatcher. +class EventDispatchRequest(BaseModel): + type: str + fingerprint: str + overlap: bool = False +# #endregion Api.ScenarioAutomation.EventDispatchRequest + + def _validate_rule_trigger(trigger: str) -> None: if trigger not in TRIGGER_TYPES: raise HTTPException(status_code=422, detail={"code": "INVALID_TRIGGER", "detail": f"trigger must be one of {sorted(TRIGGER_TYPES)}"}) @@ -98,30 +118,12 @@ def _validate_missed_policy(policy: str) -> None: raise HTTPException(status_code=422, detail={"code": "INVALID_MISSED_POLICY", "detail": f"missed_execution_policy must be one of {sorted(MISSED_POLICIES)}"}) -# #region Api.ScenarioAutomation.ClassifyProdEnvironment [C:3] [TYPE Function] [SEMANTICS scenario,automation,api,env,prod,classification] -# @ingroup Api -# @BRIEF Server-owned PROD classification of a trigger environment from ConfigManager -# (Environment.stage == PROD or Environment.is_production) — never a name heuristic. -# @PRE config_manager resolves the environment by id/name; unknown environments are refused. -# @POST True for PROD-classified environments; raises 422 ENVIRONMENT_NOT_FOUND when the -# environment is not configured (the server cannot classify it). -# @RATIONALE The trigger previously classified PROD with str(environment_id).startswith("prod") — -# a client-name heuristic that misclassifies e.g. "production-backup" and is spoofable -# by naming an environment to dodge the PROD gate. The server-owned classification is -# ConfigManager.get_environment() which carries stage (DEV/PREPROD/PROD) + is_production. -# @REJECTED Keeping startswith("prod") was rejected — it is not server-owned, is trivially bypassed -# by environment naming, and contradicts the 036 PROD-gate authority model. -def _classify_prod_environment(environment_id: str, config_manager) -> bool: - env = config_manager.get_environment(environment_id) if config_manager is not None else None - if env is None: - raise HTTPException(status_code=422, detail={"code": "ENVIRONMENT_NOT_FOUND", "detail": f"environment {environment_id!r} is not configured; cannot classify PROD"}) - stage = str(getattr(env, "stage", "") or "").upper() - return bool(getattr(env, "is_production", False)) or stage == "PROD" -# #endregion Api.ScenarioAutomation.ClassifyProdEnvironment - - # #region Api.ScenarioAutomation.Schedules [C:3] [TYPE Block] [SEMANTICS scenario,automation,api,schedule,crud] # @ingroup Api +# @BRIEF Persist schedule configuration and register/remove its server-owned scheduler job. +# @RELATION DEPENDS_ON -> [ScenarioAutomation.Schedule.DeriveApsParams] +# @INVARIANT Schedule CRUD may register work but never claims or executes a queued ScenarioRun; +# the separate 044 queued dispatcher remains the sole initial execution authority. @router.get("/schedules") def list_schedules(db=_DB): return db.query(ScenarioSchedule).order_by(ScenarioSchedule.created_at.desc()).all() @@ -134,6 +136,20 @@ def create_schedule(body: ScheduleRequest, db=_DB, _perm=_MANAGE): db.add(schedule) db.commit() db.refresh(schedule) + if schedule.enabled: + get_scheduler_service().add_scenario_job( + schedule_id=schedule.id, + scenario_id=schedule.scenario_id, + revision_policy=schedule.revision_policy, + revision_id=schedule.revision_id, + environment_id=schedule.environment_id, + cron_expr=schedule.cron_expr, + timezone=schedule.timezone, + policy_id=schedule.policy_id, + misfire_grace_time=schedule.misfire_grace_time, + max_instances=schedule.max_instances, + missed_execution_policy=schedule.missed_execution_policy, + ) return schedule @@ -147,6 +163,22 @@ def update_schedule(schedule_id: str, body: ScheduleRequest, db=_DB, _perm=_MANA setattr(schedule, key, value) db.commit() db.refresh(schedule) + if schedule.enabled: + get_scheduler_service().add_scenario_job( + schedule_id=schedule.id, + scenario_id=schedule.scenario_id, + revision_policy=schedule.revision_policy, + revision_id=schedule.revision_id, + environment_id=schedule.environment_id, + cron_expr=schedule.cron_expr, + timezone=schedule.timezone, + policy_id=schedule.policy_id, + misfire_grace_time=schedule.misfire_grace_time, + max_instances=schedule.max_instances, + missed_execution_policy=schedule.missed_execution_policy, + ) + else: + get_scheduler_service().remove_scenario_job(schedule.id) return schedule @@ -155,6 +187,7 @@ def delete_schedule(schedule_id: str, db=_DB, _perm=_MANAGE): schedule = db.query(ScenarioSchedule).filter(ScenarioSchedule.id == schedule_id).first() if schedule is None: raise HTTPException(status_code=404, detail={"code": "SCHEDULE_NOT_FOUND"}) + get_scheduler_service().remove_scenario_job(schedule.id) db.delete(schedule) db.commit() return None @@ -249,6 +282,44 @@ def list_notifications(limit: int = 100, db=_DB): # #endregion Api.ScenarioAutomation.Notifications +# #region Api.ScenarioAutomation.EventDispatch [C:3] [TYPE Function] [SEMANTICS scenario,automation,api,event,trigger,persistence] +# @ingroup Api +# @BRIEF Persist policy-allowed event-triggered runs without request-time execution. +# @RELATION CALLS -> [ScenarioAutomation.Trigger.Dispatch] +# @RELATION DEPENDS_ON -> [ScenarioExecution.Runner.QueuedDispatch] +# @POST Returns accepted run ids only after persistence; adapter execution waits for the server dispatcher CAS. +# @INVARIANT A malformed automated human plan is refused before creation; this HTTP route never +# fabricates manual authority or invokes a walker. +# @INVARIANT ConfigManager classifies every persisted trigger target before start; an event payload +# cannot provide an is_prod/environment_class override or create an unknown target. +@router.post("/events/dispatch", status_code=status.HTTP_202_ACCEPTED) +def api_dispatch_event( + body: EventDispatchRequest, + db=_DB, + _perm=_TRIGGER, + config_manager=_CONFIG_MANAGER, +): + if body.type not in TRIGGER_TYPES: + raise HTTPException(status_code=422, detail={"code": "INVALID_TRIGGER"}) + try: + created = dispatch_trigger_event( + db, + {"type": body.type, "fingerprint": body.fingerprint, "overlap": body.overlap}, + config_manager=config_manager, + ) + db.commit() + return {"run_ids": created, "count": len(created)} + except ValueError as exc: + db.rollback() + if str(exc) == "ENVIRONMENT_NOT_CONFIGURED": + raise HTTPException( + status_code=422, + detail={"code": str(exc), "detail": str(exc)}, + ) from exc + raise +# #endregion Api.ScenarioAutomation.EventDispatch + + # #region Api.ScenarioAutomation.Metrics [C:3] [TYPE Function] [SEMANTICS scenario,automation,api,metrics] # @ingroup Api # @BRIEF Operational metrics over persisted schedules, trigger rules, runs and notifications. @@ -288,10 +359,20 @@ def retention_defaults(): # @ingroup Api # @BRIEF External API run trigger (SCAUTO-FR-011) — starts a scenario run with an # Idempotency-Key; PROD-classified environments (server-owned ConfigManager -# classification, never a name heuristic) require the PROD scope. +# classification, never a name heuristic) require automation PROD + scenario RUN_PROD. # @PRE Idempotency-Key header is required; scenario_id path parameter names the scenario; # the environment must be configured server-side or the trigger is refused (422). # @POST Returns 202 with run_id/status; 409 IDEMPOTENCY_KEY_REUSED on changed request. +# @POST A human-containing revision returns 409 AUTOMATION_INELIGIBLE_HUMAN_STEP and creates no run. +# @RELATION CALLS -> [ScenarioExecution.Runner.Start] +# @RELATION CALLS -> [ScenarioExecution.EnvironmentPolicy.Resolve] +# @RELATION DEPENDS_ON -> [ScenarioExecution.Runner.QueuedDispatch] +# @SIDE_EFFECT Commits only a permitted ScenarioRun; rejection rolls back without a run, gate, notification, or queue row. +# @INVARIANT API automation origin is server-owned and cannot consume or substitute a HumanCheckpoint or ActionApprovalGate. +# @INVARIANT HTTP trigger/replay is persistence-only: its accepted row remains queued/pending until +# the separate server dispatcher wins the queued->running status CAS. +# @INVARIANT PROD classification and RUN_PROD validation complete before start_run; a valid +# automation intent persists pending_approval plus its ActionApprovalGate, never queued. @router.post("/scenarios/{scenario_id}/trigger", status_code=status.HTTP_202_ACCEPTED) def api_trigger_scenario( scenario_id: str, @@ -302,9 +383,18 @@ def api_trigger_scenario( _perm=_TRIGGER, config_manager=_CONFIG_MANAGER, ): - is_prod = _classify_prod_environment(body.environment_id, config_manager) - if is_prod: + try: + environment_policy = resolve_environment_execution_policy( + body.environment_id, config_manager + ) + except ValueError as exc: + raise HTTPException( + status_code=422, + detail={"code": str(exc), "detail": str(exc)}, + ) from exc + if environment_policy.is_prod: has_permission("scenario:automation", "PROD")(current_user=current_user) + has_permission("scenario", "RUN_PROD")(current_user=current_user) revision_id = body.revision_id or "" try: run = start_run( @@ -315,7 +405,8 @@ def api_trigger_scenario( body.environment_id, actor=str(getattr(current_user, "id", "") or getattr(current_user, "username", "")), idempotency_key=idempotency_key, - is_prod=is_prod, + config_manager=config_manager, + trigger_source="api", ) db.commit() except PermissionError as exc: @@ -323,7 +414,11 @@ def api_trigger_scenario( raise HTTPException(status_code=403, detail={"code": "PROD_APPROVAL_REQUIRED", "detail": str(exc)}) from exc except ValueError as exc: db.rollback() - code = "IDEMPOTENCY_KEY_REUSED" if "IDEMPOTENCY" in str(exc) else "RUN_START_CONFLICT" + code = ( + "AUTOMATION_INELIGIBLE_HUMAN_STEP" + if str(exc) == "AUTOMATION_INELIGIBLE_HUMAN_STEP" + else "IDEMPOTENCY_KEY_REUSED" if "IDEMPOTENCY" in str(exc) else "RUN_START_CONFLICT" + ) raise HTTPException(status_code=409, detail={"code": code, "detail": str(exc)}) from exc return {"run_id": run.id, "status": run.status, "scenario_id": scenario_id} # #endregion Api.ScenarioAutomation.DirectTrigger diff --git a/backend/src/api/routes/dashboard_testing/scenario_runs.py b/backend/src/api/routes/dashboard_testing/scenario_runs.py index ea848d697..07397fc07 100644 --- a/backend/src/api/routes/dashboard_testing/scenario_runs.py +++ b/backend/src/api/routes/dashboard_testing/scenario_runs.py @@ -9,7 +9,8 @@ # @RELATION DEPENDS_ON -> [ScenarioExecution.Events] # @INVARIANT Static paths (/compare) are registered before dynamic /{run_id} — FastAPI matches # routes in declaration order, so /compare must never be captured as a run id. -# @INVARIANT Start requires scenario:run; a PROD start additionally requires scenario:run:prod. +# @INVARIANT Start requires scenario:run; server-owned PROD classification additionally requires +# scenario:run:prod. Request bodies cannot select the classification. # @INVARIANT All responses are explicit dicts (no ORM leakage); datetimes ISO-8601. from __future__ import annotations @@ -20,11 +21,14 @@ from fastapi.responses import StreamingResponse from pydantic import BaseModel from src.core.database import get_db -from src.dependencies import get_current_user, has_permission +from src.dependencies import get_config_manager, get_current_user, has_permission from src.models.scenario_approval import ActionApprovalGate from src.models.scenario_run import ScenarioRun, ScenarioStepRun from src.services.dashboard_testing.execution.approval import decide_approval_gate from src.services.dashboard_testing.execution.comparison import compare_runs +from src.services.dashboard_testing.execution.environment_policy import ( + resolve_environment_execution_policy, +) from src.services.dashboard_testing.execution.events import derive_run_events, render_sse_frames from src.services.dashboard_testing.execution.lifecycle import ( cancel_run, @@ -33,7 +37,12 @@ from src.services.dashboard_testing.execution.lifecycle import ( retry_step, ) from src.services.dashboard_testing.execution.result import build_result -from src.services.dashboard_testing.execution.runner import start_run +from src.services.dashboard_testing.execution.runner import ( + continue_after_human_decision, + continue_after_infrastructure_resume, + continue_after_step_retry, + start_run, +) runs_router = APIRouter(prefix="/api/scenario-runs", tags=["scenario-runs"]) history_router = APIRouter(prefix="/api/scenarios", tags=["scenario-runs-history"]) @@ -41,6 +50,7 @@ history_router = APIRouter(prefix="/api/scenarios", tags=["scenario-runs-history _DB = Depends(get_db) _USER = Depends(get_current_user) _RUN_PERMISSION = Depends(has_permission("scenario", "RUN")) +_CONFIG_MANAGER = Depends(get_config_manager) # PROD-classified run authority (catalog pair ("scenario", "RUN_PROD"), display scenario:run_prod). # Guards PROD starts AND the ActionApprovalGate decision endpoint — never the bare RUN scope. _RUN_PROD_PERMISSION = Depends(has_permission("scenario", "RUN_PROD")) @@ -51,7 +61,11 @@ class StartRunRequest(BaseModel): revision_id: str environment_id: str params: dict[str, Any] = {} + # Compatibility-only: execution authority comes from ConfigManager, never this body field. is_prod: bool = False + dashboard_release_id: str | None = None + baseline_set: str | None = None + execution_toggles: dict[str, bool] = {} class HumanDecisionRequest(BaseModel): disposition: str @@ -108,7 +122,18 @@ def _require_run_prod(current_user) -> None: # #region Api.ScenarioExecution.Routes.Start [C:4] [TYPE Function] [SEMANTICS api,scenario,run,start,idempotency,rbac] # @ingroup Api # @BRIEF POST /api/scenario-runs — create a run (Idempotency-Key); PROD requires scenario:run:prod. -# @POST 201 with run dict; 409 IDEMPOTENCY_KEY_REUSED / RUN_START_CONFLICT; 403 on missing RBAC. +# @RELATION CALLS -> [ScenarioExecution.Runner.Start] +# @RELATION CALLS -> [ScenarioExecution.EnvironmentPolicy.Resolve] +# @POST 201 with queued/pending-approval run; 409 IDEMPOTENCY_KEY_REUSED / RUN_START_CONFLICT; +# 403 on missing RBAC; 422 ENVIRONMENT_NOT_CONFIGURED before any durable side effect. +# @INVARIANT This authenticated analyst route supplies the trusted manual origin itself; clients +# cannot select trigger_source in StartRunRequest. +# @INVARIANT HTTP start/replay is persistence-only: it creates or returns a queued/pending row but +# never creates a step, lease, notification, queue signal, or adapter invocation. +# @INVARIANT PROD authority and gate selection use server ConfigManager policy even when the +# compatibility is_prod field is omitted, false, or true for a non-PROD environment. +# @REJECTED Treating StartRunRequest.is_prod as PROD authority was rejected: a caller could suppress +# a real gate or manufacture one for PREPROD. @runs_router.post("", status_code=status.HTTP_201_CREATED) def api_start_run( body: StartRunRequest, @@ -116,16 +141,23 @@ def api_start_run( db=_DB, current_user=_USER, _=_RUN_PERMISSION, + config_manager=_CONFIG_MANAGER, ): - if body.is_prod: - _require_run_prod(current_user) try: + if resolve_environment_execution_policy( + body.environment_id, config_manager + ).is_prod: + _require_run_prod(current_user) run = start_run( db, body.scenario_id, body.revision_id, body.params, body.environment_id, actor=str(getattr(current_user, "id", "unknown")), idempotency_key=idempotency_key, - is_prod=body.is_prod, - approval_granted=body.is_prod, + config_manager=config_manager, + auto_advance=False, + dashboard_release_id=body.dashboard_release_id, + baseline_set=body.baseline_set, + execution_toggles=body.execution_toggles, + trigger_source="manual", ) db.commit() return _run_to_dict(run) @@ -134,6 +166,11 @@ def api_start_run( raise HTTPException(status_code=403, detail={"code": "PROD_APPROVAL_REQUIRED", "detail": str(exc)}) from exc except ValueError as exc: db.rollback() + if str(exc) == "ENVIRONMENT_NOT_CONFIGURED": + raise HTTPException( + status_code=422, + detail={"code": str(exc), "detail": str(exc)}, + ) from exc code = "IDEMPOTENCY_KEY_REUSED" if "IDEMPOTENCY" in str(exc) else "RUN_START_CONFLICT" raise HTTPException(status_code=409, detail={"code": code, "detail": str(exc)}) from exc # #endregion Api.ScenarioExecution.Routes.Start @@ -210,11 +247,14 @@ def api_cancel_run(run_id: str, db=_DB, _=_RUN_PERMISSION): # #region Api.ScenarioExecution.Routes.Resume [C:3] [TYPE Function] [SEMANTICS api,scenario,run,resume,token] # @ingroup Api # @BRIEF POST /api/scenario-runs/{run_id}/resume — resume an infrastructure-paused run. -# @POST 200 run dict; 409 for human checkpoints, stale tokens, terminal runs. +# @POST 200 terminal/current run dict after advancing the persisted frontier; 409 for human +# checkpoints, stale tokens, terminal runs or a run that is not paused. +# @INVARIANT This route never consumes a HumanCheckpoint or a PROD ActionApprovalGate. @runs_router.post("/{run_id}/resume") def api_resume_run(run_id: str, body: ResumeRequest, db=_DB, _=_RUN_PERMISSION): try: result = resume_run(db, run_id, resume_token=body.resume_token, resume_reason=body.resume_reason) + continue_after_infrastructure_resume(db, run_id, worker_id="infrastructure-resume") db.commit() return _run_to_dict(result) except ValueError as exc: @@ -226,7 +266,9 @@ def api_resume_run(run_id: str, body: ResumeRequest, db=_DB, _=_RUN_PERMISSION): # #region Api.ScenarioExecution.Routes.HumanDecision [C:3] [TYPE Function] [SEMANTICS api,scenario,run,human,decision,checkpoint] # @ingroup Api # @BRIEF POST /api/scenario-runs/{run_id}/human/decision — resolve a HumanCheckpoint disposition. -# @POST 200 checkpoint; 404 CHECKPOINT_NOT_FOUND; 409 CHECKPOINT_CONFLICT. +# @POST 200 checkpoint after internally continuing ready dependents; 404 CHECKPOINT_NOT_FOUND; +# 409 CHECKPOINT_CONFLICT. +# @INVARIANT This observation checkpoint is distinct from the PROD ActionApprovalGate. @runs_router.post("/{run_id}/human/decision") def api_decide_human(run_id: str, body: HumanDecisionRequest, db=_DB, current_user=_USER, _=_RUN_PERMISSION): from src.models.scenario_checkpoint import HumanCheckpoint @@ -240,6 +282,9 @@ def api_decide_human(run_id: str, body: HumanDecisionRequest, db=_DB, current_us expected_version=body.expected_version, actor_id=str(getattr(current_user, "id", "unknown")), comment=body.comment, ) + continue_after_human_decision( + db, run_id, worker_id=f"human-decision-{getattr(current_user, 'id', 'unknown')}", + ) db.commit() return { "id": result.id, @@ -265,6 +310,9 @@ def api_decide_human(run_id: str, body: HumanDecisionRequest, db=_DB, current_us # @PRE Caller holds scenario RUN_PROD (the gate's required_permission, display scenario:run_prod). # @POST 200 gate dict + run status; 404 GATE_NOT_FOUND; 409 GATE_ALREADY_DECIDED (CAS conflict); # 422 INVALID_GATE_DECISION. +# @RELATION DEPENDS_ON -> [ScenarioExecution.Runner.QueuedDispatch] +# @INVARIANT Approval CAS can expose an approved run as queued, but only the separate server +# dispatcher may claim and execute its initial frontier. # @TEST_EDGE approve -> queued; deny -> blocked; already_decided -> 409; missing_gate -> 404; # missing_run_prod_scope -> 403; invalid_decision -> 422. @runs_router.post("/{run_id}/approval/decision") @@ -309,11 +357,14 @@ def api_decide_approval_gate(run_id: str, body: ApprovalDecisionRequest, db=_DB, # #region Api.ScenarioExecution.Routes.Retry [C:3] [TYPE Function] [SEMANTICS api,scenario,run,retry,step] # @ingroup Api # @BRIEF POST /api/scenario-runs/{run_id}/steps/{logical_step_id}/retry — retry + downstream closure. -# @POST 200 list of affected steps; 409 RETRY_CONFLICT. +# @POST 200 list of re-executed affected steps; 409 RETRY_CONFLICT. +# @INVARIANT The route first invalidates the entire retry closure, then advances only its queued +# frontier; historical artifacts remain audit rows but cannot project into the result. @runs_router.post("/{run_id}/steps/{logical_step_id}/retry") def api_retry_step(run_id: str, logical_step_id: str, db=_DB, _=_RUN_PERMISSION): try: result = retry_step(db, run_id, logical_step_id) + continue_after_step_retry(db, run_id, worker_id="api-step-retry") db.commit() return [ { diff --git a/backend/src/app.py b/backend/src/app.py index ea35cbc63..aedcf6f4a 100755 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -3,8 +3,11 @@ # @BRIEF The main entry point for the FastAPI application. # @LAYER API # @RELATION DEPENDS_ON -> [Api.Init.ApiRoutesModule] +# @RELATION CALLS -> [Dependencies.AppDependencies.LiveExecutionComposition] # @INVARIANT All WebSocket connections must be properly cleaned up on disconnect. # @INVARIANT All WebSocket connections must be authenticated via JWT or API key token (see [SEC:C-3]). +# @INVARIANT Startup initializes a fail-closed 044 LiveExecutionCompositionRoot; runtime providers +# must be registered server-side and never inferred from HTTP/run metadata. # @PRE Python environment and dependencies installed; configuration database available. # @POST FastAPI app instance is created, middleware configured, and routes registered. # @SIDE_EFFECT Starts background scheduler and binds network ports for HTTP/WS traffic. @@ -86,19 +89,47 @@ from .core.logger import belief_scope, logger from .core.utils.network import NetworkError from .dependencies import ( get_async_job_runner, + get_config_manager, get_current_user, + get_live_execution_composition_root, get_scheduler_service, get_task_manager, ) from .models.auth import Role, User +# #region App.AppModule.LiveExecutionCompositionBootstrap [C:4] [TYPE Function] [SEMANTICS scenario,execution,composition,startup] +# @BRIEF Register trusted deployment live bindings before the scheduler can dispatch queued ScenarioRuns. +# @RELATION CALLS -> [ScenarioExecution.LiveCompositionRoot.Bootstrap] +# @POST Default dispatcher sees exact configured 037 providers or typed unavailable bindings; HTTP/run +# metadata never creates a live client. +# @INVARIANT Browser and Screenshot providers are separately registered server-side; their absence does +# not prevent startup and remains typed unavailable. +# @REJECTED Deferring Superset provider creation to a queued run was rejected — that would derive +# execution authority from a persisted/request identifier instead of deployment config. +def initialize_live_execution_composition() -> int: + from .services.dashboard_testing.execution.live_composition import ( + bootstrap_live_execution_composition, + ) + + return bootstrap_live_execution_composition( + get_live_execution_composition_root(), + config_manager=get_config_manager(), + run_async=get_async_job_runner().run, + ) +# #endregion App.AppModule.LiveExecutionCompositionBootstrap + + # #region App.AppModule.Lifespan [C:3] [TYPE Function] # @ingroup Module # @BRIEF Async context manager for FastAPI startup/shutdown lifecycle. # @RELATION CALLS -> [Core.Database.InitDb] # @RELATION CALLS -> [Dependencies.AppDependencies] +# @RELATION CALLS -> [App.AppModule.LiveExecutionCompositionBootstrap] # @POST On startup: admin exists, scheduler started. On shutdown: scheduler stopped. +# @INVARIANT The trusted live-composition bootstrap runs after AsyncJobRunner initialization and +# before scheduler startup; queued dispatch therefore cannot observe an unbootstrapped +# default root during normal application startup. # @RATIONALE Alembic migrations removed from lifespan — they now run exclusively # in docker/backend.entrypoint.sh (wait_for_db → alembic upgrade head). # Running migrations in both places added ~5s startup overhead and masked @@ -193,6 +224,9 @@ async def lifespan(app: FastAPI): logger.reason("Initializing AsyncJobRunner") get_async_job_runner() # Initialize singleton with running event loop BEFORE scheduler starts + logger.reason("Bootstrapping fail-closed ScenarioRun live composition root") + live_bindings = initialize_live_execution_composition() + logger.reason("ScenarioRun live composition bootstrap complete", payload={"bindings": live_bindings}) logger.reason("Starting scheduler") scheduler = get_scheduler_service() scheduler.start() diff --git a/backend/src/core/config_models.py b/backend/src/core/config_models.py index 79391334c..f04bda9c2 100755 --- a/backend/src/core/config_models.py +++ b/backend/src/core/config_models.py @@ -17,6 +17,7 @@ import uuid from datetime import datetime, timezone +from typing import Any from pydantic import BaseModel, Field, field_validator @@ -218,6 +219,22 @@ class DatabaseConnection(BaseModel): # #endregion Core.ConfigModels.DatabaseConnection +# #region Core.ConfigModels.ScenarioLiveExecutionBinding [C:3] [TYPE DataClass] [SEMANTICS scenario,execution,live,composition,config] +# @BRIEF Trusted deployment configuration for one exact 044 live Superset binding. +# @RELATION DEPENDS_ON -> [ScenarioExecution.LiveBinding.Identity] +# @INVARIANT This record contains only the persisted identity snapshot and immutable query-model +# snapshot; credentials remain solely in the configured Environment record. +# @INVARIANT `binding_snapshot` is not authority by itself: before startup uses an enabled record it +# must satisfy the exact LiveExecutionBinding identity-field allowlist. +# @REJECTED Environment IDs alone are not a live binding — query/principal/RLS fingerprints must be +# present in binding_snapshot before startup can create a SupersetClient. +class ScenarioLiveExecutionBindingConfig(BaseModel): + enabled: bool = False + binding_snapshot: dict[str, Any] + query_model_snapshot: dict[str, Any] +# #endregion Core.ConfigModels.ScenarioLiveExecutionBinding + + # #region Core.ConfigModels.GlobalSettings [TYPE DataClass] # @ingroup Core # @BRIEF Represents global application settings. @@ -229,6 +246,9 @@ class GlobalSettings(BaseModel): logging: LoggingConfig = Field(default_factory=LoggingConfig) features: FeaturesConfig = Field(default_factory=FeaturesConfig) connections: list[DatabaseConnection] = [] + scenario_live_execution_bindings: list[ScenarioLiveExecutionBindingConfig] = Field( + default_factory=list + ) llm: dict = Field( default_factory=lambda: { "providers": [], diff --git a/backend/src/core/scheduler.py b/backend/src/core/scheduler.py index 57e320ed6..33e34d8cf 100644 --- a/backend/src/core/scheduler.py +++ b/backend/src/core/scheduler.py @@ -5,7 +5,7 @@ # @RELATION DEPENDS_ON -> Core.Manager.TaskManager # @RELATION DEPENDS_ON -> [Core.AsyncJobRunner] import asyncio -from datetime import date, datetime, time, timedelta +from datetime import UTC, date, datetime, time, timedelta from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore from apscheduler.schedulers.background import BackgroundScheduler @@ -42,6 +42,55 @@ def execute_scheduled_lifecycle_retention() -> None: # #endregion Core.Scheduler.ExecuteScheduledLifecycleRetention +# #region Core.Scheduler.ExecuteScenarioCancelFinalizer [C:4] [TYPE Function] [SEMANTICS scheduler,scenario,execution,cancel,drain] +# @BRIEF APScheduler callback that finalizes persisted ScenarioRun cancellations after their bounded drain deadline. +# @RELATION CALLS -> [ScenarioExecution.Lifecycle.CancelFinalizer] +# @POST Expired cancel_requested runs become cancelled in one committed lifecycle transaction. +# @INVARIANT The scheduler never dispatches an executor, decides a HumanCheckpoint, approves a gate, +# or deletes artifact/lease audit history while finalizing a cancelled run. +# @REJECTED Relying on the API process that requested cancellation was rejected — an in-flight I/O +# worker or process restart otherwise leaves cancel_requested work stranded. +def execute_scheduled_scenario_cancel_finalizer() -> None: + db = SessionLocal() + try: + from src.services.dashboard_testing.execution.lifecycle import finalize_expired_cancellations + + terminalized = finalize_expired_cancellations(db) + db.commit() + logger.reason( + "Scenario cancellation drain finalizer completed", + payload={"terminalized": len(terminalized)}, + ) + except Exception as exc: + db.rollback() + logger.explore("Scenario cancellation drain finalizer failed", error=str(exc)) + finally: + db.close() +# #endregion Core.Scheduler.ExecuteScenarioCancelFinalizer + + +# #region Core.Scheduler.ExecuteQueuedScenarioDispatch [C:4] [TYPE Function] [SEMANTICS scheduler,scenario,execution,queue,dispatch] +# @BRIEF Advance durable queued ScenarioRuns through the server worker composition, never an HTTP request. +# @RELATION CALLS -> [ScenarioExecution.Runner.QueuedDispatch] +# @INVARIANT The dispatcher relies on persisted queued->running CAS; overlapping scheduler ticks +# cannot invoke the same run's executor twice. +# @REJECTED Treating HTTP POST as a worker loop was rejected — it would duplicate side effects on replay. +def execute_scheduled_queued_scenario_dispatch() -> None: + db = SessionLocal() + try: + from src.services.dashboard_testing.execution.runner import dispatch_queued_runs + + outcomes = dispatch_queued_runs(db, worker_id="scenario-queue-scheduler") + db.commit() + logger.reason("Queued ScenarioRun dispatch completed", payload={"dispatched": len(outcomes)}) + except Exception as exc: + db.rollback() + logger.explore("Queued ScenarioRun dispatch failed", error=str(exc)) + finally: + db.close() +# #endregion Core.Scheduler.ExecuteQueuedScenarioDispatch + + # #region Core.Scheduler.ExecuteScheduledBackup [C:3] [TYPE Function] [SEMANTICS scheduler,backup,apscheduler,persistence] # @ingroup Core # @BRIEF APScheduler callback for backup jobs that resolves runtime dependencies at execution time. @@ -145,6 +194,75 @@ def execute_scheduled_verification_check() -> None: # #endregion Core.Scheduler.ExecuteScheduledVerificationCheck +# #region Core.Scheduler.ExecuteScheduledScenario [C:4] [TYPE Function] [SEMANTICS scheduler,scenario,apscheduler,cron,auto] +# @defgroup Core APScheduler callback for 046 scenario schedule triggers. +# @BRIEF Resolve the scenario schedule and persist a scheduled run for separate 044 dispatch. +# @RELATION CALLS -> [ScenarioExecution.Runner.Start] +# @RELATION DEPENDS_ON -> [ScenarioExecution.Runner.QueuedDispatch] +# @PRE schedule_id exists in scenario_schedules table. +# @POST A scheduled ScenarioRun row is created queued or pending_approval under the shared 044 +# policy; an adapter may run only after gate approval and the separate dispatcher wins CAS. +# @SIDE_EFFECT Persists ScenarioRun; failures logged but never raised (observability-only). +# @INVARIANT The scheduler passes its server-owned scheduled origin before 044 creates a run; +# a human-containing revision is rejected as manual_run_only rather than created as manual. +# @INVARIANT Scheduler callback/replay is persistence-only and cannot bypass the 044 initial-dispatch CAS. +# @INVARIANT Scheduled PROD classification and its durable ActionApprovalGate are selected only by +# ScenarioExecution.Runner.Start's server-owned EnvironmentPolicy; callback arguments +# cannot downgrade a target or dispatch pending_approval work. +# @RATIONALE Module-level callback follows the same pattern as backup/validation callbacks so +# the persistent job store never serializes dependencies. Dependencies (runner, models) +# are resolved at execution time to avoid import cycles. +# @NOTE start_run is synchronous (SQLAlchemy Session), so BackgroundScheduler thread pool +# executes it directly; no AsyncJobRunner bridge needed. +def execute_scheduled_scenario(schedule_id: str, scenario_id: str, revision_policy: str, revision_id: str | None, environment_id: str, cron_expr: str, timezone: str, policy_id: str | None) -> None: + seed_trace_id() + del cron_expr, timezone, policy_id + db = SessionLocal() + try: + from ..services.dashboard_testing.execution.runner import start_run + + resolved_revision_id = revision_id + if revision_policy == "current": + from src.models.scenario_registry import ScenarioRegistryEntry + entry = db.query(ScenarioRegistryEntry).filter(ScenarioRegistryEntry.scenario_id == scenario_id).first() + if entry and entry.current_revision_id: + resolved_revision_id = entry.current_revision_id + + if not resolved_revision_id: + logger.reason( + "Scheduled scenario skipped: no revision", + payload={"schedule_id": schedule_id, "scenario_id": scenario_id}, + ) + return + + start_run( + db, + scenario_id=scenario_id, + revision_id=resolved_revision_id, + params={}, + environment_id=environment_id, + actor="apscheduler", + idempotency_key=f"scheduled-{schedule_id}-{datetime.now(UTC).isoformat()}", + auto_advance=False, + trigger_source="scheduled", + ) + db.commit() + logger.reason( + "Scheduled scenario run started", + payload={"schedule_id": schedule_id, "scenario_id": scenario_id, "revision_id": resolved_revision_id}, + ) + except Exception as exc: + db.rollback() + logger.explore( + "Scheduled scenario run failed", + error=str(exc), + payload={"schedule_id": schedule_id}, + ) + finally: + db.close() +# #endregion Core.Scheduler.ExecuteScheduledScenario + + # #region Core.Scheduler.SchedulerService [C:3] [TYPE Class] [SEMANTICS scheduler, service, apscheduler] # @defgroup Core Module group. # @BRIEF Provides a service to manage scheduled backup tasks. @@ -189,8 +307,11 @@ class SchedulerService: # #region Core.Scheduler.Start [C:3] [TYPE Function] [SEMANTICS scheduler,service,start] # @ingroup Core # @BRIEF Start the background scheduler and load initial schedules. + # @RELATION DEPENDS_ON -> [Core.Scheduler.ExecuteQueuedScenarioDispatch] # @PRE Scheduler should be initialized. # @POST Scheduler is running and schedules are loaded. + # @INVARIANT The recurring queued-run worker is scheduler-owned; HTTP routes only persist rows + # for its durable CAS claim and never replace this execution boundary. def start(self): with belief_scope("SchedulerService.start"): if not self.scheduler.running: @@ -203,6 +324,22 @@ class SchedulerService: id="agent_lifecycle_retention", replace_existing=True, ) + self.scheduler.add_job( + execute_scheduled_scenario_cancel_finalizer, + IntervalTrigger(seconds=5), + id="scenario_cancel_drain_finalizer", + replace_existing=True, + max_instances=1, + coalesce=True, + ) + self.scheduler.add_job( + execute_scheduled_queued_scenario_dispatch, + IntervalTrigger(seconds=5), + id="scenario_queued_dispatch", + replace_existing=True, + max_instances=1, + coalesce=True, + ) # Log restored jobs from persistent jobstore try: restored_jobs = self.scheduler.get_jobs() @@ -350,6 +487,31 @@ class SchedulerService: except Exception as e: logger.explore("Failed to load validation schedules on startup", error=str(e)) + try: + from src.models.scenario_automation import ScenarioSchedule + + db = SessionLocal() + try: + schedules = db.query(ScenarioSchedule).filter(ScenarioSchedule.enabled.is_(True)).all() + for schedule in schedules: + self.add_scenario_job( + schedule_id=schedule.id, + scenario_id=schedule.scenario_id, + revision_policy=schedule.revision_policy, + revision_id=schedule.revision_id, + environment_id=schedule.environment_id, + cron_expr=schedule.cron_expr, + timezone=schedule.timezone, + policy_id=schedule.policy_id, + misfire_grace_time=schedule.misfire_grace_time, + max_instances=schedule.max_instances, + missed_execution_policy=schedule.missed_execution_policy, + ) + desired_job_ids.add(f"scenario_{schedule.id}") + finally: + db.close() + except Exception as e: + logger.explore("Failed to load scenario schedules on startup", error=str(e)) # Differential cleanup using persistent jobstore: # remove APS jobs that are no longer in the authoritative set (backups + translates + validations) @@ -691,6 +853,94 @@ class SchedulerService: finally: db.close() # #endregion Core.Scheduler.TriggerValidation + # #region Core.Scheduler.AddScenarioJob [C:3] [TYPE Function] [SEMANTICS scheduler,scenario,cron,apscheduler] + # @ingroup Core + # @BRIEF Register a scenario schedule with APScheduler. + # @PRE schedule_id, scenario_id, cron_expr and timezone are valid. + # @POST A new APScheduler job is registered or replaced if it already exists. + # @SIDE_EFFECT Mutates APScheduler state; calls execute_scheduled_scenario on trigger. + # @RELATION DEPENDS_ON -> [EXT:APScheduler:CronTrigger] + # @RELATION CALLS -> [Core.Scheduler.ExecuteScheduledScenario] + # @INVARIANT Registration configures cron delivery only; it does not claim or execute a ScenarioRun. + # @RATIONALE Scenario schedules are managed centrally through SchedulerService so all + # scheduled work follows the same lifecycle (start/stop/reload) regardless of + # job type. Self-registering schedules would bypass centralized management and + # create split-brain state when the scheduler is stopped or reloaded. + # @NOTE Only primitive args are passed so that SQLAlchemyJobStore can reliably persist/restore. + def add_scenario_job( + self, + schedule_id: str, + scenario_id: str, + revision_policy: str, + revision_id: str | None, + environment_id: str, + cron_expr: str, + timezone: str = "UTC", + policy_id: str | None = None, + misfire_grace_time: int = 300, + max_instances: int = 1, + missed_execution_policy: str = "skip", + ) -> None: + with belief_scope( + "SchedulerService.add_scenario_job", + f"schedule_id={schedule_id}, scenario_id={scenario_id}, cron={cron_expr}", + ): + from zoneinfo import ZoneInfo + + job_id_aps = f"scenario_{schedule_id}" + try: + tz = ZoneInfo(timezone) + self.scheduler.add_job( + execute_scheduled_scenario, + CronTrigger.from_crontab(cron_expr, timezone=tz), + id=job_id_aps, + args=[ + schedule_id, + scenario_id, + revision_policy, + revision_id, + environment_id, + cron_expr, + timezone, + policy_id, + ], + replace_existing=True, + max_instances=max_instances, + misfire_grace_time=misfire_grace_time, + coalesce=missed_execution_policy != "queue_all", + ) + logger.reason( + f"Scenario schedule registered: {job_id_aps} ({cron_expr}) [{timezone}]" + ) + except Exception as e: + logger.explore( + "Failed to register scenario schedule", + payload={"job": job_id_aps}, + error=str(e), + ) + # #endregion Core.Scheduler.AddScenarioJob + # #region Core.Scheduler.RemoveScenarioJob [C:2] [TYPE Function] [SEMANTICS scheduler,scenario,remove,apscheduler] + # @ingroup Core + # @BRIEF Remove a scenario schedule from APScheduler. + # @PRE schedule_id is a valid ScenarioSchedule.id. + # @POST The APScheduler job is removed if it exists; silently ignored otherwise. + # @SIDE_EFFECT Mutates APScheduler state. + # @RATIONALE Provides a clean removal path that mirrors add_scenario_job so APScheduler + # stays consistent with the database when schedules are deactivated or deleted. + # @REJECTED Relying on APScheduler auto-cleanup was rejected — it would leave transient job + # registrations that could fire unexpected executions after schedule deactivation. + def remove_scenario_job(self, schedule_id: str) -> None: + with belief_scope( + "SchedulerService.remove_scenario_job", + f"schedule_id={schedule_id}", + ): + job_id_aps = f"scenario_{schedule_id}" + try: + self.scheduler.remove_job(job_id_aps) + logger.reason("Scenario schedule removed", payload={"job": job_id_aps}) + except Exception: + logger.reason(f"Scenario schedule not found (already removed): {job_id_aps}") + # #endregion Core.Scheduler.RemoveScenarioJob # #endregion Core.Scheduler.SchedulerService # #region Core.Scheduler.ThrottledSchedulerConfigurator [C:5] [TYPE Class] [SEMANTICS scheduler, throttling, distribution] # @defgroup Core Module group. diff --git a/backend/src/dependencies.py b/backend/src/dependencies.py index 7a314e1f9..8c3f9faf6 100755 --- a/backend/src/dependencies.py +++ b/backend/src/dependencies.py @@ -105,6 +105,7 @@ async_job_runner: AsyncJobRunner | None = None resource_service: ResourceService | None = None storage_service: StorageService | None = None storage_service_root: Path | None = None +live_execution_composition_root = None # #region Dependencies.AppDependencies.GetConfigManager [C:1] [TYPE Function] @@ -269,6 +270,25 @@ def get_storage_service(root: str | Path | None = None) -> StorageService: # #endregion Dependencies.AppDependencies.GetStorageService +# #region Dependencies.AppDependencies.LiveExecutionComposition [C:3] [TYPE Function] [SEMANTICS scenario,execution,composition,live,di] +# @BRIEF Return the application-owned 044 live-provider composition root. +# @POST The default root is empty and fail-closed until startup/server composition registers an exact provider. +# @INVARIANT This dependency never derives a Superset client, browser context, or capture provider from +# environment IDs, request metadata, or persisted binding fields. +# @RATIONALE A process-local DI singleton is the lawful home for runtime clients and secrets while +# ScenarioRun stores only the corresponding immutable identity snapshot. +# @REJECTED Request-scoped construction of live clients was rejected — it would turn untrusted IDs +# into execution authority and bypass the exact binding resolver. +def get_live_execution_composition_root(): + global live_execution_composition_root + if live_execution_composition_root is None: + from .services.dashboard_testing.execution.live_composition import LiveExecutionCompositionRoot + + live_execution_composition_root = LiveExecutionCompositionRoot() + return live_execution_composition_root +# #endregion Dependencies.AppDependencies.LiveExecutionComposition + + _clean_release_repository = CleanReleaseRepository() diff --git a/backend/src/models/scenario_artifact.py b/backend/src/models/scenario_artifact.py index 763e3bbc6..ef019181f 100644 --- a/backend/src/models/scenario_artifact.py +++ b/backend/src/models/scenario_artifact.py @@ -5,6 +5,9 @@ # @RELATION DEPENDS_ON -> [ScenarioExecution.Artifacts.Register] # @INVARIANT owner_type in {scenario_run, agent_run, verification_run, load_run}; owner_id is opaque. # @INVARIANT content_ref is opaque — never a raw filesystem path exposed to clients. +# @INVARIANT Artifact content is immutable audit evidence. is_active controls only whether a +# step attempt projects it into the current ScenarioRun result; retry invalidation +# retires the projection without deleting the artifact row. # @RATIONALE 036 DraftArtifact is FK-bound to agent_runs.id; scenario evidence must attach to the # ScenarioRun itself without fabricating an AgentRun (SCEX-FR-009 reuse constraint). # @REJECTED Reusing DraftArtifact for scenario evidence was rejected — its run_id FK would force @@ -14,7 +17,7 @@ from __future__ import annotations from datetime import UTC, datetime import uuid -from sqlalchemy import Column, DateTime, Index, String +from sqlalchemy import Boolean, Column, DateTime, Index, Integer, String from .mapping import Base @@ -33,6 +36,8 @@ def _now() -> datetime: # @ingroup Models # @BRIEF One registered artifact owned by a scenario run (or another generic owner type). # @INVARIANT sha256 is the content digest; kind is one of evidence|screenshot|report|xlsx|other. +# @INVARIANT A step-bound artifact records its logical_step_id and attempt. Retired artifacts +# remain queryable with is_active=false and invalidated_at set. class ScenarioArtifact(Base): __tablename__ = "scenario_artifacts" @@ -44,6 +49,10 @@ class ScenarioArtifact(Base): content_ref = Column(String(256), nullable=False) sha256 = Column(String(64), nullable=False) retention_class = Column(String(32), nullable=False, default="standard") + logical_step_id = Column(String(128), nullable=True, index=True) + attempt = Column(Integer, nullable=True) + is_active = Column(Boolean, nullable=False, default=True, index=True) + invalidated_at = Column(DateTime, nullable=True) created_at = Column(DateTime, nullable=False, default=_now) __table_args__ = ( diff --git a/backend/src/models/scenario_investigation.py b/backend/src/models/scenario_investigation.py index 5ab57ed84..73a72ff20 100644 --- a/backend/src/models/scenario_investigation.py +++ b/backend/src/models/scenario_investigation.py @@ -24,6 +24,7 @@ class InvestigationQueueItem(Base): severity = Column(String(16), nullable=False, default="warning") status = Column(String(24), nullable=False, default="queued", index=True) occurrence_count = Column(Integer, nullable=False, default=1) + evidence_snapshot = Column(JSON, nullable=False, default=dict) created_at = Column(DateTime, nullable=False, default=_now) # #endregion Models.ScenarioInvestigation.QueueItem @@ -36,6 +37,9 @@ class InvestigationCase(Base): status = Column(String(24), nullable=False, default="open", index=True) disposition = Column(String(32), nullable=True) decision_version = Column(Integer, nullable=False, default=1) + evidence_snapshot = Column(JSON, nullable=False, default=dict) + linked_run_ids = Column(JSON, nullable=False, default=list) + owner_id = Column(String(128), nullable=True, index=True) created_at = Column(DateTime, nullable=False, default=_now) # #endregion Models.ScenarioInvestigation.Case diff --git a/backend/src/models/scenario_run.py b/backend/src/models/scenario_run.py index b9c2a349c..f93568b83 100644 --- a/backend/src/models/scenario_run.py +++ b/backend/src/models/scenario_run.py @@ -4,6 +4,12 @@ # @RELATION DEPENDS_ON -> [Models.ScenarioRegistry.Revision] # @INVARIANT scenario_content_hash mirrors the pinned revision's content_hash (provenance), # while request_hash carries the canonical idempotency request fingerprint (data-model.md). +# @INVARIANT Live execution binding data is an immutable identity snapshot only; credentials, raw +# browser contexts, and callable principals remain runtime-only composition dependencies. +# @INVARIANT A cancellation deadline records only lifecycle timing; it cannot authorize, revive, +# or alter the immutable execution target/binding snapshot. +# @REJECTED Persisting executable live clients with ScenarioRun was rejected — database state cannot +# safely serialise or authorize a client/session capability. # @RATIONALE request_hash is a separate column so scenario_content_hash keeps provenance semantics: # a run pinned to revision R must report R's content hash even when the launch payload # differs across idempotent replays. @@ -26,6 +32,9 @@ def _now() -> datetime: # #region Models.ScenarioExecution.Run [C:4] [TYPE Class] [SEMANTICS scenario,execution,run,state] +# @BRIEF Persist one revision-pinned scenario execution, immutable live binding identity, and bounded cancel timing. +# @DATA_CONTRACT ScenarioRun.live_execution_binding_ref + live_execution_binding_snapshot -> LiveExecutionBinding.snapshot(). +# @INVARIANT Null binding fields preserve historical runs and cause live I/O to fail closed until a composition root resolves authorized dependencies. class ScenarioRun(Base): __tablename__ = "scenario_runs" @@ -42,8 +51,12 @@ class ScenarioRun(Base): idempotency_key = Column(String(128), nullable=False, unique=True) request_hash = Column(String(64), nullable=True, index=True) resume_token = Column(String(128), nullable=True) + cancel_requested_at = Column(DateTime, nullable=True) + cancel_drain_deadline_at = Column(DateTime, nullable=True, index=True) runner_plan = Column(JSON, nullable=False, default=dict) execution_principal_fingerprint = Column(String(64), nullable=True) + live_execution_binding_ref = Column(String(128), nullable=True, index=True) + live_execution_binding_snapshot = Column(JSON, nullable=True) error_code = Column(String(64), nullable=True) created_at = Column(DateTime, nullable=False, default=_now) started_at = Column(DateTime, nullable=True) diff --git a/backend/src/services/dashboard_testing/analytics/flakiness.py b/backend/src/services/dashboard_testing/analytics/flakiness.py index 616ca823d..547304bbe 100644 --- a/backend/src/services/dashboard_testing/analytics/flakiness.py +++ b/backend/src/services/dashboard_testing/analytics/flakiness.py @@ -15,6 +15,24 @@ def detect_flakiness(results: list[dict[str, Any]], *, context_key: str) -> dict return {"context_key": context_key, "usable_runs": len(usable), "transitions": transitions, "classification": "regression" if regression else "flaky" if transitions >= 2 else "stable"} # #endregion ScenarioAnalytics.Flakiness.Detect + +# #region ScenarioAnalytics.Client [C:3] [TYPE Class] [SEMANTICS scenario,analytics,client,health,flakiness] +# @defgroup ScenarioAnalytics Bound analytics client used by get_health_badge(analytics_client=...). +class ScenarioAnalyticsClient: + # @BRIEF Load run rows for a scenario and return a health projection on demand. + def __init__(self, db: Any) -> None: + self.db = db + + def get_scenario_health(self, scenario_id: str) -> dict[str, Any] | None: + from src.models.scenario_run import ScenarioRun + rows = self.db.query(ScenarioRun).filter(ScenarioRun.scenario_id == scenario_id).all() + if not rows: + return None + context_key = f"{scenario_id}:{rows[0].environment_id}" + results = [{"status": r.status, "context_key": context_key} for r in rows] + return derive_health(results, context_key=context_key) +# #endregion ScenarioAnalytics.Client + # #region ScenarioAnalytics.Health.Derive [C:3] [TYPE Function] [SEMANTICS scenario,analytics,health,derive] def derive_health(results: list[dict[str, Any]], *, context_key: str) -> dict[str, Any]: signal = detect_flakiness(results, context_key=context_key) diff --git a/backend/src/services/dashboard_testing/analytics/investigation.py b/backend/src/services/dashboard_testing/analytics/investigation.py index f8627b410..bf19c7b3b 100644 --- a/backend/src/services/dashboard_testing/analytics/investigation.py +++ b/backend/src/services/dashboard_testing/analytics/investigation.py @@ -2,51 +2,284 @@ # @defgroup ScenarioAnalytics Queue projection and explicit case lifecycle. # @BRIEF Queue immutable run signals without starting agents automatically. # @INVARIANT Case disposition uses decision-version CAS and never changes run/graph rows. +# @INVARIANT Terminal ScenarioRun signal identity is stable over run/status/pinned provenance and +# durable artifact digest/ref; retrying the same terminal context cannot rewrite it. +# @REJECTED Opening an InvestigationCase, AgentRun, chat, or remediation action while ingesting a +# terminal signal was rejected — queue projection is analyst-owned and non-executing. from __future__ import annotations +from hashlib import sha256 +import json +from typing import Any import uuid from sqlalchemy.orm import Session +from src.models.scenario_artifact import ScenarioArtifact from src.models.scenario_investigation import AgentAction, InvestigationCase, InvestigationQueueItem +from src.models.scenario_run import ScenarioRun, ScenarioStepRun # #region ScenarioAnalytics.Investigation.Queue [C:3] [TYPE Function] [SEMANTICS scenario,analytics,queue,project] -def queue_signal(db: Session, *, fingerprint: str, scenario_id: str, run_id: str | None, severity: str = "warning") -> InvestigationQueueItem: +def queue_signal( + db: Session, + *, + fingerprint: str, + scenario_id: str, + run_id: str | None, + severity: str = "warning", + evidence: dict[str, Any] | None = None, +) -> InvestigationQueueItem: + """Idempotently project immutable evidence into an analyst-owned queue item.""" item = db.query(InvestigationQueueItem).filter(InvestigationQueueItem.fingerprint == fingerprint, InvestigationQueueItem.status == "queued").first() if item is not None: item.occurrence_count += 1 + if run_id and run_id not in (item.evidence_snapshot or {}).get("run_ids", []): + snapshot = dict(item.evidence_snapshot or {}) + snapshot["run_ids"] = [*snapshot.get("run_ids", []), run_id] + item.evidence_snapshot = snapshot return item - item = InvestigationQueueItem(fingerprint=fingerprint, scenario_id=scenario_id, run_id=run_id, severity=severity) + snapshot = {**(evidence or {})} + if run_id: + snapshot["run_ids"] = [run_id] + item = InvestigationQueueItem( + fingerprint=fingerprint, + scenario_id=scenario_id, + run_id=run_id, + severity=severity, + evidence_snapshot=snapshot, + ) db.add(item) db.flush() return item # #endregion ScenarioAnalytics.Investigation.Queue + +# #region ScenarioAnalytics.Investigation.TerminalEvidence [C:2] [TYPE Function] [SEMANTICS scenario,analytics,terminal,evidence,provenance] +# @BRIEF Read only registered ScenarioRun evidence, ordered deterministically by opaque ref and digest. +# @INVARIANT Retired retry-attempt artifacts remain audit-visible but are excluded from the active +# terminal signal; only is_active=true evidence can describe the current result. +def _terminal_evidence(db: Session, run_id: str) -> list[dict[str, str]]: + rows = ( + db.query(ScenarioArtifact) + .filter( + ScenarioArtifact.owner_type == "scenario_run", + ScenarioArtifact.owner_id == run_id, + ScenarioArtifact.is_active.is_(True), + ) + .order_by(ScenarioArtifact.content_ref.asc(), ScenarioArtifact.sha256.asc(), ScenarioArtifact.id.asc()) + .all() + ) + return [ + {"kind": row.kind, "content_ref": row.content_ref, "sha256": row.sha256} + for row in rows + ] +# #endregion ScenarioAnalytics.Investigation.TerminalEvidence + + +# #region ScenarioAnalytics.Investigation.TerminalAttempts [C:3] [TYPE Function] [SEMANTICS scenario,analytics,terminal,retry,attempt,provenance] +# @BRIEF Freeze the active step attempt vector that identifies a terminal execution context. +# @INVARIANT A reterminalized retry has a distinct vector from its earlier terminal attempt, while +# duplicate emission of the same vector remains idempotent. +def _terminal_attempts(db: Session, run_id: str) -> list[dict[str, Any]]: + rows = ( + db.query(ScenarioStepRun) + .filter(ScenarioStepRun.run_id == run_id) + .order_by(ScenarioStepRun.step_position.asc(), ScenarioStepRun.logical_step_id.asc(), ScenarioStepRun.id.asc()) + .all() + ) + return [ + { + "logical_step_id": row.logical_step_id, + "attempt": row.attempt, + "status": row.status, + "artifact_refs": list(row.artifact_refs or []), + } + for row in rows + ] +# #endregion ScenarioAnalytics.Investigation.TerminalAttempts + + +# #region ScenarioAnalytics.Investigation.TerminalSnapshot [C:3] [TYPE Function] [SEMANTICS scenario,analytics,terminal,snapshot,provenance] +# @BRIEF Freeze the durable terminal provenance that a 047 analyst queue may inspect. +def _terminal_snapshot( + run: ScenarioRun, + evidence: list[dict[str, str]], + attempts: list[dict[str, Any]], +) -> dict[str, Any]: + snapshot = { + "signal_type": "scenario_terminal", + "scenario_id": run.scenario_id, + "scenario_run_id": run.id, + "status": run.status, + "environment_id": run.environment_id, + "scenario_revision_id": run.scenario_revision_id, + "scenario_content_hash": run.scenario_content_hash, + "target_snapshot": run.target_snapshot or {}, + "execution_principal_fingerprint": run.execution_principal_fingerprint, + "evidence": evidence, + "step_attempts": attempts, + } + return json.loads(json.dumps(snapshot, sort_keys=True, separators=(",", ":"))) +# #endregion ScenarioAnalytics.Investigation.TerminalSnapshot + + +# #region ScenarioAnalytics.Investigation.TerminalSignal [C:4] [TYPE Function] [SEMANTICS scenario,analytics,terminal,signal,queue,idempotency] +# @BRIEF Emit one canonical 036-compatible queue input for a non-pass terminal ScenarioRun. +# @PRE run.status is persisted; evidence rows, if any, are ScenarioRun-owned durable artifacts. +# @POST Returns the same InvestigationQueueItem for the same immutable terminal context, without +# opening a case or starting agent/remediation work; passed/nonterminal runs return None. +# @SIDE_EFFECT At most one 047 InvestigationQueueItem insert for the terminal signal fingerprint. +# @INVARIANT Failed, blocked and inconclusive runs carry their pinned revision, target/principal, +# status, active verified artifact ref/digest provenance and attempt vector into the +# queue snapshot. Same context reuses one signal; a retried terminal attempt is new +# immutable context and never mutates the earlier signal. +# @REJECTED Reusing a scenario/environment-only fingerprint was rejected — it conflates distinct +# terminal runs and makes a retry mutate occurrence count or evidence history. +def emit_terminal_run_signal( + db: Session, + run: ScenarioRun, +) -> InvestigationQueueItem | None: + if run.status not in {"failed", "blocked", "inconclusive"}: + return None + evidence = _terminal_evidence(db, run.id) + snapshot = _terminal_snapshot(run, evidence, _terminal_attempts(db, run.id)) + fingerprint = "scenario-terminal:" + sha256( + json.dumps(snapshot, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + existing = ( + db.query(InvestigationQueueItem) + .filter(InvestigationQueueItem.fingerprint == fingerprint) + .order_by(InvestigationQueueItem.created_at.asc(), InvestigationQueueItem.id.asc()) + .first() + ) + if existing is not None: + return existing + return ingest_investigation_signal( + db, + { + "fingerprint": fingerprint, + "scenario_id": run.scenario_id, + "run_id": run.id, + "severity": "warning", + "evidence": snapshot, + }, + ) +# #endregion ScenarioAnalytics.Investigation.TerminalSignal + +# #region ScenarioAnalytics.Investigation.IngestSignal [C:4] [TYPE Function] [SEMANTICS scenario,analytics,terminal,signal,queue,producer] +# @BRIEF Validate and project one canonical signal into the analyst-owned queue without opening work. +# @PRE Signal has a deterministic fingerprint and scenario_id; evidence is immutable caller-supplied context. +# @POST Returns a queue item only; no InvestigationCase, AgentRun, chat, remediation action, or recurrence classification is started. +# @SIDE_EFFECT Creates or updates the canonical queue projection through queue_signal. +# @INVARIANT This is a producer boundary, not 047 recurrence classification or case lifecycle. +def ingest_investigation_signal(db: Session, signal: dict[str, Any]) -> InvestigationQueueItem: + """Canonical 036-compatible signal boundary; never starts agent work or mutates run truth.""" + fingerprint = str(signal.get("fingerprint") or signal.get("idempotency_key") or "") + scenario_id = str(signal.get("scenario_id") or "") + if not fingerprint or not scenario_id: + raise ValueError("signal requires fingerprint and scenario_id") + return queue_signal( + db, + fingerprint=fingerprint, + scenario_id=scenario_id, + run_id=signal.get("run_id"), + severity=str(signal.get("severity") or "warning"), + evidence=dict(signal.get("evidence") or {}), + ) +# #endregion ScenarioAnalytics.Investigation.IngestSignal + # #region ScenarioAnalytics.Investigation.Case [C:4] [TYPE Function] [SEMANTICS scenario,analytics,case,open,disposition] + +def auto_queue_failed_run( + db: Session, + *, + scenario_id: str, + run_id: str, + environment_id: str, +) -> InvestigationQueueItem: + """Compatibility boundary for 044 terminal failures; it only queues immutable context.""" + return ingest_investigation_signal( + db, + { + "fingerprint": f"{scenario_id}:{environment_id}", + "scenario_id": scenario_id, + "run_id": run_id, + "severity": "warning", + "evidence": {"environment_id": environment_id, "source": "scenario_run"}, + }, + ) + + def open_case(db: Session, *, fingerprint: str, scenario_id: str, actor_id: str) -> InvestigationCase: case = db.query(InvestigationCase).filter(InvestigationCase.fingerprint == fingerprint).first() if case is not None: return case - case = InvestigationCase(id=str(uuid.uuid4()), fingerprint=fingerprint, scenario_id=scenario_id) + queue = db.query(InvestigationQueueItem).filter( + InvestigationQueueItem.fingerprint == fingerprint, + InvestigationQueueItem.status == "queued", + ).first() + snapshot = dict(queue.evidence_snapshot or {}) if queue else {} + run_ids = list(snapshot.get("run_ids", [])) + case = InvestigationCase( + id=str(uuid.uuid4()), + fingerprint=fingerprint, + scenario_id=scenario_id, + evidence_snapshot=snapshot, + linked_run_ids=run_ids, + owner_id=actor_id, + ) db.add(case) db.flush() db.add(AgentAction(case_id=case.id, action_type="case_opened", payload={"actor_id": actor_id})) db.flush() return case -def set_disposition(db: Session, case_id: str, *, disposition: str, expected_version: int, actor_id: str) -> InvestigationCase: +def set_disposition( + db: Session, + case_id: str, + *, + disposition: str, + expected_version: int, + actor_id: str, + verification_evidence: dict[str, Any] | None = None, + rationale: str = "", +) -> InvestigationCase: case = db.query(InvestigationCase).filter(InvestigationCase.id == case_id).first() if case is None: raise ValueError("case not found") if case.decision_version != expected_version: raise ValueError("stale disposition version") + if disposition == "resolved": + if not verification_evidence or not verification_evidence.get("reconciled"): + raise ValueError("resolved disposition requires reconciled verification evidence") + case.status = "resolved" + elif disposition == "accepted": + if not rationale.strip(): + raise ValueError("accepted disposition requires accepted-risk rationale") + case.status = "accepted" + else: + raise ValueError("disposition must be resolved or accepted") case.disposition = disposition - case.status = "resolved" case.decision_version += 1 - db.add(AgentAction(case_id=case.id, action_type="disposition", payload={"actor_id": actor_id, "disposition": disposition})) + db.add(AgentAction(case_id=case.id, action_type="disposition", payload={"actor_id": actor_id, "disposition": disposition, "verification_evidence": verification_evidence or {}, "rationale": rationale})) + from src.models.scenario_investigation import RecurringFailureEpisode + from src.services.dashboard_testing.analytics.recurring import resolve_episode + episode = db.query(RecurringFailureEpisode).filter( + RecurringFailureEpisode.fingerprint == case.fingerprint, + RecurringFailureEpisode.status == "open", + ).first() + if episode is not None and disposition in {"resolved", "accepted"}: + resolve_episode(db, episode.id) db.flush() return case + + +def can_access_case(case: InvestigationCase, *, actor_id: str, triage: bool) -> bool: + """Object ACL is orthogonal to scenario-result:view/triage scopes.""" + if triage: + return True + return case.owner_id in {None, actor_id} # #endregion ScenarioAnalytics.Investigation.Case # #endregion ScenarioAnalytics.Investigation diff --git a/backend/src/services/dashboard_testing/analytics/recurring.py b/backend/src/services/dashboard_testing/analytics/recurring.py index ec7fb58e8..fe457d03f 100644 --- a/backend/src/services/dashboard_testing/analytics/recurring.py +++ b/backend/src/services/dashboard_testing/analytics/recurring.py @@ -11,15 +11,36 @@ from src.models.scenario_investigation import RecurringFailureEpisode # #region ScenarioAnalytics.Recurring.Record [C:4] [TYPE Function] [SEMANTICS scenario,analytics,recurring,episode,record] def record_occurrence(db: Session, *, fingerprint: str, scenario_id: str) -> RecurringFailureEpisode: - episode = db.query(RecurringFailureEpisode).filter(RecurringFailureEpisode.fingerprint == fingerprint, RecurringFailureEpisode.status == "open").first() now = datetime.now(UTC) - if episode is not None: - episode.occurrence_count += 1 - episode.last_seen_at = now - return episode + open_episode = db.query(RecurringFailureEpisode).filter( + RecurringFailureEpisode.fingerprint == fingerprint, + RecurringFailureEpisode.status == "open", + ).first() + if open_episode is not None: + open_episode.occurrence_count += 1 + open_episode.last_seen_at = now + return open_episode + resolved = ( + db.query(RecurringFailureEpisode) + .filter(RecurringFailureEpisode.fingerprint == fingerprint, RecurringFailureEpisode.status == "resolved") + .order_by(RecurringFailureEpisode.last_seen_at.desc()) + .first() + ) episode = RecurringFailureEpisode(fingerprint=fingerprint, scenario_id=scenario_id, first_seen_at=now, last_seen_at=now) db.add(episode) db.flush() + if resolved is not None: + from src.services.dashboard_testing.analytics.investigation import ingest_investigation_signal + + ingest_investigation_signal( + db, + { + "fingerprint": f"recurrence:{fingerprint}:{episode.id}", + "scenario_id": scenario_id, + "severity": "warning", + "evidence": {"previous_episode_id": resolved.id, "fingerprint": fingerprint, "source": "recurring_failure"}, + }, + ) return episode # #endregion ScenarioAnalytics.Recurring.Record diff --git a/backend/src/services/dashboard_testing/automation/notify.py b/backend/src/services/dashboard_testing/automation/notify.py index b0b97ac40..e887012d6 100644 --- a/backend/src/services/dashboard_testing/automation/notify.py +++ b/backend/src/services/dashboard_testing/automation/notify.py @@ -4,6 +4,10 @@ from __future__ import annotations from typing import Any +from sqlalchemy.orm import Session + +from src.models.scenario_automation import ScenarioNotificationEvent + def notify(events: list[dict[str, Any]], *, event_type: str, scenario_id: str, run_id: str | None, severity: str = "info") -> dict[str, Any]: if event_type not in {"completed", "failed", "blocked", "human", "stale", "flaky"}: @@ -11,4 +15,28 @@ def notify(events: list[dict[str, Any]], *, event_type: str, scenario_id: str, r event = {"event_type": event_type, "scenario_id": scenario_id, "run_id": run_id, "severity": severity} events.append(event) return event + + +def persist_notification( + db: Session, + *, + event_type: str, + scenario_id: str, + run_id: str | None, + severity: str = "info", + payload: dict[str, Any] | None = None, +) -> ScenarioNotificationEvent: + """Persist one typed automation outcome; callers own delivery channels.""" + if event_type not in {"completed", "failed", "blocked", "stale", "flaky"}: + raise ValueError("unsupported notification event") + row = ScenarioNotificationEvent( + event_type=event_type, + scenario_id=scenario_id, + run_id=run_id, + severity=severity, + payload=payload or {}, + ) + db.add(row) + db.flush() + return row # #endregion ScenarioAutomation.Notify diff --git a/backend/src/services/dashboard_testing/automation/trigger.py b/backend/src/services/dashboard_testing/automation/trigger.py index 7f9836440..7d92cd80b 100644 --- a/backend/src/services/dashboard_testing/automation/trigger.py +++ b/backend/src/services/dashboard_testing/automation/trigger.py @@ -1,20 +1,122 @@ # #region ScenarioAutomation.Trigger [C:4] [TYPE Module] [SEMANTICS scenario,automation,trigger,event,dedup] # @defgroup ScenarioAutomation Map typed events to pinned run candidates. +# @RELATION DEPENDS_ON -> [ScenarioExecution.Runner.QueuedDispatch] +# @RELATION DEPENDS_ON -> [ScenarioExecution.EnvironmentPolicy.Resolve] +# @INVARIANT A dispatch passes its server-owned event type to 044 before run creation; it never +# creates a manual run and mutates provenance afterward. Human revisions therefore +# reject atomically at the 044 manual-run-only boundary. +# @INVARIANT Event/rule payloads carry only target identity; EnvironmentExecutionPolicy resolves +# PROD authority from ConfigManager before each ScenarioRun can be persisted. +# @REJECTED Post-create trigger_source mutation was rejected because a human graph could be +# created as manual before the automation provenance was recorded. +# @REJECTED Trusting rule.environment_class or an event boolean for PROD was rejected because a +# stale or forged payload could bypass the durable approval gate. from __future__ import annotations from typing import Any +from sqlalchemy.orm import Session + from .policy import apply_policy +# #region ScenarioAutomation.Trigger.Handle [C:3] [TYPE Function] [SEMANTICS scenario,automation,trigger,event,policy] +# @BRIEF Map one typed event to policy-allowed pinned run candidates without creating runs. +# @POST Returned candidates retain the event's server-owned trigger_source and target identity; +# PROD classification is intentionally deferred to the 044 policy resolver. def handle_trigger_event(event: dict[str, Any], rules: list[dict[str, Any]], active_runs: list[dict[str, Any]], policy: dict[str, Any]) -> list[dict[str, Any]]: candidates = [] for rule in rules: if not rule.get("enabled", True) or rule.get("event_type") != event.get("type"): continue - candidate = {"scenario_id": rule["scenario_id"], "revision_id": rule["revision_id"], "environment_id": rule["environment_id"], "trigger_source": event.get("type"), "dedup_fingerprint": event.get("fingerprint"), "is_prod": rule.get("environment_class") == "PROD"} + candidate = {"scenario_id": rule["scenario_id"], "revision_id": rule["revision_id"], "environment_id": rule["environment_id"], "trigger_source": event.get("type"), "dedup_fingerprint": event.get("fingerprint")} decision = apply_policy(policy, candidate, active_runs) if decision["allowed"]: candidates.append(candidate) return candidates +# #endregion ScenarioAutomation.Trigger.Handle + + +# #region ScenarioAutomation.Trigger.Dispatch [C:4] [TYPE Function] [SEMANTICS scenario,automation,trigger,event,dispatch,manual-only] +# @BRIEF Dispatch one persisted automation event through policy and into the trusted 044 start boundary. +# @RELATION CALLS -> [ScenarioExecution.Runner.Start] +# @RELATION DEPENDS_ON -> [ScenarioExecution.EnvironmentPolicy.Resolve] +# @RELATION DEPENDS_ON -> [ScenarioExecution.Runner.QueuedDispatch] +# @POST A candidate's real event type becomes trigger_source before run creation; accepted candidates +# are persisted queued/pending only, while manual-only rejection has no run-side effect. +# @SIDE_EFFECT May persist policy-allowed ScenarioRuns through start_run; it never invokes an adapter. +# @INVARIANT This dispatcher never creates a manual run and later mutates its provenance. A malformed +# automated human plan is rejected at the 044 pre-create boundary; legacy queued rows are +# separately blocked by the 044 dispatcher before CAS/walker execution. +# @INVARIANT ConfigManager resolution occurs before each start; an unknown target raises before that +# rule creates a run, gate, queue, notification, or adapter side effect. +def dispatch_trigger_event( + db: Session, + event: dict[str, Any], + *, + actor: str = "automation", + start_run_fn=None, + config_manager=None, +) -> list[str]: + """Project a typed deploy/release/ETL event into policy-checked ScenarioRuns.""" + from src.models.scenario_automation import AutomationPolicy, ScenarioTriggerRule + from src.models.scenario_registry import ScenarioRegistryEntry + from src.models.scenario_run import ScenarioRun + from src.services.dashboard_testing.execution.runner import start_run + + start = start_run_fn or start_run + if config_manager is None: + from src.dependencies import get_config_manager + + config_manager = get_config_manager() + rules = db.query(ScenarioTriggerRule).filter( + ScenarioTriggerRule.enabled.is_(True), + ScenarioTriggerRule.trigger == event.get("type"), + ).all() + active = [ + { + "environment_id": run.environment_id, + "dedup_fingerprint": (run.parameter_bindings or {}).get("automation_fingerprint"), + "status": run.status, + "created_at": run.created_at.isoformat() if run.created_at else None, + } + for run in db.query(ScenarioRun).filter(ScenarioRun.status.in_(["queued", "running", "pending_approval"])).all() + ] + created: list[str] = [] + for rule in rules: + policy_row = db.query(AutomationPolicy).filter(AutomationPolicy.id == rule.policy_id).first() if rule.policy_id else None + policy = { + "max_concurrent_per_env": policy_row.max_concurrent_per_env if policy_row else 1, + "dedup_window_seconds": policy_row.dedup_window_seconds if policy_row else 0, + "overlap_rule": policy_row.overlap_rule if policy_row else "block", + } + revision_id = rule.revision_id + if rule.revision_policy == "current": + entry = db.query(ScenarioRegistryEntry).filter(ScenarioRegistryEntry.scenario_id == rule.scenario_id).first() + revision_id = entry.current_revision_id if entry else None + candidate = { + "scenario_id": rule.scenario_id, + "revision_id": revision_id, + "environment_id": rule.environment_id, + "dedup_fingerprint": event.get("fingerprint"), + "overlap": bool(event.get("overlap")), + } + decision = apply_policy(policy, candidate, active) + if not decision["allowed"] or not candidate["revision_id"]: + continue + run = start( + db, + scenario_id=rule.scenario_id, + revision_id=candidate["revision_id"], + params={"automation_fingerprint": event.get("fingerprint")}, + environment_id=rule.environment_id, + actor=actor, + idempotency_key=f"trigger-{rule.id}-{event.get('fingerprint')}", + config_manager=config_manager, + auto_advance=False, + trigger_source=str(event.get("type")), + ) + created.append(run.id) + return created +# #endregion ScenarioAutomation.Trigger.Dispatch # #endregion ScenarioAutomation.Trigger diff --git a/backend/src/services/dashboard_testing/execution/approval.py b/backend/src/services/dashboard_testing/execution/approval.py index d0616e70b..f077bd154 100644 --- a/backend/src/services/dashboard_testing/execution/approval.py +++ b/backend/src/services/dashboard_testing/execution/approval.py @@ -12,6 +12,7 @@ from __future__ import annotations from datetime import UTC, datetime +from sqlalchemy import update from sqlalchemy.orm import Session from src.models.scenario_approval import ActionApprovalGate @@ -56,16 +57,24 @@ def create_prod_gate( def decide_approval_gate(db: Session, gate_id: str, *, decision: str, actor_id: str, comment: str = "") -> ActionApprovalGate: if decision not in {"approve", "deny"}: raise ValueError("invalid gate decision") + decided_at = datetime.now(UTC) + consumed = db.execute( + update(ActionApprovalGate) + .where(ActionApprovalGate.id == gate_id, ActionApprovalGate.status == "pending") + .values( + status="approved" if decision == "approve" else "denied", + decision=decision, + actor_id=actor_id, + comment=comment, + decided_at=decided_at, + ) + ) + if consumed.rowcount != 1: + exists = db.query(ActionApprovalGate.id).filter(ActionApprovalGate.id == gate_id).first() + raise ValueError("approval gate already decided" if exists is not None else "approval gate not found") gate = db.query(ActionApprovalGate).filter(ActionApprovalGate.id == gate_id).first() if gate is None: raise ValueError("approval gate not found") - if gate.status != "pending": - raise ValueError("approval gate already decided") - gate.status = "approved" if decision == "approve" else "denied" - gate.decision = decision - gate.actor_id = actor_id - gate.comment = comment - gate.decided_at = datetime.now(UTC) run = db.query(ScenarioRun).filter(ScenarioRun.id == gate.owner_id).first() if run is not None and run.status == "pending_approval": run.status = "queued" if decision == "approve" else "blocked" diff --git a/backend/src/services/dashboard_testing/execution/artifacts.py b/backend/src/services/dashboard_testing/execution/artifacts.py index da67ced84..98c66d852 100644 --- a/backend/src/services/dashboard_testing/execution/artifacts.py +++ b/backend/src/services/dashboard_testing/execution/artifacts.py @@ -7,9 +7,11 @@ # @INVARIANT owner_type must be one of {scenario_run, agent_run, verification_run, load_run}; # owner_id must reference an existing row of that type where enforced. # @INVARIANT sha256 is a 64-hex content digest; content_ref is opaque (never a raw FS path). +# @INVARIANT Registered scenario evidence never uses a manufactured all-zero SHA-256 placeholder. # @REJECTED Reusing DraftArtifact (agent-run FK) for scenario evidence was rejected in the model. from __future__ import annotations +from datetime import UTC, datetime from typing import Any from sqlalchemy.orm import Session @@ -20,6 +22,46 @@ from src.models.scenario_run import ScenarioRun, ScenarioStepRun _ARTIFACT_KINDS = frozenset({"evidence", "screenshot", "report", "xlsx", "other"}) +# #region ScenarioExecution.Artifacts.ValidDigest [C:2] [TYPE Function] [SEMANTICS scenario,execution,artifact,digest,integrity] +# @BRIEF Accept only a concrete 64-hex SHA-256 digest, never the all-zero placeholder. +# @INVARIANT A registered evidence digest denotes supplied or locally computed content, not a sentinel. +def is_valid_sha256(value: object) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and value.lower() != "0" * 64 + and all(char in "0123456789abcdef" for char in value.lower()) + ) +# #endregion ScenarioExecution.Artifacts.ValidDigest + + +# #region ScenarioExecution.Artifacts.VerifiedEvidence [C:3] [TYPE Function] [SEMANTICS scenario,execution,artifact,evidence,digest,integrity] +# @BRIEF Verify screenshot evidence refs agree with their canonical forwarded digest map. +# @POST Every evidence ref has one valid non-zero SHA-256 matching the forwarded digest. +# @INVARIANT Screenshot evidence is durable only when every ref and digest agree exactly. +# @REJECTED Treating a shape-valid but mismatched digest as capture evidence was rejected. +def has_verified_evidence(artifact_refs: list[str], step_outcome: dict[str, Any]) -> bool: + if not artifact_refs: + return False + digest_map = step_outcome.get("artifact_digests") + if not isinstance(digest_map, dict): + return False + first_digest = step_outcome.get("sha256") + return is_valid_sha256(first_digest) and digest_map.get(artifact_refs[0]) == first_digest and all( + is_valid_sha256(digest_map.get(ref)) + for ref in artifact_refs + ) and all(ref in digest_map for ref in artifact_refs) +# #endregion ScenarioExecution.Artifacts.VerifiedEvidence + + +# #region ScenarioExecution.Artifacts.VerifiedSingleEvidence [C:2] [TYPE Function] [SEMANTICS scenario,execution,artifact,evidence,digest,compatibility] +# @ingroup ScenarioExecution +# @BRIEF Preserve the legacy one-ref evidence predicate for callers with a single artifact contract. +def has_verified_single_evidence(artifact_refs: list[str], step_outcome: dict[str, Any]) -> bool: + return len(artifact_refs) == 1 and has_verified_evidence(artifact_refs, step_outcome) +# #endregion ScenarioExecution.Artifacts.VerifiedSingleEvidence + + # #region ScenarioExecution.Artifacts.Register [C:4] [TYPE Function] [SEMANTICS scenario,execution,artifact,register] # @ingroup ScenarioExecution # @BRIEF Persist one artifact row owned by a scenario run (or other generic owner). @@ -36,13 +78,15 @@ def register_artifact( content_ref: str, sha256: str, retention_class: str = "standard", + logical_step_id: str | None = None, + attempt: int | None = None, ) -> ScenarioArtifact: if owner_type not in _OWNER_TYPES: raise ValueError(f"unsupported artifact owner_type: {owner_type}") if kind not in _ARTIFACT_KINDS: raise ValueError(f"unsupported artifact kind: {kind}") - if len(sha256) != 64 or any(c not in "0123456789abcdef" for c in sha256.lower()): - raise ValueError("sha256 must be a 64-char hex digest") + if not is_valid_sha256(sha256): + raise ValueError("sha256 must be a non-placeholder 64-char hex digest") if owner_type == "scenario_run": run = db.query(ScenarioRun).filter(ScenarioRun.id == owner_id).first() if run is None: @@ -55,6 +99,8 @@ def register_artifact( content_ref=content_ref, sha256=sha256.lower(), retention_class=retention_class, + logical_step_id=logical_step_id, + attempt=attempt, ) db.add(artifact) db.flush() @@ -62,6 +108,95 @@ def register_artifact( # #endregion ScenarioExecution.Artifacts.Register +# #region ScenarioExecution.Artifacts.RegisterStepEvidence [C:4] [TYPE Function] [SEMANTICS scenario,execution,artifact,evidence,digest,integrity] +# @ingroup ScenarioExecution +# @BRIEF Register verified executor evidence refs; retain a typed integrity condition for bad refs. +# @INVARIANT A ref with absent, malformed, or all-zero digest never creates a ScenarioArtifact row. +# @POST Valid evidence refs persist with their exact supplied/local digest; invalid refs return an +# inconclusive integrity payload and remain unregistered. +# @REJECTED Filling absent digest with zeros was rejected — it makes unverifiable evidence appear durable. +def register_step_evidence( + db: Session, + *, + run_id: str, + logical_step_id: str, + artifact_refs: list[str], + outcome: dict[str, Any], + attempt: int, +) -> dict[str, Any] | None: + if not artifact_refs: + return None + step_outcome = outcome.get("step_outcome") if isinstance(outcome.get("step_outcome"), dict) else outcome + raw_digests = step_outcome.get("artifact_digests") + digest_map = raw_digests if isinstance(raw_digests, dict) else {} + unregistered: list[str] = [] + invalid_digest = False + for ref in artifact_refs: + digest = digest_map.get(ref) + if digest is None and len(artifact_refs) == 1: + digest = step_outcome.get("sha256") + if not is_valid_sha256(digest): + unregistered.append(ref) + invalid_digest = invalid_digest or digest is not None + continue + register_artifact( + db, + owner_type="scenario_run", + owner_id=run_id, + kind="evidence", + name=f"step-{logical_step_id}", + content_ref=ref, + sha256=digest, + logical_step_id=logical_step_id, + attempt=attempt, + ) + attach_artifact_refs(db, run_id, logical_step_id, [ref]) + if not unregistered: + return None + return { + "status": "inconclusive", + "reason_code": "ARTIFACT_DIGEST_INVALID" if invalid_digest else "ARTIFACT_DIGEST_MISSING", + "unregistered_refs": unregistered, + } +# #endregion ScenarioExecution.Artifacts.RegisterStepEvidence + + +# #region ScenarioExecution.Artifacts.InvalidateStepEvidence [C:4] [TYPE Function] [SEMANTICS scenario,execution,artifact,retry,invalidation,provenance] +# @ingroup ScenarioExecution +# @BRIEF Retire active evidence projection for a retry closure while retaining every artifact row. +# @RELATION CALLED_BY -> [ScenarioExecution.Lifecycle.Retry] +# @POST Matching current-attempt artifacts remain durable and queryable with is_active=false; +# terminal evidence excludes them until a retried attempt registers fresh evidence. +# @INVARIANT No artifact content_ref, sha256, owner, or creation record is deleted or overwritten. +# @REJECTED Deleting retry-invalidated artifacts was rejected — they are immutable audit evidence, +# even though they must not leak into the active ScenarioRun result. +def invalidate_step_evidence( + db: Session, + *, + run_id: str, + logical_step_ids: set[str], +) -> int: + if not logical_step_ids: + return 0 + rows = ( + db.query(ScenarioArtifact) + .filter( + ScenarioArtifact.owner_type == "scenario_run", + ScenarioArtifact.owner_id == run_id, + ScenarioArtifact.logical_step_id.in_(sorted(logical_step_ids)), + ScenarioArtifact.is_active.is_(True), + ) + .all() + ) + invalidated_at = datetime.now(UTC) + for artifact in rows: + artifact.is_active = False + artifact.invalidated_at = invalidated_at + db.flush() + return len(rows) +# #endregion ScenarioExecution.Artifacts.InvalidateStepEvidence + + # #region ScenarioExecution.Artifacts.List [C:3] [TYPE Function] [SEMANTICS scenario,execution,artifact,list] # @ingroup ScenarioExecution # @BRIEF List artifacts owned by one owner, newest first, optional kind filter. @@ -112,6 +247,10 @@ def artifact_to_dict(artifact: ScenarioArtifact) -> dict[str, Any]: "content_ref": artifact.content_ref, "sha256": artifact.sha256, "retention_class": artifact.retention_class, + "logical_step_id": artifact.logical_step_id, + "attempt": artifact.attempt, + "is_active": artifact.is_active, + "invalidated_at": artifact.invalidated_at.isoformat() if artifact.invalidated_at else None, "created_at": artifact.created_at.isoformat() if artifact.created_at else None, } # #endregion ScenarioExecution.Artifacts.ToDict diff --git a/backend/src/services/dashboard_testing/execution/dispatch.py b/backend/src/services/dashboard_testing/execution/dispatch.py index ecc581e05..2d8bf5711 100644 --- a/backend/src/services/dashboard_testing/execution/dispatch.py +++ b/backend/src/services/dashboard_testing/execution/dispatch.py @@ -3,6 +3,8 @@ # @BRIEF Dispatch only ready steps and mark dependent descendants blocked after failure. # @RELATION DEPENDS_ON -> [ScenarioExecution.ExecutorRegistry] # @INVARIANT A failed producer blocks descendants; unknown tools fail before executor invocation. +# @INVARIANT An executor receives only an immutable validated ActionExecutionDescriptor; missing or +# mismatched descriptor fields are typed blocked before adapter I/O. # @INVARIANT Every dispatch decision is traced with a molecular CoT marker (REASON/REFLECT/EXPLORE). from __future__ import annotations @@ -29,8 +31,8 @@ def _descendants(step_id: str, edges: list[dict[str, str]]) -> set[str]: # #region ScenarioExecution.Dispatch.Step [C:4] [TYPE Function] [SEMANTICS scenario,execution,dispatch,step] # @ingroup ScenarioExecution -# @BRIEF Dispatch one step: dependency gate -> human control -> typed executor. -# @POST Returns a step outcome dict; unknown tools reject before any executor I/O. +# @BRIEF Dispatch one step: dependency gate -> human control -> descriptor-bound typed executor. +# @POST Returns a step outcome dict; invalid descriptors reject before any executor I/O. def dispatch_step(step: dict[str, Any], *, completed: dict[str, dict[str, Any]], registry: ScenarioExecutorRegistry, edges: list[dict[str, str]]) -> dict[str, Any]: step_id = str(step["logical_step_id"]) dependencies = [str(edge["source"]) for edge in edges if str(edge["target"]) == step_id] @@ -55,6 +57,17 @@ def dispatch_step(step: dict[str, Any], *, completed: dict[str, dict[str, Any]], error="dependency_failed", ) return {"status": "blocked", "reason": "dependency_failed", "logical_step_id": step_id, "blocked_descendants": blocked_descendants} + descriptor = step.get("action_descriptor") + if not isinstance(descriptor, dict): + return { + "status": "blocked", "reason": "ACTION_DESCRIPTOR_REQUIRED", + "logical_step_id": step_id, + } + if descriptor.get("tool") != step.get("tool") or descriptor.get("action") != step.get("action"): + return { + "status": "blocked", "reason": "ACTION_DESCRIPTOR_MISMATCH", + "logical_step_id": step_id, + } if step.get("tool") == "human": logger.reflect( "ScenarioExecution.Dispatch.dispatch_step", @@ -62,7 +75,7 @@ def dispatch_step(step: dict[str, Any], *, completed: dict[str, dict[str, Any]], payload={"logical_step_id": step_id}, ) return {"status": "waiting_human", "logical_step_id": step_id} - executor = registry.resolve(str(step.get("tool", "assertion"))) + executor = registry.resolve(descriptor) result = executor(step, completed) logger.reflect( "ScenarioExecution.Dispatch.dispatch_step", diff --git a/backend/src/services/dashboard_testing/execution/environment_policy.py b/backend/src/services/dashboard_testing/execution/environment_policy.py new file mode 100644 index 000000000..f53f74dfd --- /dev/null +++ b/backend/src/services/dashboard_testing/execution/environment_policy.py @@ -0,0 +1,52 @@ +# #region ScenarioExecution.EnvironmentPolicy [C:4] [TYPE Module] [SEMANTICS scenario,execution,environment,prod,authority] +# @BRIEF Resolve the execution environment class exclusively from server-owned configuration. +# @INVARIANT A request/run body cannot classify an environment; unknown configuration is rejected before +# ScenarioRun, gate, queue, notification, or adapter side effects exist. +# @RATIONALE PROD is an execution authority decision, not display metadata. ConfigManager owns the +# deployment Environment record including stage and is_production policy. +# @REJECTED Inferring PROD from environment_id text or accepting a caller is_prod flag was rejected — +# either allows a forged PREPROD classification to bypass the durable approval gate. +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class EnvironmentExecutionPolicy: + environment_id: str + environment_class: str + + @property + def is_prod(self) -> bool: + return self.environment_class == "PROD" + + +# #region ScenarioExecution.EnvironmentPolicy.Resolve [C:4] [TYPE Function] [SEMANTICS scenario,execution,environment,prod,authority] +# @ingroup ScenarioExecution +# @BRIEF Resolve one configured environment into immutable non-PROD/PROD execution policy. +# @PRE config_manager is an application-owned configuration provider, never request metadata. +# @POST Returns a policy whose environment_id equals the supplied target, or raises +# ENVIRONMENT_NOT_CONFIGURED before any caller may create a durable execution side effect. +# @INVARIANT Stage=PROD or is_production=true always resolves PROD; no caller-supplied field can +# downgrade it, and a PREPROD record cannot be upgraded by request data. +# @RELATION CALLED_BY -> [ScenarioExecution.Runner.Start] +# @RELATION CALLED_BY -> [Api.ScenarioExecution.Routes.Start] +# @RELATION CALLED_BY -> [Api.ScenarioAutomation.DirectTrigger] +# @RELATION CALLED_BY -> [ScenarioAutomation.Trigger.Dispatch] +def resolve_environment_execution_policy( + environment_id: str, config_manager: Any +) -> EnvironmentExecutionPolicy: + """Return the configured class or fail closed without looking at caller supplied fields.""" + environment = config_manager.get_environment(environment_id) if config_manager is not None else None + if environment is None: + raise ValueError("ENVIRONMENT_NOT_CONFIGURED") + is_prod = bool(getattr(environment, "is_production", False)) or ( + str(getattr(environment, "stage", "") or "").upper() == "PROD" + ) + return EnvironmentExecutionPolicy( + environment_id=environment_id, + environment_class="PROD" if is_prod else "PREPROD", + ) +# #endregion ScenarioExecution.EnvironmentPolicy.Resolve +# #endregion ScenarioExecution.EnvironmentPolicy diff --git a/backend/src/services/dashboard_testing/execution/executor_registry.py b/backend/src/services/dashboard_testing/execution/executor_registry.py index deceb3d66..22e1c6759 100644 --- a/backend/src/services/dashboard_testing/execution/executor_registry.py +++ b/backend/src/services/dashboard_testing/execution/executor_registry.py @@ -2,6 +2,7 @@ # @defgroup ScenarioExecution Typed executor registry for deterministic step dispatch. # @BRIEF Resolve registered tool executors before external I/O. # @INVARIANT human is a lifecycle control and is never registered as an executor. +# @INVARIANT Resolution accepts an immutable ActionExecutionDescriptor, never a tool string/default. from __future__ import annotations from collections.abc import Callable @@ -13,15 +14,31 @@ class ScenarioExecutorRegistry: def __init__(self) -> None: self._executors: dict[str, Executor] = {} - def register(self, tool: str, executor: Executor) -> None: + def register(self, tool: str, executor: Executor, *, action: str | None = None) -> None: if tool == "human": raise ValueError("human is a lifecycle checkpoint, not an executor") - self._executors[tool] = executor + if action is None: + from src.services.dashboard_testing.scenario.templates import REGISTERED_ACTIONS - def resolve(self, tool: str) -> Executor: + actions = [ + registered_action + for registered_action, entry in REGISTERED_ACTIONS.items() + if entry["tool"] == tool + ] + if not actions: + raise ValueError("ACTION_DESCRIPTOR_REQUIRED") + for registered_action in actions: + self._executors[f"{tool}:{registered_action}"] = executor + return + self._executors[f"{tool}:{action}"] = executor + + def resolve(self, descriptor: dict[str, Any]) -> Executor: + tool, action = descriptor.get("tool"), descriptor.get("action") + if not isinstance(tool, str) or not isinstance(action, str): + raise ValueError("ACTION_DESCRIPTOR_REQUIRED") try: - return self._executors[tool] + return self._executors[f"{tool}:{action}"] except KeyError as exc: - raise ValueError(f"unknown executor tool: {tool}") from exc + raise ValueError(f"unknown action executor: {tool}:{action}") from exc # #endregion ScenarioExecution.ExecutorRegistry diff --git a/backend/src/services/dashboard_testing/execution/executors.py b/backend/src/services/dashboard_testing/execution/executors.py new file mode 100644 index 000000000..0d74ef3af --- /dev/null +++ b/backend/src/services/dashboard_testing/execution/executors.py @@ -0,0 +1,362 @@ +# #region ScenarioExecution.Executors [C:4] [TYPE Module] [SEMANTICS scenario,execution,executor,adapter,037,038,036] +# @defgroup ScenarioExecution Typed executors that reuse existing 037/038/036 infrastructure. +# @BRIEF Dispatch by tool to bounded adapters. Missing live sessions never manufacture PASS. +# @INVARIANT human is not registered. Unavailable live I/O returns typed inconclusive. +# @INVARIANT Assertion outcomes come from 037 compare_values, never from a hardcoded pass. +# @INVARIANT Browser PASS is materialized only from an explicit BrowserExecutionAdapter success, +# never from session_id or page_ref metadata. +# @REJECTED Treating a persisted browser session/page identifier as evidence of completed live I/O +# was rejected — a dead or absent Playwright context must remain typed inconclusive. +# @INVARIANT Superset API PASS is materialized only from an explicit SupersetExecutionAdapter +# success, never from query_result metadata. +# @REJECTED Treating a static query_result as proof of a live 037/Superset execution was rejected — +# no client response, authoritative model, or response envelope was verified. +# @INVARIANT Screenshot PASS is materialized only from an explicit ScreenshotExecutionAdapter +# success with durable evidence refs, never from capture_bytes metadata. +# @REJECTED Treating raw capture_bytes as proof of a real ScreenshotService/036 capture was +# rejected — bytes without a registered evidence boundary are not execution evidence. +from __future__ import annotations + +from hashlib import sha256 +from io import BytesIO +from typing import Any + +from src.schemas.dashboard_testing import ComparisonPolicy, ComparisonPolicyType, ComparisonStatus, NormalizedValue, ValueKind +from src.services.dashboard_testing.comparison import compare_values + +from .artifacts import has_verified_evidence, is_valid_sha256 +from .executor_registry import ScenarioExecutorRegistry +from .live_adapter import ( + BrowserAdapterResult, # noqa: F401 + BrowserExecutionAdapter, + ScreenshotAdapterResult, # noqa: F401 + ScreenshotExecutionAdapter, + SupersetAdapterResult, # noqa: F401 + SupersetExecutionAdapter, + dispatch_live_adapter, +) + +_STATUS = { + ComparisonStatus.PASS: "passed", + ComparisonStatus.FAIL: "failed", + ComparisonStatus.INCONCLUSIVE: "inconclusive", + ComparisonStatus.MISSING_BASELINE: "blocked", + ComparisonStatus.STALE_BASELINE: "blocked", + ComparisonStatus.STALE_VISUAL_BASELINE: "blocked", + ComparisonStatus.IMMUTABILITY_VIOLATION: "failed", + ComparisonStatus.PERMISSION_DENIED: "blocked", + ComparisonStatus.SOURCE_ERROR: "inconclusive", +} + +# #region ScenarioExecution.Executors.Outcome [C:2] [TYPE Function] [SEMANTICS scenario,execution,executor,outcome] +# @BRIEF Build one typed ScenarioStep outcome without manufacturing a passing result. +def _outcome( + tool: str, + status: str, + *, + reason: str, + extra: dict[str, Any] | None = None, + refs: list[str] | None = None, + output_refs: list[str] | None = None, + artifact_refs: list[str] | None = None, +) -> dict[str, Any]: + payload = {**(extra or {}), "tool": tool, "reason_code": reason} + refs = refs or [] + return { + "status": status, + "step_outcome": payload, + "output_refs": output_refs if output_refs is not None else refs, + "artifact_refs": artifact_refs if artifact_refs is not None else refs, + "error_code": None if status == "passed" else reason, + } +# #endregion ScenarioExecution.Executors.Outcome + + +# #region ScenarioExecution.Executors.StepPayload [C:1] [TYPE Function] [SEMANTICS scenario,execution,executor,payload] +# @BRIEF Merge structured step metadata with its top-level execution fields. +def _step_payload(step: dict[str, Any]) -> dict[str, Any]: + meta = step.get("step_meta") if isinstance(step.get("step_meta"), dict) else {} + return {**meta, **{k: v for k, v in step.items() if k != "step_meta"}} +# #endregion ScenarioExecution.Executors.StepPayload + + +# #region ScenarioExecution.Executors.NormalizeValue [C:2] [TYPE Function] [SEMANTICS scenario,execution,executor,value,comparison] +# @BRIEF Convert scalar/table inputs to the canonical 037 comparison value representation. +def _as_normalized(value: Any, *, source: str) -> NormalizedValue: + if isinstance(value, NormalizedValue): + return value + if isinstance(value, dict) and "kind" in value: + return NormalizedValue.model_validate(value) + if value is None: + return NormalizedValue(kind=ValueKind.NULL, raw_value=None, canonical_value=None, source=source) + if isinstance(value, bool): + return NormalizedValue(kind=ValueKind.BOOLEAN, raw_value=value, canonical_value=str(value).lower(), source=source) + if isinstance(value, int) and not isinstance(value, bool): + return NormalizedValue(kind=ValueKind.INTEGER, raw_value=value, canonical_value=str(value), source=source) + if isinstance(value, float): + return NormalizedValue(kind=ValueKind.DECIMAL, raw_value=value, canonical_value=str(value), source=source) + if isinstance(value, list): + return NormalizedValue(kind=ValueKind.TABLE, raw_value=value, canonical_value=str(value), source=source) + return NormalizedValue(kind=ValueKind.STRING, raw_value=value, canonical_value=str(value), source=source) +# #endregion ScenarioExecution.Executors.NormalizeValue + + +# #region ScenarioExecution.Executors.ComparisonPolicy [C:2] [TYPE Function] [SEMANTICS scenario,execution,executor,policy,comparison] +# @BRIEF Resolve one supported comparison policy from a scenario step payload. +def _policy_from(payload: dict[str, Any]) -> ComparisonPolicy: + raw = payload.get("policy") or payload.get("comparison_policy") or {} + if isinstance(raw, ComparisonPolicy): + return raw + if isinstance(raw, dict) and raw.get("type"): + return ComparisonPolicy.model_validate(raw) + policy_type = str(payload.get("policy_type") or "exact") + mapping = { + "exact": ComparisonPolicyType.EXACT, + "absolute_tolerance": ComparisonPolicyType.ABSOLUTE_TOLERANCE, + "relative_tolerance": ComparisonPolicyType.RELATIVE_TOLERANCE, + "range": ComparisonPolicyType.RANGE, + "row_set": ComparisonPolicyType.ROW_SET, + } + return ComparisonPolicy(type=mapping.get(policy_type, ComparisonPolicyType.EXACT)) +# #endregion ScenarioExecution.Executors.ComparisonPolicy + + +# #region ScenarioExecution.Executors.CompletedValue [C:2] [TYPE Function] [SEMANTICS scenario,execution,executor,dependency,value] +# @BRIEF Read the actual value previously materialized by one completed dependency step. +def _completed_value(completed: dict[str, dict[str, Any]], logical_step_id: str | None) -> Any: + if not logical_step_id: + return None + parent = completed.get(logical_step_id) or {} + return ( + parent.get("actual") + or (parent.get("step_outcome") or {}).get("actual") + or (parent.get("outputs") or {}).get("actual") + or parent.get("output") + ) +# #endregion ScenarioExecution.Executors.CompletedValue + + +# #region ScenarioExecution.Executors.Assertion [C:3] [TYPE Function] [SEMANTICS scenario,execution,executor,assertion,comparison] +def assertion(step: dict[str, Any], completed: dict[str, dict[str, Any]]) -> dict[str, Any]: + payload = _step_payload(step) + expected = payload.get("expected") + actual = payload.get("actual") + if actual is None: + actual = _completed_value(completed, payload.get("actual_from") or payload.get("source_step_id")) + if expected is None and actual is None: + return _outcome("assertion", "inconclusive", reason="ASSERTION_INPUTS_MISSING") + result = compare_values( + _as_normalized(actual, source="scenario_step"), + _as_normalized(expected, source="baseline"), + _policy_from(payload), + ) + status = _STATUS.get(result.status, "inconclusive") + return _outcome( + "assertion", + status, + reason=result.status.value.upper(), + extra={"comparison_status": result.status.value, "diff": [d.model_dump() for d in result.diff]}, + ) +# #endregion ScenarioExecution.Executors.Assertion + + +# #region ScenarioExecution.Executors.Browser [C:3] [TYPE Function] [SEMANTICS scenario,execution,executor,browser] +# @BRIEF Dispatch browser work only through an injected live adapter and map its typed result. +# @RELATION DEPENDS_ON -> [EXT:Playwright:BrowserTransport] +# @INVARIANT Browser PASS is materialized only from an explicit BrowserExecutionAdapter success, +# never from session_id or page_ref metadata. +# @REJECTED Synthesizing BROWSER_SESSION_REPLAY from IDs was rejected — identifiers cannot prove +# that a live browser action occurred. +def browser( + step: dict[str, Any], + completed: dict[str, dict[str, Any]], + *, + adapter: BrowserExecutionAdapter | None = None, +) -> dict[str, Any]: + payload = _step_payload(step) + context = { + key: payload[key] + for key in ("session_id", "page_ref") + if payload.get(key) is not None + } + return dispatch_live_adapter( + _outcome, + "browser", + step, + completed, + adapter, + context, + unavailable_code="BROWSER_ADAPTER_UNAVAILABLE", + timeout_code="BROWSER_ADAPTER_TIMEOUT", + error_code="BROWSER_ADAPTER_ERROR", + invalid_code="BROWSER_ADAPTER_INVALID_RESULT", + ) +# #endregion ScenarioExecution.Executors.Browser + + +# #region ScenarioExecution.Executors.SupersetApi [C:3] [TYPE Function] [SEMANTICS scenario,execution,executor,superset] +# @BRIEF Dispatch Superset work only through an injected 037 adapter and map its typed result. +# @RELATION DEPENDS_ON -> [EXT:Superset:ChartDataTransport] +# @INVARIANT Superset API PASS is materialized only from an explicit SupersetExecutionAdapter +# success, never from query_result metadata. +# @REJECTED Synthesizing SUPERSET_QUERY_BOUND from step metadata was rejected — static values do +# not prove the 037 authoritative query pipeline ran. +def superset_api( + step: dict[str, Any], + completed: dict[str, dict[str, Any]], + *, + adapter: SupersetExecutionAdapter | None = None, +) -> dict[str, Any]: + payload = _step_payload(step) + context = { + key: payload[key] + for key in ("environment_id", "dashboard_id", "chart_id", "dataset_id", "result_key") + if payload.get(key) is not None + } + return dispatch_live_adapter( + _outcome, + "superset_api", + step, + completed, + adapter, + context, + unavailable_code="SUPERSET_ADAPTER_UNAVAILABLE", + timeout_code="SUPERSET_ADAPTER_TIMEOUT", + error_code="SUPERSET_ADAPTER_ERROR", + invalid_code="SUPERSET_ADAPTER_INVALID_RESULT", + ) +# #endregion ScenarioExecution.Executors.SupersetApi + + +# #region ScenarioExecution.Executors.Screenshot [C:3] [TYPE Function] [SEMANTICS scenario,execution,executor,screenshot,evidence] +# @BRIEF Dispatch screenshot capture only through the existing ScreenshotService/036 adapter. +# @RELATION DEPENDS_ON -> [Plugin.Service.ScreenshotService] +# @INVARIANT Screenshot PASS is materialized only from explicit ScreenshotExecutionAdapter success +# with durable evidence refs, never from capture_bytes metadata. +# @INVARIANT Screenshot PASS requires one non-zero SHA-256 agreeing with the forwarded artifact digest. +# @REJECTED Computing SCREENSHOT_DIGESTED from caller metadata was rejected — only the adapter can +# prove ScreenshotService capture and 036 evidence registration completed. +def screenshot( + step: dict[str, Any], + completed: dict[str, dict[str, Any]], + *, + adapter: ScreenshotExecutionAdapter | None = None, +) -> dict[str, Any]: + payload = _step_payload(step) + context = { + key: payload[key] + for key in ("environment_id", "dashboard_id", "capture_profile_id") + if payload.get(key) is not None + } + outcome = dispatch_live_adapter( + _outcome, + "screenshot", + step, + completed, + adapter, + context, + unavailable_code="SCREENSHOT_ADAPTER_UNAVAILABLE", + timeout_code="SCREENSHOT_ADAPTER_TIMEOUT", + error_code="SCREENSHOT_ADAPTER_ERROR", + invalid_code="SCREENSHOT_ADAPTER_INVALID_RESULT", + required_pass_detail_keys=("sha256",), + require_pass_artifact_refs=True, + missing_evidence_code="SCREENSHOT_EVIDENCE_REQUIRED", + ) + if outcome["status"] != "passed" or has_verified_evidence( + outcome["artifact_refs"], outcome["step_outcome"] + ): + return outcome + return _outcome( + "screenshot", + "inconclusive", + reason="SCREENSHOT_EVIDENCE_INVALID", + extra=context, + output_refs=outcome["output_refs"], + artifact_refs=[], + ) +# #endregion ScenarioExecution.Executors.Screenshot + + +# #region ScenarioExecution.Executors.Xlsx [C:3] [TYPE Function] [SEMANTICS scenario,execution,executor,xlsx,openpyxl] +def xlsx(step: dict[str, Any], _completed: dict[str, dict[str, Any]]) -> dict[str, Any]: + payload = _step_payload(step) + content = payload.get("xlsx_bytes") or payload.get("content_bytes") + if not content: + return _outcome("xlsx", "inconclusive", reason="XLSX_BYTES_REQUIRED") + try: + from openpyxl import load_workbook + except ImportError: + return _outcome("xlsx", "inconclusive", reason="OPENPYXL_UNAVAILABLE") + workbook = load_workbook(BytesIO(content), read_only=True, data_only=True) + try: + sheet = workbook.active + rows = [list(row) for row in sheet.iter_rows(values_only=True)] + finally: + workbook.close() + digest = sha256(content if isinstance(content, bytes) else str(content).encode()).hexdigest() + return _outcome( + "xlsx", + "passed", + reason="XLSX_PARSED", + extra={"row_count": len(rows), "actual": rows, "sha256": digest}, + refs=[f"xlsx:{digest}"], + ) +# #endregion ScenarioExecution.Executors.Xlsx + + +# #region ScenarioExecution.Executors.Report [C:2] [TYPE Function] [SEMANTICS scenario,execution,executor,report] +def report(step: dict[str, Any], completed: dict[str, dict[str, Any]]) -> dict[str, Any]: + payload = _step_payload(step) + sources = payload.get("source_step_ids") or list(completed) + if not sources: + return _outcome("report", "inconclusive", reason="REPORT_SOURCES_MISSING") + digest = sha256("|".join(str(item) for item in sources).encode()).hexdigest() + report_ref = f"report:{digest}" + return _outcome( + "report", + "passed", + reason="REPORT_COMPILED", + extra={"source_step_ids": sources}, + output_refs=[report_ref], + artifact_refs=[], + ) +# #endregion ScenarioExecution.Executors.Report + + +# #region ScenarioExecution.Executors.Artifact [C:2] [TYPE Function] [SEMANTICS scenario,execution,executor,artifact] +# @INVARIANT Artifact PASS requires a canonical, non-zero SHA-256 digest for the declared ref. +# @REJECTED Accepting a 64-character sentinel or non-hex digest was rejected — it forges provenance. +def artifact(step: dict[str, Any], _completed: dict[str, dict[str, Any]]) -> dict[str, Any]: + payload = _step_payload(step) + content_ref = payload.get("content_ref") or payload.get("artifact_ref") + digest = payload.get("sha256") + if not content_ref or not digest: + return _outcome("artifact", "inconclusive", reason="ARTIFACT_REF_REQUIRED") + if not is_valid_sha256(digest): + return _outcome("artifact", "failed", reason="ARTIFACT_DIGEST_INVALID") + return _outcome( + "artifact", "passed", reason="ARTIFACT_BOUND", + extra={"content_ref": content_ref, "sha256": digest.lower()}, refs=[str(content_ref)], + ) +# #endregion ScenarioExecution.Executors.Artifact + + +# #region ScenarioExecution.Executors.RegisterDefaults [C:2] [TYPE Function] [SEMANTICS scenario,execution,executor,registry] +# @BRIEF Register bounded built-in executors and optional injected live-I/O adapters. +def _register_default_executors( + registry: ScenarioExecutorRegistry, + *, + browser_adapter: BrowserExecutionAdapter | None = None, + superset_adapter: SupersetExecutionAdapter | None = None, + screenshot_adapter: ScreenshotExecutionAdapter | None = None, +) -> None: + registry.register("assertion", assertion) + registry.register("browser", lambda step, completed: browser(step, completed, adapter=browser_adapter)) + registry.register("superset_api", lambda step, completed: superset_api(step, completed, adapter=superset_adapter)) + registry.register("xlsx", xlsx) + registry.register("screenshot", lambda step, completed: screenshot(step, completed, adapter=screenshot_adapter)) + registry.register("report", report) + registry.register("artifact", artifact) +# #endregion ScenarioExecution.Executors.RegisterDefaults +# #endregion ScenarioExecution.Executors diff --git a/backend/src/services/dashboard_testing/execution/lifecycle.py b/backend/src/services/dashboard_testing/execution/lifecycle.py index 5dc260a7b..dfba48820 100644 --- a/backend/src/services/dashboard_testing/execution/lifecycle.py +++ b/backend/src/services/dashboard_testing/execution/lifecycle.py @@ -2,13 +2,26 @@ # @defgroup ScenarioExecution Run lifecycle controls, checkpoints, retry invalidation, infra resume. from __future__ import annotations -from datetime import UTC, datetime +import copy +from datetime import UTC, datetime, timedelta import secrets +from sqlalchemy import update from sqlalchemy.orm import Session +from src.models.scenario_approval import ActionApprovalGate from src.models.scenario_checkpoint import HumanCheckpoint from src.models.scenario_run import ScenarioRun, ScenarioStepRun +from src.models.scenario_worker import ScenarioStepLease + +from .artifacts import invalidate_step_evidence +from .runner_plan import validate_pinned_runner_plan + +_DEFAULT_CANCEL_DRAIN_SECONDS = 30 + + +def _as_utc(value: datetime) -> datetime: + return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC) def _descendants(step_id: str, edges: list[dict[str, str]]) -> set[str]: @@ -25,6 +38,97 @@ def _descendants(step_id: str, edges: list[dict[str, str]]) -> set[str]: pending.extend(children.get(node, set())) return result + +# #region ScenarioExecution.Lifecycle.TimeoutClosure [C:4] [TYPE Function] [SEMANTICS scenario,execution,timeout,dag,provenance] +# @BRIEF Materialize missing descendants from the immutable persisted runner plan as blocked rows. +# @RELATION CALLED_BY -> [ScenarioExecution.Lifecycle.Timeout] +# @INVARIANT Every declared timeout descendant has a durable blocked row before terminalization; +# completed rows and nodes outside the pinned descendant closure are never overwritten. +# @REJECTED Re-deriving the graph from a mutable scenario revision was rejected — timeout closure +# must use the run's pinned plan and its original topological position/step metadata. +def _materialize_timeout_descendants( + db: Session, + run: ScenarioRun, + *, + source_step_id: str, + descendant_ids: set[str], + existing_step_ids: set[str], +) -> list[ScenarioStepRun]: + plan = run.runner_plan or {} + order = [str(step_id) for step_id in plan.get("topological_order", [])] + plan_steps = { + str(step.get("logical_step_id", step.get("id", ""))): copy.deepcopy(step) + for step in plan.get("steps", []) + if isinstance(step, dict) + } + materialized: list[ScenarioStepRun] = [] + for position, descendant_id in enumerate(order): + if descendant_id not in descendant_ids or descendant_id in existing_step_ids: + continue + metadata = plan_steps.get(descendant_id, {"logical_step_id": descendant_id}) + row = ScenarioStepRun( + run_id=run.id, + logical_step_id=descendant_id, + step_position=position, + attempt=1, + status="blocked", + inputs_snapshot={"runner_plan_step": metadata}, + outputs={}, + artifact_refs=[], + progress=100, + error_code="UPSTREAM_TIMEOUT", + step_outcome={ + "status": "blocked", + "error_code": "UPSTREAM_TIMEOUT", + "blocked_by": source_step_id, + }, + finished_at=datetime.now(UTC), + ) + db.add(row) + materialized.append(row) + if descendant_ids - set(order): + raise ValueError("runner plan timeout descendant is not topologically declared") + db.flush() + return materialized +# #endregion ScenarioExecution.Lifecycle.TimeoutClosure + + +# #region ScenarioExecution.Lifecycle.TerminalLease [C:3] [TYPE Function] [SEMANTICS scenario,execution,lifecycle,terminal,lease] +# @BRIEF Revoke active worker claims for a terminal or forcibly drained run without deleting lease audit rows. +# @INVARIANT A terminal ScenarioRun has no unexpired lease; worker claim history remains durable. +def _expire_step_leases(db: Session, run_id: str, logical_step_ids: set[str] | None = None) -> None: + query = db.query(ScenarioStepLease).filter(ScenarioStepLease.run_id == run_id).with_for_update() + if logical_step_ids is not None: + query = query.filter(ScenarioStepLease.logical_step_id.in_(sorted(logical_step_ids))) + expires_at = datetime.now(UTC) + for lease in query.all(): + lease.expires_at = expires_at +# #endregion ScenarioExecution.Lifecycle.TerminalLease + + +# #region ScenarioExecution.Lifecycle.TerminalProjection [C:3] [TYPE Function] [SEMANTICS scenario,execution,lifecycle,terminal,artifact,provenance] +# @BRIEF Archive active result attachment before a terminal cancellation or timeout replaces it. +# @INVARIANT Artifact rows and prior output are retained as explicit history, but active refs and +# outcomes cannot leak into the terminal current-result projection. +def _retire_active_step_projection(db: Session, run_id: str, steps: list[ScenarioStepRun]) -> None: + affected = {step.logical_step_id for step in steps} + invalidate_step_evidence(db, run_id=run_id, logical_step_ids=affected) + for step in steps: + prior_outputs = copy.deepcopy(step.outputs or {}) + history = list(prior_outputs.pop("terminal_history", [])) + history.append({ + "attempt": step.attempt, + "status": step.status, + "outputs": prior_outputs, + "artifact_refs": list(step.artifact_refs or []), + "error_code": step.error_code, + "step_outcome": copy.deepcopy(step.step_outcome or {}), + }) + step.outputs = {"terminal_history": history} + step.artifact_refs = [] + step.step_outcome = {} +# #endregion ScenarioExecution.Lifecycle.TerminalProjection + # #region ScenarioExecution.Lifecycle.Suspend [C:4] [TYPE Function] [SEMANTICS scenario,execution,human,checkpoint,suspend] def suspend_for_human(db: Session, run_id: str, logical_step_id: str, evidence_refs: list[str]) -> HumanCheckpoint: run = db.query(ScenarioRun).filter(ScenarioRun.id == run_id).first() @@ -39,68 +143,323 @@ def suspend_for_human(db: Session, run_id: str, logical_step_id: str, evidence_r # #endregion ScenarioExecution.Lifecycle.Suspend # #region ScenarioExecution.Lifecycle.Decide [C:4] [TYPE Function] [SEMANTICS scenario,execution,human,checkpoint,cas,decision] +# @POST Atomically consumes the checkpoint and materializes its deterministic step outcome; +# the run returns to queued so the runner can continue the ready frontier. +# @INVARIANT confirm/pass records passed; false_positive/inconclusive records inconclusive, +# never a fabricated PASS. A failed disposition remains failed and blocks dependents. +# @REJECTED Treating false_positive as a passing execution result — the observation remains an +# explicit non-pass checkpoint outcome even when dependents can collect more evidence. def decide_checkpoint(db: Session, checkpoint_id: str, *, disposition: str, expected_version: int, actor_id: str, comment: str = "") -> HumanCheckpoint: + if disposition not in {"confirm", "false_positive", "inconclusive", "pass", "fail"}: + raise ValueError("invalid disposition") + decided_at = datetime.now(UTC) + consumed = db.execute( + update(HumanCheckpoint) + .where( + HumanCheckpoint.id == checkpoint_id, + HumanCheckpoint.status == "pending", + HumanCheckpoint.decision_version == expected_version, + ) + .values( + status="decided", + disposition=disposition, + comment=comment, + decided_by=actor_id, + decided_at=decided_at, + decision_version=HumanCheckpoint.decision_version + 1, + ) + ) + if consumed.rowcount != 1: + exists = db.query(HumanCheckpoint.id).filter(HumanCheckpoint.id == checkpoint_id).first() + raise ValueError("stale checkpoint decision" if exists is not None else "checkpoint not found") checkpoint = db.query(HumanCheckpoint).filter(HumanCheckpoint.id == checkpoint_id).first() if checkpoint is None: raise ValueError("checkpoint not found") - if checkpoint.status != "pending" or checkpoint.decision_version != expected_version: - raise ValueError("stale checkpoint decision") - if disposition not in {"confirm", "false_positive", "inconclusive", "pass", "fail"}: - raise ValueError("invalid disposition") - checkpoint.status = "decided" - checkpoint.disposition = disposition - checkpoint.comment = comment - checkpoint.decided_by = actor_id - checkpoint.decided_at = datetime.now(UTC) - checkpoint.decision_version += 1 run = db.query(ScenarioRun).filter(ScenarioRun.id == checkpoint.run_id).first() if run is not None: - run.status = {"fail": "failed", "confirm": "failed", "false_positive": "passed", "pass": "passed", "inconclusive": "inconclusive"}[disposition] + outcome_status = { + "confirm": "passed", + "pass": "passed", + "false_positive": "inconclusive", + "inconclusive": "inconclusive", + "fail": "failed", + }[disposition] + step = db.query(ScenarioStepRun).filter( + ScenarioStepRun.run_id == run.id, + ScenarioStepRun.logical_step_id == checkpoint.logical_step_id, + ).first() + if step is None: + raise ValueError("checkpoint step not found") + step.status = outcome_status + step.progress = 100 + step.error_code = None if outcome_status == "passed" else f"HUMAN_{disposition.upper()}" + step.step_outcome = { + "logical_step_id": checkpoint.logical_step_id, + "status": outcome_status, + "disposition": disposition, + "comment": comment, + } + step.finished_at = checkpoint.decided_at + run.status = "queued" run.phase = "executing" db.flush() return checkpoint # #endregion ScenarioExecution.Lifecycle.Decide -# #region ScenarioExecution.Lifecycle.Cancel [C:3] [TYPE Function] [SEMANTICS scenario,execution,cancel,run] -def cancel_run(db: Session, run_id: str) -> ScenarioRun: +# #region ScenarioExecution.Lifecycle.Cancel [C:5] [TYPE Function] [SEMANTICS scenario,execution,cancel,drain,lease,artifact] +# @ingroup ScenarioExecution +# @BRIEF Request bounded cancellation, then terminalize after the active adapter completes or its persisted drain deadline expires. +# @PRE A run is not passed/failed/blocked/inconclusive; cancellation does not decide a HumanCheckpoint +# or approve a PROD ApprovalGate. +# @POST Queued work becomes skipped. The first request pins an immutable drain deadline. A running +# step may finish only before that deadline; the deadline finalizer marks remaining work cancelled. +# @INVARIANT A terminal cancelled run has no running/queued step, no unexpired worker lease, and +# no active evidence projection. Artifact/lease rows are retained for audit. +# @INVARIANT Pending human checkpoints are cancelled (not decided), and a pending PROD gate expires +# (not approved/denied by an actor), so neither control can reactivate a cancelled run. +# @RATIONALE The terminal closure intentionally remains one transaction over steps, leases, evidence, +# checkpoints, and gates so concurrent lifecycle controls cannot observe a partial cancel. +# @REJECTED Treating cancellation as a successful result or deleting partial evidence was rejected. +def _request_cancel_deadline(run: ScenarioRun, *, requested_at: datetime, drain_seconds: int) -> bool: + if run.status != "cancel_requested": + run.status = "cancel_requested" + run.cancel_requested_at = requested_at + run.cancel_drain_deadline_at = requested_at + timedelta(seconds=drain_seconds) + deadline = run.cancel_drain_deadline_at + return deadline is not None and _as_utc(deadline) <= requested_at + + +def _prepare_cancelled_steps(steps: list[ScenarioStepRun], *, cancelled_at: datetime) -> list[ScenarioStepRun]: + running: list[ScenarioStepRun] = [] + for step in steps: + if step.status == "queued": + step.status = "skipped" + step.progress = 100 + step.error_code = "CANCELLED_BEFORE_DISPATCH" + step.finished_at = cancelled_at + elif step.status == "running": + running.append(step) + return running + + +def _cancel_pending_controls(db: Session, run_id: str, *, cancelled_at: datetime) -> None: + for checkpoint in db.query(HumanCheckpoint).filter( + HumanCheckpoint.run_id == run_id, + HumanCheckpoint.status == "pending", + ).with_for_update().all(): + checkpoint.status = "cancelled" + checkpoint.decided_at = cancelled_at + for gate in db.query(ActionApprovalGate).filter( + ActionApprovalGate.owner_type == "scenario_run", + ActionApprovalGate.owner_id == run_id, + ActionApprovalGate.status == "pending", + ).with_for_update().all(): + gate.status = "expired" + gate.comment = "scenario run cancelled" + gate.decided_at = cancelled_at + + +def cancel_run( + db: Session, + run_id: str, + *, + drain_in_flight: bool = True, + drain_seconds: int = _DEFAULT_CANCEL_DRAIN_SECONDS, +) -> ScenarioRun: + if drain_seconds <= 0: + raise ValueError("drain_seconds must be positive") run = db.query(ScenarioRun).filter(ScenarioRun.id == run_id).first() if run is None: raise ValueError("run not found") - if run.status in {"passed", "failed", "cancelled", "inconclusive"}: + if run.status == "cancelled": + return run + if run.status in {"passed", "failed", "blocked", "inconclusive"}: raise ValueError("run is terminal") - run.status = "cancelled" + cancellation_at = datetime.now(UTC) + # A retry of cancel never extends the original bounded window. + deadline_expired = _request_cancel_deadline( + run, + requested_at=cancellation_at, + drain_seconds=drain_seconds, + ) run.phase = "draining" - run.finished_at = datetime.now(UTC) + steps = db.query(ScenarioStepRun).filter(ScenarioStepRun.run_id == run_id).all() + running = _prepare_cancelled_steps(steps, cancelled_at=cancellation_at) + if running and drain_in_flight and not deadline_expired: + db.flush() + return run + + if running: + for step in running: + step.status = "cancelled" + step.progress = 100 + step.error_code = "CANCEL_DRAIN_EXPIRED" + step.step_outcome = {"status": "cancelled", "error_code": "CANCEL_DRAIN_EXPIRED"} + step.finished_at = cancellation_at + _cancel_pending_controls(db, run_id, cancelled_at=cancellation_at) + _retire_active_step_projection(db, run_id, steps) + _expire_step_leases(db, run_id) + run.status = "cancelled" + run.phase = "terminal" + run.finished_at = cancellation_at db.flush() return run # #endregion ScenarioExecution.Lifecycle.Cancel -# #region ScenarioExecution.Lifecycle.Retry [C:4] [TYPE Function] [SEMANTICS scenario,execution,retry,downstream] +# #region ScenarioExecution.Lifecycle.CancelFinalizer [C:4] [TYPE Function] [SEMANTICS scenario,execution,cancel,drain,deadline] +# @BRIEF Finalize cancellation requests whose persisted bounded drain window has expired. +# @PRE The caller supplies the normal worker/scheduler database session; no adapter payload is fabricated. +# @POST Every returned run is terminal cancelled and has no active lease or active evidence projection. +# @INVARIANT A repeated sweep cannot extend a deadline, resurrect a run, or duplicate a terminal signal. +# @REJECTED Trusting an in-memory timer was rejected — a process restart would strand the persisted run. +def finalize_expired_cancellations(db: Session, *, now: datetime | None = None) -> list[ScenarioRun]: + observed_at = now or datetime.now(UTC) + candidates = ( + db.query(ScenarioRun) + .filter( + ScenarioRun.status == "cancel_requested", + ScenarioRun.cancel_drain_deadline_at.is_not(None), + ) + .with_for_update() + .all() + ) + terminalized: list[ScenarioRun] = [] + for run in candidates: + if _as_utc(run.cancel_drain_deadline_at) <= observed_at: + terminalized.append(cancel_run(db, run.id, drain_in_flight=False)) + return terminalized +# #endregion ScenarioExecution.Lifecycle.CancelFinalizer + +# #region ScenarioExecution.Lifecycle.RetryEligibility [C:3] [TYPE Function] [SEMANTICS scenario,execution,retry,lifecycle,human,approval] +# @BRIEF Refuse a retry before it can revive a terminal-success, cancelled, approval-pending, or human-held run. +# @INVARIANT A retry may reopen only queued|running|failed|blocked|inconclusive runs with no pending checkpoint. +def _require_retry_eligible_run(db: Session, run: ScenarioRun) -> None: + if run.status in {"passed", "cancel_requested", "cancelled", "pending_approval", "waiting_human"}: + raise ValueError("run is not retry-eligible") + if run.status not in {"queued", "running", "failed", "blocked", "inconclusive"}: + raise ValueError("run is not retry-eligible") + if db.query(HumanCheckpoint).filter( + HumanCheckpoint.run_id == run.id, + HumanCheckpoint.status == "pending", + ).first() is not None: + raise ValueError("cannot retry while human checkpoint is pending") +# #endregion ScenarioExecution.Lifecycle.RetryEligibility + + +# #region ScenarioExecution.Lifecycle.RetryArchive [C:3] [TYPE Function] [SEMANTICS scenario,execution,retry,attempt,archive] +# @BRIEF Move one active step attempt into explicit historical provenance before clearing its active state. +# @INVARIANT Historical attempts are labelled by their original attempt/status and never enter current artifact_refs/outcome aggregation. +def _archive_step_attempt(step: ScenarioStepRun) -> None: + previous_outputs = copy.deepcopy(step.outputs or {}) + history = list(previous_outputs.pop("attempt_history", [])) + history.append({ + "attempt": step.attempt, + "status": step.status, + "inputs_snapshot": copy.deepcopy(step.inputs_snapshot or {}), + "outputs": previous_outputs, + "artifact_refs": list(step.artifact_refs or []), + "error_code": step.error_code, + "step_outcome": copy.deepcopy(step.step_outcome or {}), + }) + step.status = "queued" + step.inputs_snapshot = {} + step.outputs = {"attempt_history": history} + step.artifact_refs = [] + step.step_outcome = {} + step.error_code = None + step.progress = 0 + step.started_at = None + step.finished_at = None + step.attempt += 1 +# #endregion ScenarioExecution.Lifecycle.RetryArchive + + +# #region ScenarioExecution.Lifecycle.RetryLease [C:3] [TYPE Function] [SEMANTICS scenario,execution,retry,lease,idempotency] +# @BRIEF Expire completed safe leases so the next retry worker can claim the persisted frontier. +# @INVARIANT An unsafe external effect is never reopened automatically; reconciliation is required. +def _expire_retry_safe_leases(db: Session, run_id: str, affected: set[str]) -> None: + leases = ( + db.query(ScenarioStepLease) + .filter( + ScenarioStepLease.run_id == run_id, + ScenarioStepLease.logical_step_id.in_(sorted(affected)), + ) + .with_for_update() + .all() + ) + if any(not (lease.idempotent or lease.retry_safe) for lease in leases): + raise ValueError("non-retry-safe side effect requires reconciliation") + retry_started_at = datetime.now(UTC) + for lease in leases: + lease.expires_at = retry_started_at +# #endregion ScenarioExecution.Lifecycle.RetryLease + + +# #region ScenarioExecution.Lifecycle.Retry [C:5] [TYPE Function] [SEMANTICS scenario,execution,retry,downstream,artifact,signal] +# @ingroup ScenarioExecution +# @BRIEF Invalidate one eligible failed frontier and all persisted descendants before a new attempt. +# @RELATION CALLS -> [ScenarioExecution.Artifacts.InvalidateStepEvidence] +# @PRE Target is failed|inconclusive|blocked, no human checkpoint is pending, the run is not +# cancelled/passed/pending approval, and every invalidated persisted row has retry budget. +# @POST The run returns to queued/executing; active outputs/outcomes/artifact refs are cleared for +# the target closure, each affected row advances its attempt, and historical attempts remain +# explicitly archived in outputs.attempt_history. +# @INVARIANT Artifact rows are retained as immutable audit evidence but their active projection is +# retired before the walker can produce a new terminal result or 047 signal. +# @INVARIANT A completed lease is expired for the new closure only when its recorded external +# effect exactly matches an immutable descriptor marked idempotent or retry-safe; +# unsafe/unknown effects require reconciliation. +# @REJECTED Inheriting retry authority from a tool or stale lease flags was rejected because a +# browser/action mutation could be replayed after its plan contract changed. +# @REJECTED Resetting only the named step was rejected — descendants can otherwise aggregate stale +# outcomes/evidence into a new result. Deleting historical artifacts was also rejected. def retry_step(db: Session, run_id: str, logical_step_id: str, *, max_attempts: int = 3) -> list[ScenarioStepRun]: + if max_attempts < 1: + raise ValueError("max_attempts must be positive") run = db.query(ScenarioRun).filter(ScenarioRun.id == run_id).first() if run is None: raise ValueError("run not found") + validate_pinned_runner_plan(run.runner_plan or {}) + _require_retry_eligible_run(db, run) steps = db.query(ScenarioStepRun).filter(ScenarioStepRun.run_id == run_id).order_by(ScenarioStepRun.step_position.asc()).all() target = next((step for step in steps if step.logical_step_id == logical_step_id), None) if target is None: raise ValueError("step not found") - if target.attempt >= max_attempts: - raise ValueError("retry attempts exceeded") + if target.status not in {"failed", "inconclusive", "blocked"}: + raise ValueError("step is not retry-eligible") affected = {logical_step_id, *_descendants(logical_step_id, list((run.runner_plan or {}).get("dependencies") or []))} - for step in steps: - if step.logical_step_id in affected: - step.status = "queued" - step.outputs = {} - step.step_outcome = {} - step.attempt += 1 + affected_steps = [step for step in steps if step.logical_step_id in affected] + descriptor_by_step = { + str(item.get("logical_step_id", item.get("id", ""))): item.get("action_descriptor") or {} + for item in (run.runner_plan or {}).get("steps", []) + if isinstance(item, dict) + } + if any( + not ( + bool(descriptor_by_step.get(step.logical_step_id, {}).get("idempotent")) + or bool(descriptor_by_step.get(step.logical_step_id, {}).get("retry_safe")) + ) + for step in affected_steps + ): + raise ValueError("non-retry-safe action descriptor requires reconciliation") + if any(step.attempt >= max_attempts for step in affected_steps): + raise ValueError("retry attempts exceeded") + _expire_retry_safe_leases(db, run_id, affected) + invalidate_step_evidence(db, run_id=run_id, logical_step_ids=affected) + for step in affected_steps: + _archive_step_attempt(step) + run.status = "queued" + run.phase = "executing" + run.error_code = None + run.finished_at = None db.flush() - return [step for step in steps if step.logical_step_id in affected] + return affected_steps # #endregion ScenarioExecution.Lifecycle.Retry # #region ScenarioExecution.Lifecycle.PauseInfra [C:4] [TYPE Function] [SEMANTICS scenario,execution,pause,infrastructure,resume-token] # @ingroup ScenarioExecution # @BRIEF Pause a non-terminal, non-human run for infrastructure reasons; mint a resume token. -# @PRE run exists, is not terminal and is not waiting on a HumanCheckpoint. +# @PRE run exists, is actively queued/running, is not terminal and is not waiting on a HumanCheckpoint. # @POST run.phase == paused; run.resume_token set; returns (run, token). # @SIDE_EFFECT DB write; token minted via secrets.token_urlsafe. # @REJECTED Using the HumanCheckpoint flow for infra pauses — checkpoint disposition is observation, @@ -113,6 +472,8 @@ def pause_for_infrastructure(db: Session, run_id: str) -> tuple[ScenarioRun, str raise ValueError("run is terminal") if run.status == "waiting_human": raise ValueError("cannot pause a run waiting on a human checkpoint") + if run.status not in {"queued", "running"}: + raise ValueError("run is not active") token = secrets.token_urlsafe(32) run.resume_token = token run.phase = "paused" @@ -125,7 +486,7 @@ def pause_for_infrastructure(db: Session, run_id: str) -> tuple[ScenarioRun, str # @ingroup ScenarioExecution # @BRIEF Resume an infrastructure-paused run from its resume token. # @PRE run exists; token matches run.resume_token; run not terminal; no pending HumanCheckpoint. -# @POST run.status -> queued, run.phase -> executing; token consumed (cleared). +# @POST run.status -> queued, run.phase -> executing; token consumed (cleared) before walker continuation. # @SIDE_EFFECT DB write; never re-runs completed steps (dispatch resumes from ready frontier). # @TEST_EDGE human_checkpoint_pending -> reject; stale_token -> reject; terminal -> reject. def resume_run(db: Session, run_id: str, *, resume_token: str, resume_reason: str) -> ScenarioRun: @@ -138,21 +499,111 @@ def resume_run(db: Session, run_id: str, *, resume_token: str, resume_reason: st raise ValueError("run is terminal") if run.status == "waiting_human": raise ValueError("cannot resume a human checkpoint; use /human/decision") + if run.status not in {"queued", "running"}: + raise ValueError("run is not active") pending_checkpoint = db.query(HumanCheckpoint).filter( HumanCheckpoint.run_id == run_id, HumanCheckpoint.status == "pending" ).first() if pending_checkpoint is not None: raise ValueError("cannot resume a human checkpoint; use /human/decision") + if run.phase != "paused": + raise ValueError("run is not paused") if not run.resume_token: raise ValueError("run is not paused") if not secrets.compare_digest(run.resume_token, resume_token): raise ValueError("stale resume token") run.resume_token = None + run.status = "queued" run.phase = "executing" - if run.status == "queued": - run.status = "queued" db.flush() return run # #endregion ScenarioExecution.Lifecycle.Resume +# #region ScenarioExecution.Lifecycle.Timeout [C:5] [TYPE Function] [SEMANTICS scenario,execution,timeout,step,lease,artifact,signal] +# @ingroup ScenarioExecution +# @BRIEF Materialize a valid step timeout as an explicit inconclusive outcome. +# @RELATION CALLS -> [ScenarioExecution.Lifecycle.TimeoutClosure] +# @PRE timeout_ms is positive and the addressed step belongs to the run. +# @POST The timed attempt becomes inconclusive; every declared queued/running or unmaterialized +# descendant is durably blocked from the pinned runner plan before the run terminalizes. +# @SIDE_EFFECT DB update of closure steps/run, active evidence projection, and worker leases. +# @INVARIANT A timeout never manufactures a passing outcome, leaves no active lease/evidence, and +# cannot emit more than one signal for the same terminal attempt context. A later walker +# sees every declared descendant as blocked and therefore cannot lazy-dispatch it. +# @RATIONALE Timeout is persisted as an execution result so later result assembly observes a durable state. +# @REJECTED Raising only at the worker boundary was rejected — the durable step/run state would remain ambiguous. +def apply_step_timeout(db: Session, run_id: str, logical_step_id: str, *, timeout_ms: int) -> ScenarioStepRun: + step = db.query(ScenarioStepRun).filter(ScenarioStepRun.run_id == run_id, ScenarioStepRun.logical_step_id == logical_step_id).first() + if step is None: + raise ValueError("step not found") + if timeout_ms <= 0: + raise ValueError("timeout_ms must be positive") + if step.status not in {"running", "queued", "waiting_human"}: + return step + if step.status == "waiting_human" and db.query(HumanCheckpoint).filter( + HumanCheckpoint.run_id == run_id, + HumanCheckpoint.status == "pending", + ).first() is not None: + raise ValueError("cannot timeout a human checkpoint") + run = db.query(ScenarioRun).filter(ScenarioRun.id == run_id).first() + if run is None: + raise ValueError("run not found") + descendant_ids = _descendants(logical_step_id, list((run.runner_plan or {}).get("dependencies") or [])) + affected_ids = {logical_step_id, *descendant_ids} + affected_steps = ( + db.query(ScenarioStepRun) + .filter( + ScenarioStepRun.run_id == run_id, + ScenarioStepRun.logical_step_id.in_(sorted(affected_ids)), + ) + .all() + ) + existing_by_step_id = {row.logical_step_id: row for row in affected_steps} + _materialize_timeout_descendants( + db, + run, + source_step_id=logical_step_id, + descendant_ids=descendant_ids, + existing_step_ids=set(existing_by_step_id), + ) + blockable_descendants = [ + row + for row in affected_steps + if row.logical_step_id != logical_step_id and row.status in {"queued", "running", "waiting_human"} + ] + for checkpoint in db.query(HumanCheckpoint).filter( + HumanCheckpoint.run_id == run_id, + HumanCheckpoint.logical_step_id.in_([row.logical_step_id for row in blockable_descendants]), + HumanCheckpoint.status == "pending", + ).with_for_update().all(): + checkpoint.status = "cancelled" + checkpoint.decided_at = datetime.now(UTC) + _retire_active_step_projection(db, run_id, [step, *blockable_descendants]) + _expire_step_leases(db, run_id, affected_ids) + step.status = "inconclusive" + step.error_code = "STEP_TIMEOUT" + step.step_outcome = {"status": "inconclusive", "error_code": "STEP_TIMEOUT"} + step.finished_at = datetime.now(UTC) + step.progress = 100 + for descendant in blockable_descendants: + descendant.status = "blocked" + descendant.error_code = "UPSTREAM_TIMEOUT" + descendant.step_outcome = { + "status": "blocked", + "error_code": "UPSTREAM_TIMEOUT", + "blocked_by": logical_step_id, + } + descendant.progress = 100 + descendant.finished_at = datetime.now(UTC) + run.status = "inconclusive" + run.phase = "completed" + run.finished_at = datetime.now(UTC) + db.flush() + from src.services.dashboard_testing.analytics.investigation import emit_terminal_run_signal + + emit_terminal_run_signal(db, run) + db.flush() + return step +# #endregion ScenarioExecution.Lifecycle.Timeout + # #endregion ScenarioExecution.Lifecycle diff --git a/backend/src/services/dashboard_testing/execution/live_adapter.py b/backend/src/services/dashboard_testing/execution/live_adapter.py new file mode 100644 index 000000000..3ab2ba081 --- /dev/null +++ b/backend/src/services/dashboard_testing/execution/live_adapter.py @@ -0,0 +1,92 @@ +# #region ScenarioExecution.LiveAdapter [C:4] [TYPE Module] [SEMANTICS scenario,execution,executor,adapter,status] +# @defgroup ScenarioExecution Typed boundary for live browser, Superset, and ScreenshotService adapters. +# @BRIEF Map only runtime-validated external adapter results into ScenarioStep outcomes. +# @INVARIANT Runtime adapter status is exactly passed, failed, or inconclusive; untrusted values fail closed. +# @REJECTED Trusting Literal annotations at the external callable boundary was rejected — Python does +# not enforce them at runtime, so an arbitrary status could corrupt aggregation. +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal, Protocol + +_LIVE_ADAPTER_STATUSES = frozenset({"passed", "failed", "inconclusive"}) + + +# #region ScenarioExecution.LiveAdapter.Result [C:2] [TYPE Class] [SEMANTICS scenario,execution,executor,adapter,result] +# @BRIEF Typed response emitted by a permitted live-I/O adapter. +@dataclass(frozen=True) +class LiveAdapterResult: + status: Literal["passed", "failed", "inconclusive"] + reason_code: str + details: dict[str, Any] = field(default_factory=dict) + output_refs: list[str] = field(default_factory=list) + artifact_refs: list[str] = field(default_factory=list) + artifact_digests: dict[str, str] = field(default_factory=dict) + + +class LiveExecutionAdapter(Protocol): + """[EXT] Existing browser, 037, or ScreenshotService transport selected by composition.""" + + def __call__( + self, + step: dict[str, Any], + completed: dict[str, dict[str, Any]], + ) -> LiveAdapterResult: ... + + +BrowserAdapterResult = SupersetAdapterResult = ScreenshotAdapterResult = LiveAdapterResult +BrowserExecutionAdapter = SupersetExecutionAdapter = ScreenshotExecutionAdapter = LiveExecutionAdapter +# #endregion ScenarioExecution.LiveAdapter.Result + + +# #region ScenarioExecution.LiveAdapter.Dispatch [C:3] [TYPE Function] [SEMANTICS scenario,execution,executor,adapter,status] +# @BRIEF Invoke a configured adapter and map its typed response without synthetic PASS. +# @INVARIANT A passed outcome can only originate from LiveAdapterResult(status="passed"). +def dispatch_live_adapter( + outcome_factory, + tool: str, + step: dict[str, Any], + completed: dict[str, dict[str, Any]], + adapter: LiveExecutionAdapter | None, + context: dict[str, Any], + *, + unavailable_code: str, + timeout_code: str, + error_code: str, + invalid_code: str, + required_pass_detail_keys: tuple[str, ...] = (), + require_pass_artifact_refs: bool = False, + missing_evidence_code: str | None = None, +) -> dict[str, Any]: + if adapter is None: + return outcome_factory(tool, "inconclusive", reason=unavailable_code, extra=context) + try: + result = adapter(step, completed) + except TimeoutError: + return outcome_factory(tool, "inconclusive", reason=timeout_code, extra=context) + except Exception: + return outcome_factory(tool, "inconclusive", reason=error_code, extra=context) + if not isinstance(result, LiveAdapterResult) or result.status not in _LIVE_ADAPTER_STATUSES: + return outcome_factory(tool, "inconclusive", reason=invalid_code, extra=context) + if result.status == "passed" and ( + (require_pass_artifact_refs and not result.artifact_refs) + or any(not result.details.get(key) for key in required_pass_detail_keys) + ): + return outcome_factory( + tool, "inconclusive", reason=missing_evidence_code or invalid_code, extra=context + ) + details = { + **result.details, + **({"artifact_digests": result.artifact_digests} if result.artifact_digests else {}), + **context, + } + return outcome_factory( + tool, + result.status, + reason=result.reason_code, + extra=details, + output_refs=result.output_refs, + artifact_refs=result.artifact_refs, + ) +# #endregion ScenarioExecution.LiveAdapter.Dispatch +# #endregion ScenarioExecution.LiveAdapter diff --git a/backend/src/services/dashboard_testing/execution/live_binding.py b/backend/src/services/dashboard_testing/execution/live_binding.py new file mode 100644 index 000000000..a19f0e8c3 --- /dev/null +++ b/backend/src/services/dashboard_testing/execution/live_binding.py @@ -0,0 +1,260 @@ +# #region ScenarioExecution.LiveBinding [C:5] [TYPE Module] [SEMANTICS scenario,execution,live-binding,superset,composition,authority] +# @defgroup ScenarioExecution Persisted identity and runtime resolver boundary for lawful live execution. +# @BRIEF Bind a ScenarioRun to immutable authorization fingerprints without persisting credentials or sessions. +# @RELATION CALLS -> [BaselineEngine.QueryExecutor.ExecuteQueryEnvelope] +# @RELATION DEPENDS_ON -> [Services.AgentRuns.Artifacts] +# @INVARIANT A live query runs only when the resolver returns the exact persisted binding and its pinned model/principal fingerprints. +# @INVARIANT Persisted binding snapshots contain identity/fingerprints only; no client, secret, raw browser context, or callable principal is serialised. +# @RATIONALE Environment identifiers are routing labels, not execution authority. The composition root must resolve a durable, immutable identity to authorized local clients. +# @REJECTED Constructing a Superset client, browser context, or principal from environment_id/metadata was rejected — it bypasses pinned RLS, release, and model checks. +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from hashlib import sha256 +from typing import Any, Protocol + +from src.schemas.dashboard_testing import DashboardQueryModel, ExecuteQueryRequest, NormalizedFilterContext, ValueKind +from src.services.dashboard_testing.query_executor import execute_dashboard_query_envelope + +from .artifacts import is_valid_sha256 +from .live_adapter import LiveAdapterResult + +_SNAPSHOT_FIELDS = frozenset({ + "binding_ref", "environment_id", "dashboard_release_id", "release_fingerprint", "dashboard_id", + "query_model_fingerprint", "execution_principal_fingerprint", "rls_security_fingerprint", + "browser_safe_checkpoint_ref", "browser_action_binding_ref", "evidence_owner_type", "evidence_ref_policy", +}) + + +# #region ScenarioExecution.LiveBinding.Identity [C:3] [TYPE Class] [SEMANTICS scenario,execution,live-binding,identity,persistence] +# @BRIEF Immutable, JSON-safe binding reference and authorization fingerprints stored on a ScenarioRun. +# @DATA_CONTRACT LiveExecutionBinding.snapshot() <-> ScenarioRun.live_execution_binding_snapshot. +# @INVARIANT The snapshot has the exact allowlisted identity fields and cannot serialize runtime capabilities. +@dataclass(frozen=True) +class LiveExecutionBinding: + binding_ref: str + environment_id: str + dashboard_release_id: str + release_fingerprint: str + dashboard_id: int + query_model_fingerprint: str + execution_principal_fingerprint: str + rls_security_fingerprint: str + browser_safe_checkpoint_ref: str | None + browser_action_binding_ref: str | None + evidence_owner_type: str + evidence_ref_policy: str + + @classmethod + def from_snapshot(cls, snapshot: object) -> LiveExecutionBinding: + if not isinstance(snapshot, dict) or set(snapshot) != _SNAPSHOT_FIELDS: + raise ValueError("LIVE_BINDING_SNAPSHOT_INVALID") + binding = cls(**snapshot) + if ( + not all(isinstance(value, str) and value for value in ( + binding.binding_ref, binding.environment_id, binding.dashboard_release_id, + binding.release_fingerprint, binding.query_model_fingerprint, + binding.execution_principal_fingerprint, binding.rls_security_fingerprint, + )) + or not isinstance(binding.dashboard_id, int) + or binding.dashboard_id <= 0 + or binding.evidence_owner_type != "scenario_run" + or binding.evidence_ref_policy != "draft_storage_raw_response" + or any(value is not None and not isinstance(value, str) for value in ( + binding.browser_safe_checkpoint_ref, binding.browser_action_binding_ref, + )) + ): + raise ValueError("LIVE_BINDING_SNAPSHOT_INVALID") + return binding + + def snapshot(self) -> dict[str, Any]: + return { + "binding_ref": self.binding_ref, + "environment_id": self.environment_id, + "dashboard_release_id": self.dashboard_release_id, + "release_fingerprint": self.release_fingerprint, + "dashboard_id": self.dashboard_id, + "query_model_fingerprint": self.query_model_fingerprint, + "execution_principal_fingerprint": self.execution_principal_fingerprint, + "rls_security_fingerprint": self.rls_security_fingerprint, + "browser_safe_checkpoint_ref": self.browser_safe_checkpoint_ref, + "browser_action_binding_ref": self.browser_action_binding_ref, + "evidence_owner_type": self.evidence_owner_type, + "evidence_ref_policy": self.evidence_ref_policy, + } + + def with_query_model_fingerprint(self, fingerprint: str) -> LiveExecutionBinding: + return LiveExecutionBinding(**{**self.snapshot(), "query_model_fingerprint": fingerprint}) +# #endregion ScenarioExecution.LiveBinding.Identity + + +# #region ScenarioExecution.LiveBinding.Runtime [C:4] [TYPE Class] [SEMANTICS scenario,execution,live-binding,resolver,runtime] +# @BRIEF Runtime-only clients and approved model returned by a composition root for one binding reference. +# @DATA_CONTRACT LiveExecutionBindingResolver(binding_ref) -> ResolvedLiveExecutionBinding | None. +# @INVARIANT Resolver-owned clients, storage, and event-loop access never enter the persisted snapshot. +class EvidenceStorage(Protocol): + def store(self, run_id: str, sha256: str, data: bytes) -> str: ... + + +@dataclass(frozen=True) +class ResolvedLiveExecutionBinding: + binding: LiveExecutionBinding + superset_client: Any + query_model: DashboardQueryModel + evidence_storage: EvidenceStorage + run_async: Callable[[Awaitable[Any]], Any] + + +class LiveExecutionBindingResolver(Protocol): + """Composition-owned mapping from persisted identity to authorized runtime dependencies.""" + + def resolve(self, binding_ref: str) -> ResolvedLiveExecutionBinding | None: ... +# #endregion ScenarioExecution.LiveBinding.Runtime + + +# #region ScenarioExecution.LiveBinding.Payload [C:3] [TYPE Function] [SEMANTICS scenario,execution,live-binding,payload,validation] +# @BRIEF Read and validate the run-persisted identity before resolver or provider access. +# @INVARIANT Tool-specific result codes distinguish missing configuration from a malformed or +# mismatched persisted identity; neither path reaches live I/O. +def binding_from_step( + step: dict[str, Any], *, tool_code: str = "SUPERSET" +) -> tuple[LiveExecutionBinding | None, str | None]: + """Return only an exact persisted binding; never infer authority from step metadata.""" + binding_ref = step.get("live_execution_binding_ref") + snapshot = step.get("live_execution_binding_snapshot") + if binding_ref is None and snapshot is None: + return None, f"{tool_code}_BINDING_MISSING" + try: + binding = LiveExecutionBinding.from_snapshot(snapshot) + except ValueError: + return None, f"{tool_code}_BINDING_INVALID" + target = step.get("target_snapshot") + if ( + binding_ref != binding.binding_ref + or not isinstance(target, dict) + or target.get("environment_id") != binding.environment_id + or target.get("dashboard_release_id") != binding.dashboard_release_id + or step.get("execution_principal_fingerprint") != binding.execution_principal_fingerprint + ): + return None, f"{tool_code}_BINDING_MISMATCH" + return binding, None +# #endregion ScenarioExecution.LiveBinding.Payload + + +# #region ScenarioExecution.LiveBinding.ResolverMatch [C:3] [TYPE Function] [SEMANTICS scenario,execution,live-binding,resolver,validation] +# @BRIEF Reject a resolver response unless its identity and authoritative model exactly match the stored binding. +def _runtime_matches(binding: LiveExecutionBinding, resolved: ResolvedLiveExecutionBinding) -> bool: + return ( + resolved.binding == binding + and resolved.query_model.environment_id == binding.environment_id + and resolved.query_model.dashboard_id == binding.dashboard_id + and resolved.query_model.query_model_fingerprint == binding.query_model_fingerprint + ) +# #endregion ScenarioExecution.LiveBinding.ResolverMatch + + +# #region ScenarioExecution.LiveBinding.Request [C:3] [TYPE Function] [SEMANTICS scenario,execution,live-binding,request,validation] +# @BRIEF Build one 037 request only from the persisted binding and immutable plan metadata. +def _request_from_step( + step: dict[str, Any], binding: LiveExecutionBinding +) -> ExecuteQueryRequest | None: + metadata = step.get("step_meta") if isinstance(step.get("step_meta"), dict) else {} + if ( + metadata.get("environment_id") != binding.environment_id + or metadata.get("dashboard_id") != binding.dashboard_id + or not isinstance(step.get("scenario_run_id"), str) + ): + return None + try: + return ExecuteQueryRequest( + environment_id=binding.environment_id, + dashboard_id=binding.dashboard_id, + chart_id=metadata.get("chart_id"), + dataset_id=metadata.get("dataset_id"), + result_key=str(metadata["result_key"]), + normalized_filters=NormalizedFilterContext.model_validate(metadata["normalized_filters"]), + query_model_fingerprint=binding.query_model_fingerprint, + ) + except (KeyError, TypeError, ValueError): + return None +# #endregion ScenarioExecution.LiveBinding.Request + + +# #region ScenarioExecution.LiveBinding.Evidence [C:4] [TYPE Function] [SEMANTICS scenario,execution,live-binding,evidence,integrity] +# @BRIEF Register exact raw query bytes under the bound opaque evidence policy. +# @POST A passed result carries the exact verified SHA-256 and opaque stored ref; invalid bytes/digest/ref are inconclusive. +# @SIDE_EFFECT Stores verified raw response bytes through the resolver-owned evidence storage boundary. +def _evidence_result( + step: dict[str, Any], resolved: ResolvedLiveExecutionBinding, envelope: Any +) -> LiveAdapterResult: + digest = envelope.source_response_hash + raw_bytes = envelope.raw_response_content + if not is_valid_sha256(digest) or sha256(raw_bytes).hexdigest() != digest: + return LiveAdapterResult(status="inconclusive", reason_code="SUPERSET_EVIDENCE_INVALID") + run_id = step["scenario_run_id"] + content_ref = resolved.evidence_storage.store(run_id, digest, raw_bytes) + expected_ref = f"draft:{run_id}:{digest}" + if content_ref != expected_ref: + return LiveAdapterResult(status="inconclusive", reason_code="SUPERSET_EVIDENCE_REF_INVALID") + return LiveAdapterResult( + status="passed", + reason_code="SUPERSET_QUERY_EXECUTED", + details={ + "actual": envelope.normalized_value.model_dump(mode="json"), + "source_response_hash": digest, + "sha256": digest, + }, + output_refs=[content_ref], + artifact_refs=[content_ref], + artifact_digests={content_ref: digest}, + ) +# #endregion ScenarioExecution.LiveBinding.Evidence + + +# #region ScenarioExecution.LiveBinding.Execute [C:4] [TYPE Function] [SEMANTICS scenario,execution,live-binding,superset,execute] +# @BRIEF Resolve exact binding and execute its immutable 037 query request through local dependencies. +# @PRE The composition root provides an authorized resolver; the step carries a valid persisted identity snapshot. +# @POST Missing or mismatched binding is typed inconclusive before external I/O; 037 failure/timeout remains non-pass. +# @SIDE_EFFECT Calls the existing 037 envelope and, only after byte/digest validation, stores opaque evidence. +# @INVARIANT No client is constructed from run metadata, environment labels, or principal fingerprints. +def _execute_bound_superset( + resolver: LiveExecutionBindingResolver, + step: dict[str, Any], + _completed: dict[str, dict[str, Any]], +) -> LiveAdapterResult: + binding, error = binding_from_step(step) + if error is not None or binding is None: + return LiveAdapterResult(status="inconclusive", reason_code=error or "SUPERSET_BINDING_INVALID") + resolved = resolver.resolve(binding.binding_ref) + if resolved is None: + return LiveAdapterResult(status="inconclusive", reason_code="SUPERSET_BINDING_UNAVAILABLE") + if not _runtime_matches(binding, resolved): + return LiveAdapterResult(status="inconclusive", reason_code="SUPERSET_BINDING_MISMATCH") + request = _request_from_step(step, binding) + if request is None: + return LiveAdapterResult(status="inconclusive", reason_code="SUPERSET_BINDING_REQUEST_REJECTED") + try: + envelope = resolved.run_async(execute_dashboard_query_envelope( + resolved.superset_client, request, resolved.query_model, + )) + except TimeoutError: + raise + except Exception: + return LiveAdapterResult(status="inconclusive", reason_code="SUPERSET_QUERY_EXECUTION_ERROR") + if envelope.normalized_value.kind == ValueKind.UNKNOWN: + return LiveAdapterResult(status="failed", reason_code="SUPERSET_QUERY_FAILED") + return _evidence_result(step, resolved, envelope) +# #endregion ScenarioExecution.LiveBinding.Execute + + +# #region ScenarioExecution.LiveBinding.SupersetAdapter [C:5] [TYPE Function] [SEMANTICS scenario,execution,live-binding,superset,adapter,evidence] +# @BRIEF Build a typed Superset adapter that invokes 037 only after exact resolver validation. +# @POST Returns passed only after the existing envelope yields verified raw bytes, digest, and opaque evidence ref. +# @INVARIANT Missing/mismatched binding returns typed inconclusive and never calls the Superset client. +# @REJECTED Trusting a resolver's reference match without comparing the full immutable snapshot was rejected — a stale reference could otherwise cross environments or RLS scopes. +def superset_adapter_from(resolver: LiveExecutionBindingResolver): + return lambda step, completed: _execute_bound_superset(resolver, step, completed) +# #endregion ScenarioExecution.LiveBinding.SupersetAdapter + +# #endregion ScenarioExecution.LiveBinding diff --git a/backend/src/services/dashboard_testing/execution/live_composition.py b/backend/src/services/dashboard_testing/execution/live_composition.py new file mode 100644 index 000000000..9bd94172d --- /dev/null +++ b/backend/src/services/dashboard_testing/execution/live_composition.py @@ -0,0 +1,252 @@ +# #region ScenarioExecution.LiveCompositionRoot [C:5] [TYPE Module] [SEMANTICS scenario,execution,composition,live,browser,superset,screenshot] +# @BRIEF Own the application-only registration of authorized live providers for persisted 044 bindings. +# @RELATION CALLS -> [ScenarioExecution.LiveBinding.SupersetAdapter] +# @RELATION DEPENDS_ON -> [BaselineEngine.QueryExecutor.ExecuteQueryEnvelope] +# @RELATION DEPENDS_ON -> [Plugin.Service.ScreenshotService] +# @INVARIANT No runtime client, secret, cookie, callable principal, or browser context is persisted; +# providers are registered only in the application process against an exact binding snapshot. +# @INVARIANT Unavailable and mismatched providers are typed non-pass before I/O. A provider may return +# PASS only with the evidence validation enforced by its typed executor. +# @RATIONALE Startup composition is the authority boundary: environment IDs are labels and cannot safely +# select a client, RLS scope, browser session, or capture store. +# @REJECTED Constructing clients from ScenarioRun metadata or using ScreenshotService output paths as +# durable evidence was rejected — both bypass pinned identity and provenance checks. +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +from src.core.superset_client import SupersetClient +from src.schemas.dashboard_testing import DashboardQueryModel +from src.services.agent_runs.artifacts import get_draft_storage + +from .live_adapter import LiveAdapterResult +from .live_binding import ( + LiveExecutionBinding, + ResolvedLiveExecutionBinding, + binding_from_step, +) + + +@dataclass(frozen=True) +class LiveProviderContext: + """Runtime-only data passed to a registered browser or capture provider.""" + + binding: LiveExecutionBinding + step: dict[str, Any] + completed: dict[str, dict[str, Any]] + cancellation_capability: Callable[[], None] | None = None + + +LiveProvider = Callable[[LiveProviderContext], LiveAdapterResult] +CancellationCapability = Callable[[], None] +_RegisteredProvider = tuple[LiveExecutionBinding, LiveProvider, CancellationCapability | None] + + +# #region ScenarioExecution.LiveCompositionRoot.Runtime [C:4] [TYPE Class] [SEMANTICS scenario,execution,composition,provider,resolver] +# @BRIEF In-memory composition registry initialized by application startup, never by a request. +# @DATA_CONTRACT LiveExecutionBinding + registered provider -> typed adapter result. +# @INVARIANT Every registration is keyed by binding_ref and retains the full immutable binding for +# equality comparison before the provider/client receives a step. +class LiveExecutionCompositionRoot: + def __init__(self) -> None: + self._superset: dict[str, ResolvedLiveExecutionBinding] = {} + self._superset_unavailable: set[str] = set() + self._browser: dict[str, _RegisteredProvider] = {} + self._browser_unavailable: set[str] = set() + self._screenshot: dict[str, _RegisteredProvider] = {} + self._screenshot_unavailable: set[str] = set() + + @property + def has_superset_provider(self) -> bool: + return bool(self._superset or self._superset_unavailable) + + @property + def has_browser_provider(self) -> bool: + return bool(self._browser or self._browser_unavailable) + + @property + def has_screenshot_provider(self) -> bool: + return bool(self._screenshot or self._screenshot_unavailable) + + def register_superset(self, resolved: ResolvedLiveExecutionBinding) -> None: + """Register an already-authorized 037 client/model/evidence tuple at startup.""" + binding = resolved.binding + if not binding.binding_ref or not _resolved_binding_matches(binding, resolved): + raise ValueError("LIVE_COMPOSITION_BINDING_INVALID") + self._superset[binding.binding_ref] = resolved + self._superset_unavailable.discard(binding.binding_ref) + + def mark_superset_unavailable(self, binding_ref: str) -> None: + """Retain an explicitly configured-but-unavailable binding as a typed non-I/O outcome.""" + if binding_ref: + self._superset_unavailable.add(binding_ref) + + def register_browser( + self, + binding: LiveExecutionBinding, + provider: LiveProvider, + *, + cancellation_capability: CancellationCapability | None = None, + ) -> None: + """Register a browser-safe action provider; it must own safe reconstruction itself.""" + if not binding.browser_action_binding_ref or not callable(provider): + raise ValueError("LIVE_BROWSER_PROVIDER_INVALID") + self._browser[binding.binding_ref] = (binding, provider, cancellation_capability) + self._browser_unavailable.discard(binding.binding_ref) + + def mark_browser_unavailable(self, binding_ref: str) -> None: + if binding_ref: + self._browser_unavailable.add(binding_ref) + + def register_screenshot( + self, + binding: LiveExecutionBinding, + provider: LiveProvider, + *, + cancellation_capability: CancellationCapability | None = None, + ) -> None: + """Register a ScreenshotService-to-durable-evidence provider, never bare capture bytes.""" + if not callable(provider): + raise ValueError("LIVE_SCREENSHOT_PROVIDER_INVALID") + self._screenshot[binding.binding_ref] = (binding, provider, cancellation_capability) + self._screenshot_unavailable.discard(binding.binding_ref) + + def mark_screenshot_unavailable(self, binding_ref: str) -> None: + if binding_ref: + self._screenshot_unavailable.add(binding_ref) + + def resolve(self, binding_ref: str) -> ResolvedLiveExecutionBinding | None: + """Resolve only a registered 037 provider; callers still compare the full snapshot.""" + return self._superset.get(binding_ref) + + def browser_adapter(self): + return self._adapter_for("BROWSER", self._browser, require_browser_checkpoint=True) + + def screenshot_adapter(self): + return self._adapter_for("SCREENSHOT", self._screenshot, require_browser_checkpoint=False) + + def _adapter_for( + self, + tool_code: str, + providers: dict[str, _RegisteredProvider], + *, + require_browser_checkpoint: bool, + ): + def invoke(step: dict[str, Any], completed: dict[str, dict[str, Any]]) -> LiveAdapterResult: + binding, error = binding_from_step(step, tool_code=tool_code) + if binding is None: + return LiveAdapterResult(status="inconclusive", reason_code=error or f"{tool_code}_BINDING_INVALID") + registered = providers.get(binding.binding_ref) + if registered is None: + return LiveAdapterResult(status="inconclusive", reason_code=f"{tool_code}_BINDING_UNAVAILABLE") + registered_binding, provider, cancellation_capability = registered + if registered_binding != binding: + return LiveAdapterResult(status="inconclusive", reason_code=f"{tool_code}_BINDING_MISMATCH") + step_meta = step.get("step_meta") if isinstance(step.get("step_meta"), dict) else {} + if require_browser_checkpoint and step_meta.get("recovery_mode") and ( + not binding.browser_safe_checkpoint_ref + or step_meta.get("browser_safe_checkpoint_ref") != binding.browser_safe_checkpoint_ref + ): + return LiveAdapterResult(status="inconclusive", reason_code="BROWSER_SAFE_CHECKPOINT_REQUIRED") + if require_browser_checkpoint and step_meta.get("mutation_contract") and _is_prod(step): + return LiveAdapterResult(status="failed", reason_code="BROWSER_MUTATION_PROD_REJECTED") + if require_browser_checkpoint and step_meta.get("mutation_contract") and not step_meta.get( + "action_registry_fingerprint" + ): + return LiveAdapterResult(status="inconclusive", reason_code="BROWSER_MUTATION_CONTRACT_REQUIRED") + return provider(LiveProviderContext( + binding=binding, + step=step, + completed=completed, + cancellation_capability=cancellation_capability, + )) + + return invoke +# #endregion ScenarioExecution.LiveCompositionRoot.Runtime + + +# #region ScenarioExecution.LiveCompositionRoot.Bootstrap [C:5] [TYPE Function] [SEMANTICS scenario,execution,composition,startup,superset,config] +# @BRIEF Populate a root from trusted server configuration, never from a run or HTTP request. +# @RELATION DEPENDS_ON -> [Core.ConfigModels.ScenarioLiveExecutionBinding] +# @PRE config_manager supplies deployment-owned environments and typed live binding records. +# @POST Every enabled valid record registers the existing SupersetClient/037 model/evidence tuple; +# invalid or unavailable records remain typed unavailable without constructing authority from IDs. +# @SIDE_EFFECT Creates process-local SupersetClient instances using encrypted deployment environment credentials. +# @INVARIANT The bootstrap parses every binding and query model before client construction. A missing +# environment, malformed snapshot, or unavailable storage is recorded as unavailable and +# cannot execute I/O. Runtime credentials never enter ScenarioRun. +# @INVARIANT This bootstrap registers only the exact 037 client/model/evidence tuple. Browser-safe +# and Screenshot durable-evidence providers remain explicitly unavailable until a +# separate server-owned registration supplies them. +# @REJECTED Inspecting a fresh mutable query model at dispatch was rejected — the configured model +# must exactly match the persisted binding fingerprint, preserving the run's pinned authority. +def bootstrap_live_execution_composition( + root: LiveExecutionCompositionRoot, + *, + config_manager: Any, + run_async: Callable[[Any], Any], +) -> int: + configured = getattr(config_manager.get_config().settings, "scenario_live_execution_bindings", []) + registered = 0 + for record in configured: + binding_snapshot = getattr(record, "binding_snapshot", None) + if isinstance(record, dict): + binding_snapshot = record.get("binding_snapshot") + binding_ref = binding_snapshot.get("binding_ref") if isinstance(binding_snapshot, dict) else None + enabled = getattr(record, "enabled", False) + if isinstance(record, dict): + enabled = bool(record.get("enabled", False)) + if not enabled: + continue + try: + binding = LiveExecutionBinding.from_snapshot(binding_snapshot) + # Existing 038 ScreenshotService has no principal/RLS-bound durable-evidence adapter, + # and no browser-safe provider is installed by this deployment bootstrap. Preserve + # those configured capabilities as explicit typed-unavailable bindings until a + # server-owned registration calls register_browser/register_screenshot. + root.mark_browser_unavailable(binding.binding_ref) + root.mark_screenshot_unavailable(binding.binding_ref) + query_snapshot = getattr(record, "query_model_snapshot", None) + if isinstance(record, dict): + query_snapshot = record.get("query_model_snapshot") + query_model = DashboardQueryModel.model_validate(query_snapshot) + environment = config_manager.get_environment(binding.environment_id) + if environment is None: + raise ValueError("LIVE_COMPOSITION_ENVIRONMENT_UNAVAILABLE") + root.register_superset(ResolvedLiveExecutionBinding( + binding=binding, + superset_client=SupersetClient(environment), + query_model=query_model, + evidence_storage=get_draft_storage(), + run_async=run_async, + )) + registered += 1 + except Exception: + if isinstance(binding_ref, str) and binding_ref: + root.mark_superset_unavailable(binding_ref) + return registered +# #endregion ScenarioExecution.LiveCompositionRoot.Bootstrap + + +# #region ScenarioExecution.LiveCompositionRoot.Match [C:2] [TYPE Function] [SEMANTICS scenario,execution,composition,identity] +# @BRIEF Confirm that a registered 037 tuple preserves the immutable binding identity. +def _resolved_binding_matches( + binding: LiveExecutionBinding, resolved: ResolvedLiveExecutionBinding +) -> bool: + return ( + resolved.binding == binding + and resolved.query_model.environment_id == binding.environment_id + and resolved.query_model.dashboard_id == binding.dashboard_id + and resolved.query_model.query_model_fingerprint == binding.query_model_fingerprint + ) + + +def _is_prod(step: dict[str, Any]) -> bool: + target = step.get("target_snapshot") if isinstance(step.get("target_snapshot"), dict) else {} + metadata = step.get("step_meta") if isinstance(step.get("step_meta"), dict) else {} + return target.get("environment_class") == "PROD" or metadata.get("environment_class") == "PROD" +# #endregion ScenarioExecution.LiveCompositionRoot.Match + +# #endregion ScenarioExecution.LiveCompositionRoot diff --git a/backend/src/services/dashboard_testing/execution/result.py b/backend/src/services/dashboard_testing/execution/result.py index 060dc60be..8ff1875a1 100644 --- a/backend/src/services/dashboard_testing/execution/result.py +++ b/backend/src/services/dashboard_testing/execution/result.py @@ -5,6 +5,10 @@ # @INVARIANT The execution snapshot is immutable: it deep-copies run/step state at build time, # so later mutations of the run row never change an already-published snapshot. # @INVARIANT provenance carries the pinned revision identity, never a runtime-derived value. +# @INVARIANT Lifecycle-held states (pending approval, waiting_human, cancellation) report their +# persisted run status, never an aggregate synthetic PASS from skipped/cancelled/waiting rows. +# @INVARIANT A persisted terminal blocked/failed/inconclusive status remains non-pass even when a +# fail-closed preflight produced no materialized step row. from __future__ import annotations import copy @@ -113,6 +117,14 @@ def build_execution_snapshot(run: ScenarioRun, steps: list[ScenarioStepRun]) -> # step_counts from aggregate_result. def build_result(run: ScenarioRun, steps: list[ScenarioStepRun]) -> dict[str, Any]: aggregate = aggregate_result([{"status": step.status} for step in steps]) + status = ( + run.status + if run.status in { + "pending_approval", "waiting_human", "cancel_requested", "cancelled", + "passed", "failed", "blocked", "inconclusive", + } + else aggregate["status"] + ) failures = [ { "logical_step_id": step.logical_step_id, @@ -125,7 +137,7 @@ def build_result(run: ScenarioRun, steps: list[ScenarioStepRun]) -> dict[str, An ] return { "run_id": run.id, - "status": aggregate["status"], + "status": status, "step_counts": { "passed": aggregate["passed"], "failed": aggregate["failed"], diff --git a/backend/src/services/dashboard_testing/execution/runner.py b/backend/src/services/dashboard_testing/execution/runner.py index c69afa1e5..8beddecdf 100644 --- a/backend/src/services/dashboard_testing/execution/runner.py +++ b/backend/src/services/dashboard_testing/execution/runner.py @@ -3,44 +3,134 @@ # @BRIEF Create queued/pending-approval ScenarioRun records with deterministic idempotency. # @RELATION CALLS -> [ScenarioExecution.RunnerPlan.Derive] # @RELATION CALLS -> [ScenarioExecution.Approval.CreateGate] +# @RELATION CALLS -> [ScenarioExecution.LiveBinding.SupersetAdapter] +# @RELATION CALLS -> [ScenarioAnalytics.Investigation.TerminalSignal] # @RELATION DEPENDS_ON -> [Models.ScenarioExecution.Run] # @INVARIANT Same idempotency key and request hash returns the same run; changed request rejects. # @INVARIANT PROD starts are pinned to a durable ActionApprovalGate; pending_approval -> queued # only through gate approval (036), never by silent dispatch. # @INVARIANT scenario_content_hash mirrors the pinned revision's content hash; request_hash # carries the idempotency fingerprint (provenance vs replay identity). +# @INVARIANT A revision with a human step is manual_run_only: only the trusted manual +# interactive entry point may create its run. Automation is refused before a +# ScenarioRun, PROD gate, notification, or queue side effect exists. from __future__ import annotations +import copy +from datetime import UTC, datetime import hashlib import json from typing import Any import uuid +from sqlalchemy import delete, update from sqlalchemy.orm import Session from src.core.logger import logger from src.models.scenario_registry import ScenarioRegistryEntry -from src.models.scenario_run import ScenarioRun +from src.models.scenario_run import ScenarioRun, ScenarioStepRun +from src.models.scenario_worker import ScenarioStepLease +from src.services.dashboard_testing.analytics.investigation import emit_terminal_run_signal +from src.services.dashboard_testing.automation.notify import persist_notification from .approval import create_prod_gate -from .runner_plan import derive_runner_plan +from .artifacts import invalidate_step_evidence, register_step_evidence +from .dispatch import dispatch_step +from .environment_policy import resolve_environment_execution_policy +from .executor_registry import ScenarioExecutorRegistry +from .executors import _register_default_executors +from .lifecycle import suspend_for_human +from .live_binding import LiveExecutionBinding, LiveExecutionBindingResolver, superset_adapter_from +from .result import build_result +from .runner_plan import derive_runner_plan, validate_pinned_runner_plan +from .worker import claim_step + +# #region ScenarioExecution.Runner.TriggerSource [C:3] [TYPE Block] [SEMANTICS scenario,execution,start,trigger,automation,human] +# @BRIEF Enumerate server-owned run origins accepted by the execution boundary. +# @INVARIANT trigger_source is selected by a trusted entry point, never by an API payload; +# unknown origins fail closed before idempotency replay or row creation. +# @RATIONALE A human graph can only be observed by an authenticated analyst through the +# manual interactive route. 046 sources must carry their actual origin into the +# start boundary rather than mutating a row after it has been created. +# @REJECTED Inferring manual authority from actor text or a caller-supplied boolean was rejected: +# either could turn an automation request into an automatic HumanCheckpoint bypass. +TRIGGER_SOURCE_MANUAL = "manual" +TRIGGER_SOURCE_SCHEDULED = "scheduled" +TRIGGER_SOURCE_DEPLOY = "deploy_to_preprod" +TRIGGER_SOURCE_RELEASE = "release_created" +TRIGGER_SOURCE_ETL = "etl_completed" +TRIGGER_SOURCE_API = "api" +_AUTOMATION_TRIGGER_SOURCES = frozenset({ + TRIGGER_SOURCE_SCHEDULED, + TRIGGER_SOURCE_DEPLOY, + TRIGGER_SOURCE_RELEASE, + TRIGGER_SOURCE_ETL, + TRIGGER_SOURCE_API, +}) +_TRUSTED_TRIGGER_SOURCES = _AUTOMATION_TRIGGER_SOURCES | {TRIGGER_SOURCE_MANUAL} -# #region ScenarioExecution.Runner.ActionGate [C:3] [TYPE Function] [SEMANTICS scenario,execution,approval,gate,prod] -def require_action_gate(*, is_prod: bool, approval_granted: bool) -> None: - if is_prod and not approval_granted: - raise PermissionError("PROD scenario execution requires ActionApprovalGate") -# #endregion ScenarioExecution.Runner.ActionGate +# #region ScenarioExecution.Runner.TriggerSource.RequireTrusted [C:2] [TYPE Function] [SEMANTICS scenario,execution,trigger,source,authority] +# @BRIEF Accept only a server-owned run origin before idempotency lookup or row creation. +# @INVARIANT Unknown origins fail closed and cannot be converted into manual authority. +def _require_trusted_trigger_source(trigger_source: str) -> str: + if trigger_source not in _TRUSTED_TRIGGER_SOURCES: + raise ValueError("UNTRUSTED_TRIGGER_SOURCE") + return trigger_source +# #endregion ScenarioExecution.Runner.TriggerSource.RequireTrusted -def _request_hash(scenario_id: str, revision_id: str, params: dict[str, Any], environment_id: str) -> str: - payload = json.dumps({"scenario_id": scenario_id, "revision_id": revision_id, "params": params, "environment_id": environment_id}, sort_keys=True, separators=(",", ":")) - return hashlib.sha256(payload.encode()).hexdigest() +# #region ScenarioExecution.Runner.TriggerSource.RejectAutomatedHuman [C:3] [TYPE Function] [SEMANTICS scenario,execution,trigger,human,manual-only] +# @BRIEF Reject an immutable human-containing plan from every trusted automation origin. +# @POST Rejection occurs before a ScenarioRun, approval gate, notification, queue signal, or dispatch exists. +# @INVARIANT HumanCheckpoint observation is distinct from ActionApprovalGate approval and cannot be automated. +def _reject_automated_human_plan(plan: dict[str, Any], trigger_source: str) -> None: + if plan.get("manual_run_only", bool(plan.get("human_checkpoints"))) and trigger_source != TRIGGER_SOURCE_MANUAL: + raise ValueError("AUTOMATION_INELIGIBLE_HUMAN_STEP") +# #endregion ScenarioExecution.Runner.TriggerSource.RejectAutomatedHuman +# #endregion ScenarioExecution.Runner.TriggerSource + +def _request_hash( + scenario_id: str, + revision_id: str, + params: dict[str, Any], + environment_id: str, + environment_class: str, + live_binding_snapshot: dict[str, Any] | None = None, +) -> str: + payload = { + "scenario_id": scenario_id, + "revision_id": revision_id, + "params": params, + "environment_id": environment_id, + "environment_class": environment_class, + } + if live_binding_snapshot is not None: + payload["live_binding_snapshot"] = live_binding_snapshot + return hashlib.sha256( + json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() # #region ScenarioExecution.Runner.Start [C:5] [TYPE Function] [SEMANTICS scenario,execution,start,idempotency,prod] # @ingroup ScenarioExecution # @BRIEF Create one durable run, derive its immutable plan, and gate PROD starts. +# @DATA_CONTRACT LiveExecutionBinding -> ScenarioRun.live_execution_binding_ref + live_execution_binding_snapshot. +# @RELATION CALLS -> [ScenarioExecution.EnvironmentPolicy.Resolve] +# @PRE config_manager, when supplied by an entry point, is an application-owned trusted provider; +# start_run always resolves the target itself and never accepts a caller-built policy object. +# @POST Creates or replays only a durable queued/pending-approval row; adapter dispatch is reserved +# for ScenarioExecution.Runner.QueuedDispatch after its persisted status CAS. +# @INVARIANT An optional binding snapshot contributes to request idempotency and must exactly match the start environment, release, and actor fingerprint. +# @INVARIANT An idempotency replay returns its existing row without a step, lease, notification, +# queue signal, or adapter call. A manual human plan reaches HumanCheckpoint only after +# the separate queued dispatcher wins its claim. +# @INVARIANT Human-containing plans reject all 046 automation sources before idempotency lookup, +# ScenarioRun creation, PROD gate creation, dispatch, notification, or queue projection. +# @INVARIANT Environment class is resolved only from the server-owned policy before idempotency and +# gate creation. is_prod/approval_granted compatibility arguments are never authority. +# @REJECTED Request-body is_prod was rejected as an execution authority because it could bypass or +# fabricate PROD gating; only the configured Environment policy may select a gate. def start_run( db: Session, scenario_id: str, @@ -52,13 +142,57 @@ def start_run( idempotency_key: str, is_prod: bool = False, approval_granted: bool = False, + config_manager: Any | None = None, + auto_advance: bool = False, + dashboard_release_id: str | None = None, + baseline_set: str | None = None, + execution_toggles: dict[str, bool] | None = None, + live_execution_binding: LiveExecutionBinding | dict[str, Any] | None = None, + trigger_source: str = TRIGGER_SOURCE_MANUAL, ) -> ScenarioRun: - require_action_gate(is_prod=is_prod, approval_granted=approval_granted) - request_hash = _request_hash(scenario_id, revision_id, params, environment_id) + del auto_advance, is_prod, approval_granted + trigger_source = _require_trusted_trigger_source(trigger_source) + if config_manager is None: + from src.dependencies import get_config_manager + + config_manager = get_config_manager() + environment_policy = resolve_environment_execution_policy( + environment_id, config_manager + ) + entry = db.query(ScenarioRegistryEntry).filter(ScenarioRegistryEntry.scenario_id == scenario_id).first() + if entry is None: + raise ValueError("scenario not found") + plan = derive_runner_plan(db, scenario_id, revision_id) + _reject_automated_human_plan(plan, trigger_source) + if environment_policy.is_prod and any( + bool((step.get("action_descriptor") or {}).get("mutating")) + and step.get("tool") == "browser" + for step in plan.get("steps") or [] + if isinstance(step, dict) + ): + raise ValueError("PROD_BROWSER_MUTATION_FORBIDDEN") + binding = ( + None if live_execution_binding is None + else live_execution_binding + if isinstance(live_execution_binding, LiveExecutionBinding) + else LiveExecutionBinding.from_snapshot(live_execution_binding) + ) + principal_fingerprint = hashlib.sha256(actor.encode()).hexdigest() + if binding is not None and ( + binding.environment_id != environment_id + or binding.execution_principal_fingerprint != principal_fingerprint + or (dashboard_release_id is not None and binding.dashboard_release_id != dashboard_release_id) + ): + raise ValueError("LIVE_BINDING_START_MISMATCH") + binding_snapshot = binding.snapshot() if binding is not None else None + request_hash = _request_hash( + scenario_id, revision_id, params, environment_id, environment_policy.environment_class, + binding_snapshot, + ) logger.reason( "ScenarioExecution.Runner.start_run", "Start scenario run request", - payload={"scenario_id": scenario_id, "revision_id": revision_id, "is_prod": is_prod, "idempotency_key": idempotency_key}, + payload={"scenario_id": scenario_id, "revision_id": revision_id, "environment_class": environment_policy.environment_class, "idempotency_key": idempotency_key}, ) existing = db.query(ScenarioRun).filter(ScenarioRun.idempotency_key == idempotency_key).first() if existing is not None: @@ -76,22 +210,27 @@ def start_run( payload={"run_id": existing.id, "status": existing.status}, ) return existing - entry = db.query(ScenarioRegistryEntry).filter(ScenarioRegistryEntry.scenario_id == scenario_id).first() - if entry is None: - raise ValueError("scenario not found") - plan = derive_runner_plan(db, scenario_id, revision_id) run = ScenarioRun( id=str(uuid.uuid4()), scenario_id=scenario_id, scenario_revision_id=revision_id, scenario_content_hash=plan["scenario_content_hash"], request_hash=request_hash, environment_id=environment_id, - status="pending_approval" if is_prod else "queued", phase="preflight", - parameter_bindings=params, target_snapshot={"environment_id": environment_id}, - trigger_source="manual", idempotency_key=idempotency_key, runner_plan=plan, - execution_principal_fingerprint=hashlib.sha256(actor.encode()).hexdigest(), + status="pending_approval" if environment_policy.is_prod else "queued", phase="preflight", + parameter_bindings=params, + target_snapshot={ + "environment_id": environment_id, + "environment_class": environment_policy.environment_class, + "dashboard_release_id": binding.dashboard_release_id if binding is not None else dashboard_release_id, + "baseline_set": baseline_set, + "execution_toggles": execution_toggles or {}, + }, + trigger_source=trigger_source, idempotency_key=idempotency_key, runner_plan=plan, + execution_principal_fingerprint=principal_fingerprint, + live_execution_binding_ref=binding.binding_ref if binding is not None else None, + live_execution_binding_snapshot=binding_snapshot, ) db.add(run) db.flush() - if is_prod: + if environment_policy.is_prod: create_prod_gate(db, run_id=run.id, request_hash=request_hash) logger.reflect( "ScenarioExecution.Runner.start_run", @@ -104,7 +243,773 @@ def start_run( "Run created and queued", payload={"run_id": run.id, "status": run.status, "plan_hash": plan["plan_hash"]}, ) + # Start is persistence-only even if a legacy caller supplies auto_advance=True. + # The scheduler/worker's durable queued->running CAS is the sole dispatch authority. + db.flush() return run # #endregion ScenarioExecution.Runner.Start + +# #region ScenarioExecution.Runner.DefaultRegistry [C:2] [TYPE Function] [SEMANTICS scenario,execution,runner,registry,executor] +# @ingroup ScenarioExecution +# @BRIEF Build the bounded default executor registry used by runner continuations. +# @INVARIANT Default composition never resolves a live client from run metadata alone; unavailable +# browser/Superset/Screenshot dependencies remain typed inconclusive. A supplied resolver +# must still match the immutable ScenarioRun binding before it can invoke 037. +# @RELATION CALLS -> [ScenarioExecution.LiveBinding.SupersetAdapter] +# @RELATION CALLS -> [ScenarioExecution.LiveCompositionRoot] +# @RATIONALE ScenarioRun persists a binding identity snapshot, never a callable principal, client, +# raw browser session, or secret. The composition root owns runtime capability resolution. +# @REJECTED Constructing a Superset client or Playwright/ScreenshotService context from environment +# fields was rejected — it could execute with an unpinned identity, RLS scope, or mutable query. +def _build_default_registry( + live_binding_resolver: LiveExecutionBindingResolver | None = None, +) -> ScenarioExecutorRegistry: + composition_root = None + if live_binding_resolver is None: + from src.dependencies import get_live_execution_composition_root + + composition_root = get_live_execution_composition_root() + live_binding_resolver = ( + composition_root if composition_root.has_superset_provider else None + ) + registry = ScenarioExecutorRegistry() + _register_default_executors( + registry, + superset_adapter=( + superset_adapter_from(live_binding_resolver) + if live_binding_resolver is not None + else None + ), + browser_adapter=( + composition_root.browser_adapter() + if composition_root is not None and composition_root.has_browser_provider + else None + ), + screenshot_adapter=( + composition_root.screenshot_adapter() + if composition_root is not None and composition_root.has_screenshot_provider + else None + ), + ) + return registry +# #endregion ScenarioExecution.Runner.DefaultRegistry + + +# #region ScenarioExecution.Runner.RejectMalformedPlan [C:4] [TYPE Function] [SEMANTICS scenario,execution,plan,preflight,lease] +# @BRIEF Terminalize a queued/continuing legacy plan that lacks a valid immutable action descriptor. +# @INVARIANT Invalid plan preflight revokes any active lease and blocks active step projection before +# it can reach an executor; completed historical evidence remains untouched. +def _reject_malformed_plan(db: Session, run: ScenarioRun, error_code: str) -> dict[str, Any]: + db.execute(delete(ScenarioStepLease).where(ScenarioStepLease.run_id == run.id)) + for step in db.query(ScenarioStepRun).filter(ScenarioStepRun.run_id == run.id).all(): + if step.status not in {"passed", "failed", "blocked", "inconclusive", "skipped"}: + step.status = "blocked" + step.error_code = error_code + step.step_outcome = {"status": "blocked", "error_code": error_code} + step.progress = 100 + step.finished_at = datetime.now(UTC) + run.status = "blocked" + run.phase = "completed" + run.error_code = error_code + run.finished_at = datetime.now(UTC) + db.flush() + _record_terminal_side_effects(db, run) + db.flush() + return build_result( + run, + db.query(ScenarioStepRun).filter(ScenarioStepRun.run_id == run.id).all(), + ) +# #endregion ScenarioExecution.Runner.RejectMalformedPlan + + +# #region ScenarioExecution.Runner.CloseDispatchError [C:4] [TYPE Function] [SEMANTICS scenario,execution,dispatch,error,lease,terminal] +# @ingroup ScenarioExecution +# @BRIEF Close every active projection after a queued dispatcher exception. +# @POST The run is inconclusive with no running/queued step, unexpired lease, or active evidence refs. +# @INVARIANT Historical evidence remains durable while the current result projection is retired. +def _close_queued_dispatch_error(db: Session, run: ScenarioRun) -> dict[str, Any]: + steps = db.query(ScenarioStepRun).filter(ScenarioStepRun.run_id == run.id).all() + now = datetime.now(UTC) + for step in steps: + if step.status not in {"passed", "failed", "blocked", "inconclusive", "skipped", "cancelled"}: + step.status = "inconclusive" + step.error_code = "QUEUED_DISPATCH_ERROR" + step.step_outcome = { + "logical_step_id": step.logical_step_id, + "status": "inconclusive", + "error_code": "QUEUED_DISPATCH_ERROR", + } + step.progress = 100 + step.finished_at = now + step.artifact_refs = [] + from .lifecycle import _expire_step_leases + + invalidate_step_evidence( + db, + run_id=run.id, + logical_step_ids={step.logical_step_id for step in steps}, + ) + _expire_step_leases(db, run.id) + run.status = "inconclusive" + run.phase = "completed" + run.error_code = "QUEUED_DISPATCH_ERROR" + run.finished_at = now + db.flush() + _record_terminal_side_effects(db, run) + db.flush() + return build_result(run, steps) +# #endregion ScenarioExecution.Runner.CloseDispatchError + + +# #region ScenarioExecution.Runner.QueuedDispatch [C:5] [TYPE Function] [SEMANTICS scenario,execution,dispatch,queue,cas,scheduler] +# @BRIEF Claim and advance eligible durable queued runs outside the HTTP request lifecycle. +# @RELATION DEPENDS_ON -> [ScenarioExecution.Runner.Start] +# @RELATION CALLS -> [ScenarioExecution.Runner.Walker] +# @RELATION CALLS -> [ScenarioExecution.Runner.TerminalSignal] +# @PRE The caller is a trusted server worker/scheduler with a database session; HTTP start remains auto_advance=false. +# @POST Each selected run first CAS-transitions queued -> running; only the winning worker invokes the walker. +# @INVARIANT Pending approval, human wait, cancellation, capacity-blocked, and malformed automated +# human plans never dispatch. Repeated ticks/racing workers cannot duplicate a side effect. +# @INVARIANT The persisted queued->running compare-and-set is the sole initial-dispatch authority: +# HTTP/trigger start, ActionApprovalGate decision, and idempotency replay may persist or +# queue a run but cannot invoke an adapter. +# @REJECTED Running a queued ScenarioRun in the API handler or relying on an in-memory worker flag was +# rejected — only the persisted status CAS is shared across server workers/processes. +def dispatch_queued_runs( + db: Session, + *, + worker_id: str, + registry: ScenarioExecutorRegistry | None = None, + live_binding_resolver: LiveExecutionBindingResolver | None = None, + limit: int = 25, +) -> list[dict[str, Any]]: + if limit <= 0: + raise ValueError("dispatch limit must be positive") + candidates = ( + db.query(ScenarioRun.id) + .filter(ScenarioRun.status == "queued") + .order_by(ScenarioRun.created_at.asc(), ScenarioRun.id.asc()) + .limit(limit) + .all() + ) + outcomes: list[dict[str, Any]] = [] + selected_registry = registry or _build_default_registry(live_binding_resolver) + for (run_id,) in candidates: + run = db.query(ScenarioRun).filter(ScenarioRun.id == run_id).first() + if run is None or run.error_code == "CAPACITY_BLOCKED": + continue + plan = run.runner_plan or {} + try: + validate_pinned_runner_plan(plan) + except ValueError as exc: + outcomes.append(_reject_malformed_plan(db, run, str(exc))) + continue + if (run.target_snapshot or {}).get("environment_class") == "PROD" and any( + isinstance(step, dict) + and step.get("tool") == "browser" + and bool((step.get("action_descriptor") or {}).get("mutating")) + for step in plan.get("steps") or [] + ): + outcomes.append(_reject_malformed_plan(db, run, "PROD_BROWSER_MUTATION_FORBIDDEN")) + continue + has_human_step = any( + isinstance(step, dict) and str(step.get("tool", step.get("executor", ""))) == "human" + for step in plan.get("steps", []) + ) + if (plan.get("manual_run_only") or has_human_step) and run.trigger_source != TRIGGER_SOURCE_MANUAL: + run.status = "blocked" + run.phase = "completed" + run.error_code = "AUTOMATION_INELIGIBLE_HUMAN_STEP" + run.finished_at = datetime.now(UTC) + db.flush() + _record_terminal_side_effects(db, run) + outcomes.append(build_result(run, [])) + continue + claimed = db.execute( + update(ScenarioRun) + .where(ScenarioRun.id == run_id, ScenarioRun.status == "queued") + .values(status="running", phase="executing", started_at=run.started_at or datetime.now(UTC)) + ) + if claimed.rowcount != 1: + continue + db.flush() + db.refresh(run) + try: + outcomes.append(_advance_run(db, run, selected_registry, worker_id=worker_id)) + except Exception: + outcomes.append(_close_queued_dispatch_error(db, run)) + return outcomes +# #endregion ScenarioExecution.Runner.QueuedDispatch + + +# #region ScenarioExecution.Runner.ContinueAfterHuman [C:4] [TYPE Function] [SEMANTICS scenario,execution,human,checkpoint,resume,dag] +# @ingroup ScenarioExecution +# @BRIEF Resume the ready DAG frontier after a HumanCheckpoint decision without re-running completed steps. +# @PRE Checkpoint CAS has materialized the human ScenarioStepRun outcome and returned the run to queued. +# @POST Dispatches only missing dependent steps; terminal result reflects the persisted checkpoint outcome. +# @INVARIANT Human remains a lifecycle primitive and is never added to the executor registry. +def continue_after_human_decision( + db: Session, + run_id: str, + *, + worker_id: str, + live_binding_resolver: LiveExecutionBindingResolver | None = None, +) -> dict[str, Any]: + run = db.query(ScenarioRun).filter(ScenarioRun.id == run_id).first() + if run is None: + raise ValueError("run not found") + if run.status != "queued": + raise ValueError("run is not ready for human continuation") + return _advance_run(db, run, _build_default_registry(live_binding_resolver), worker_id=worker_id) +# #endregion ScenarioExecution.Runner.ContinueAfterHuman + + +# #region ScenarioExecution.Runner.ContinueAfterInfrastructureResume [C:4] [TYPE Function] [SEMANTICS scenario,execution,infrastructure,resume,recovery,dag] +# @ingroup ScenarioExecution +# @BRIEF Continue a recovered run from persisted step rows after one-time infrastructure resume. +# @PRE Infrastructure resume consumed a valid token and returned the run to queued; no HumanCheckpoint is pending. +# @POST Dispatches only absent frontier steps, preserving completed attempt records and their side effects. +# @REJECTED Reusing HumanCheckpoint continuation — infrastructure recovery is neither an observation +# disposition nor a PROD ActionApprovalGate decision. +def continue_after_infrastructure_resume( + db: Session, + run_id: str, + *, + worker_id: str, + live_binding_resolver: LiveExecutionBindingResolver | None = None, +) -> dict[str, Any]: + run = db.query(ScenarioRun).filter(ScenarioRun.id == run_id).first() + if run is None: + raise ValueError("run not found") + if run.status != "queued": + raise ValueError("run is not ready for infrastructure continuation") + return _advance_run(db, run, _build_default_registry(live_binding_resolver), worker_id=worker_id) +# #endregion ScenarioExecution.Runner.ContinueAfterInfrastructureResume + + +# #region ScenarioExecution.Runner.TerminalSignal [C:4] [TYPE Function] [SEMANTICS scenario,execution,terminal,investigation,signal] +# @BRIEF Project one terminal non-pass run into the canonical analyst queue without starting work. +# @RELATION CALLS -> [ScenarioAnalytics.Investigation.TerminalSignal] +# @POST Failed, blocked, and inconclusive runs project one canonical queue signal; passed runs project none. +# @SIDE_EFFECT Emits the idempotent analyst-queue signal and the existing terminal notification. +# @INVARIANT Terminal retries reuse the immutable 047 signal; they never open a case, AgentRun, chat, +# or remediation action. A new retry attempt has a new terminal context; it creates a +# new immutable signal only after re-terminalization and never mutates the prior one. +# @RATIONALE The runner is a queue producer only, preserving 047's separate analyst/case and recurrence boundaries. +# @REJECTED Calling the legacy scenario/environment queue projection was rejected — it conflates +# distinct runs and cannot carry durable artifact provenance. +def _record_terminal_side_effects(db: Session, run: ScenarioRun) -> None: + if run.status in {"failed", "blocked", "inconclusive"}: + emit_terminal_run_signal(db, run) + persist_notification( + db, + event_type="blocked" if run.status == "blocked" else "failed", + scenario_id=run.scenario_id, + run_id=run.id, + severity="warning", + payload={"status": run.status, "environment_id": run.environment_id}, + ) + elif run.status == "passed": + persist_notification( + db, + event_type="completed", + scenario_id=run.scenario_id, + run_id=run.id, + payload={"environment_id": run.environment_id}, + ) +# #endregion ScenarioExecution.Runner.TerminalSignal + + +# #region ScenarioExecution.Runner.ContinueAfterRetry [C:4] [TYPE Function] [SEMANTICS scenario,execution,retry,continuation,dag] +# @ingroup ScenarioExecution +# @BRIEF Execute the queued retry closure from its persisted frontier after lifecycle invalidation. +# @RELATION DEPENDS_ON -> [ScenarioExecution.Lifecycle.Retry] +# @PRE retry_step has atomically retired active closure evidence and returned the run to queued. +# @POST The walker executes queued attempts only; unchanged completed ancestors remain untouched. +# @INVARIANT Retry continuation cannot consume a HumanCheckpoint or a pending PROD approval gate. +def continue_after_step_retry( + db: Session, + run_id: str, + *, + worker_id: str, + live_binding_resolver: LiveExecutionBindingResolver | None = None, +) -> dict[str, Any]: + run = db.query(ScenarioRun).filter(ScenarioRun.id == run_id).first() + if run is None: + raise ValueError("run not found") + if run.status != "queued": + raise ValueError("run is not ready for retry continuation") + return _advance_run(db, run, _build_default_registry(live_binding_resolver), worker_id=worker_id) +# #endregion ScenarioExecution.Runner.ContinueAfterRetry + + +# #region ScenarioExecution.Runner.CrashRecovery.RecoverRunning [C:4] [TYPE Function] [SEMANTICS scenario,execution,recovery,crash,lease,descriptor,browser] +# @ingroup ScenarioExecution +# @BRIEF Validate and archive persisted running steps before crash recovery continues the frontier. +# @POST Returns a terminal rejection result for unsafe state, otherwise the recovered step IDs. +# @INVARIANT No running step is replayed unless its expired lease exactly matches an idempotent or +# retry-safe pinned descriptor and browser recovery has a valid safe checkpoint. +def _recover_running_steps( + db: Session, + run: ScenarioRun, + persisted_steps: list[ScenarioStepRun], + plan: dict[str, Any], + *, + worker_id: str, +) -> tuple[list[str], dict[str, Any] | None]: + plan_steps = { + str(item.get("logical_step_id", item.get("id", ""))): item + for item in plan.get("steps", []) + if isinstance(item, dict) + } + running_steps = [step for step in persisted_steps if step.status == "running"] + for step in running_steps: + lease = ( + db.query(ScenarioStepLease) + .filter( + ScenarioStepLease.run_id == run.id, + ScenarioStepLease.logical_step_id == step.logical_step_id, + ) + .with_for_update() + .first() + ) + if lease is None: + return [], _terminalize_recovery_rejection( + db, run, step, "blocked", "RECOVERY_LEASE_MISSING" + ) + expires_at = ( + lease.expires_at.replace(tzinfo=UTC) + if lease.expires_at.tzinfo is None + else lease.expires_at.astimezone(UTC) + ) + if expires_at > datetime.now(UTC): + raise ValueError("RECOVERY_LEASE_ACTIVE") + descriptor = (plan_steps.get(step.logical_step_id) or {}).get("action_descriptor") or {} + if not (bool(descriptor.get("idempotent")) or bool(descriptor.get("retry_safe"))): + return [], _terminalize_recovery_rejection( + db, run, step, "blocked", "RECOVERY_RECONCILIATION_REQUIRED" + ) + if ( + lease.idempotent != bool(descriptor.get("idempotent")) + or lease.retry_safe != bool(descriptor.get("retry_safe")) + ): + return [], _terminalize_recovery_rejection( + db, run, step, "blocked", "RECOVERY_DESCRIPTOR_LEASE_MISMATCH" + ) + step_meta = plan_steps.get(step.logical_step_id, {}) + if ( + str(step_meta.get("tool", step_meta.get("executor", "assertion"))) == "browser" + and not _browser_recovery_checkpoint(run, step_meta) + ): + return [], _terminalize_recovery_rejection( + db, run, step, "inconclusive", "BROWSER_RECOVERY_CHECKPOINT_REQUIRED" + ) + _archive_recovery_attempt(db, run, step, recovery_worker_id=worker_id) + return [step.logical_step_id for step in running_steps], None +# #endregion ScenarioExecution.Runner.CrashRecovery.RecoverRunning + + +# #region ScenarioExecution.Runner.CrashRecovery [C:5] [TYPE Function] [SEMANTICS scenario,execution,recovery,crash,lease,browser] +# @ingroup ScenarioExecution +# @BRIEF Recover one persisted run from its pinned plan after an abandoned worker claim. +# @RELATION DEPENDS_ON -> [ScenarioExecution.Worker.Claim] +# @RELATION CALLS -> [ScenarioExecution.Artifacts.InvalidateStepEvidence] +# @RELATION CALLS -> [ScenarioExecution.Runner.Walker] +# @RELATION CALLS -> [ScenarioExecution.Runner.TerminalSignal] +# @RELATION DEPENDS_ON -> [ScenarioExecution.LiveBinding.Identity] +# @PRE Recovery is server-driven by run_id; no infrastructure resume token, HumanCheckpoint, or +# ActionApprovalGate decision is accepted or consumed. +# @POST Only an expired idempotent/retry-safe claimed frontier is re-queued with a new attempt. +# Unsafe/missing claims and browser work without a pinned safe checkpoint terminalize non-pass. +# @INVARIANT Completed steps are never re-dispatched. Historical evidence is retained but retired +# from the new attempt projection before recovery can invoke an adapter. +# @INVARIANT Recovery reads only persisted ScenarioRun, ScenarioStepRun, lease, and runner_plan +# state; it never derives a new graph or execution authority from a mutable revision. +# @INVARIANT Browser recovery requires a non-empty checkpoint declared in the pinned step metadata +# or valid persisted LiveExecutionBinding; a dead browser session is never replayed as PASS. +# @REJECTED Re-reading a mutable revision, accepting a public resume token, or replaying an unsafe +# external effect after crash was rejected — persisted run/lease state is authoritative. +def recover_run( + db: Session, + run_id: str, + *, + worker_id: str, + registry: ScenarioExecutorRegistry | None = None, + live_binding_resolver: LiveExecutionBindingResolver | None = None, +) -> dict[str, Any]: + run = db.query(ScenarioRun).filter(ScenarioRun.id == run_id).first() + if run is None: + raise ValueError("run not found") + if run.status in {"waiting_human", "pending_approval", "cancel_requested", "cancelled"}: + raise ValueError("run is not crash-recoverable") + persisted_steps = ( + db.query(ScenarioStepRun) + .filter(ScenarioStepRun.run_id == run_id) + .order_by(ScenarioStepRun.step_position.asc()) + .all() + ) + if run.status in {"passed", "failed", "blocked", "inconclusive"}: + return build_result(run, persisted_steps) + if run.status not in {"queued", "running"}: + raise ValueError("run is not crash-recoverable") + + plan = run.runner_plan or {} + try: + validate_pinned_runner_plan(plan) + except ValueError as exc: + return _reject_malformed_plan(db, run, str(exc)) + recovered_step_ids, rejection = _recover_running_steps( + db, run, persisted_steps, plan, worker_id=worker_id + ) + if rejection is not None: + return rejection + + run.status = "queued" + run.phase = "executing" + db.flush() + logger.reason( + "ScenarioExecution.Runner.recover_run", + "Recovered persisted scenario frontier", + payload={"run_id": run.id, "worker_id": worker_id, "recovered_steps": recovered_step_ids}, + ) + return _advance_run( + db, + run, + registry or _build_default_registry(live_binding_resolver), + worker_id=worker_id, + ) + + +# #region ScenarioExecution.Runner.CrashRecovery.BrowserCheckpoint [C:3] [TYPE Function] [SEMANTICS scenario,execution,recovery,browser,checkpoint,fail-closed] +# @BRIEF Accept only one non-empty browser-safe checkpoint consistent with pinned plan and binding identity. +# @INVARIANT Missing, malformed, or mismatched checkpoint metadata returns no recovery authority; +# it never reconstructs a browser session or turns a crash into PASS. +def _browser_recovery_checkpoint(run: ScenarioRun, step_meta: dict[str, Any]) -> str | None: + plan_checkpoint = step_meta.get("browser_safe_checkpoint_ref") + binding_checkpoint: str | None = None + try: + if run.live_execution_binding_snapshot is not None: + binding_checkpoint = LiveExecutionBinding.from_snapshot( + run.live_execution_binding_snapshot + ).browser_safe_checkpoint_ref + except ValueError: + return None + if plan_checkpoint is not None and (not isinstance(plan_checkpoint, str) or not plan_checkpoint): + return None + if plan_checkpoint and binding_checkpoint and plan_checkpoint != binding_checkpoint: + return None + return plan_checkpoint or binding_checkpoint +# #endregion ScenarioExecution.Runner.CrashRecovery.BrowserCheckpoint + + +# #region ScenarioExecution.Runner.CrashRecovery.ArchiveAttempt [C:3] [TYPE Function] [SEMANTICS scenario,execution,recovery,attempt,artifact,provenance] +# @BRIEF Preserve an abandoned safe attempt before clearing only its active projection for recovery. +# @INVARIANT Recovery history and artifact rows remain auditable; only active refs/outcome are retired +# before an eligible new attempt can invoke an adapter. +def _archive_recovery_attempt( + db: Session, + run: ScenarioRun, + step: ScenarioStepRun, + *, + recovery_worker_id: str, + advance_attempt: bool = True, +) -> None: + prior_outputs = copy.deepcopy(step.outputs or {}) + history = list(prior_outputs.get("recovery_history", [])) + history.append({ + "attempt": step.attempt, + "status": step.status, + "artifact_refs": list(step.artifact_refs or []), + "error_code": step.error_code, + "step_outcome": copy.deepcopy(step.step_outcome or {}), + "recovery_worker_id": recovery_worker_id, + "reason": "EXPIRED_RETRY_SAFE_LEASE", + }) + prior_outputs["recovery_history"] = history + invalidate_step_evidence(db, run_id=run.id, logical_step_ids={step.logical_step_id}) + step.outputs = prior_outputs + step.artifact_refs = [] + step.step_outcome = {} + step.error_code = None + step.progress = 0 + step.status = "queued" + step.started_at = None + step.finished_at = None + if advance_attempt: + step.attempt += 1 + db.flush() +# #endregion ScenarioExecution.Runner.CrashRecovery.ArchiveAttempt + + +# #region ScenarioExecution.Runner.CrashRecovery.Reject [C:3] [TYPE Function] [SEMANTICS scenario,execution,recovery,reconciliation,terminal,signal] +# @BRIEF Terminalize an unsafe, missing-lease, or checkpointless recovery as typed non-pass. +# @INVARIANT Terminalization preserves historical evidence and reuses the canonical idempotent +# terminal signal for the same immutable rejected context. +def _terminalize_recovery_rejection( + db: Session, + run: ScenarioRun, + step: ScenarioStepRun, + status: str, + error_code: str, +) -> dict[str, Any]: + _archive_recovery_attempt( + db, + run, + step, + recovery_worker_id="recovery-rejected", + advance_attempt=False, + ) + step.status = status + step.progress = 100 + step.error_code = error_code + step.step_outcome = {"status": status, "error_code": error_code} + step.finished_at = datetime.now(UTC) + run.status = status + run.phase = "completed" + run.error_code = error_code + run.finished_at = step.finished_at + db.flush() + _record_terminal_side_effects(db, run) + db.flush() + return build_result( + run, + db.query(ScenarioStepRun).filter(ScenarioStepRun.run_id == run.id).all(), + ) +# #endregion ScenarioExecution.Runner.CrashRecovery.Reject +# #endregion ScenarioExecution.Runner.CrashRecovery + + +# #region ScenarioExecution.Runner.Walker [C:5] [TYPE Module] [SEMANTICS scenario,execution,walker,dag,step,artifact] +# @defgroup ScenarioExecution Deterministic DAG walker: create step rows, dispatch ready steps, handle human, aggregate. +# @RELATION DEPENDS_ON -> [ScenarioExecution.Lifecycle.Timeout] +# @INVARIANT Walker mutates only ScenarioRun/ScenarioStepRun owned by this run; scenario/revision rows are immutable. +# @INVARIANT Human steps never reach an executor; they suspend and wait for decide_checkpoint. +# @INVARIANT Artifact refs are attached to the owning step row; owner_type=scenario_run artifacts are registered. +# @INVARIANT Scenario evidence rows use only executor-supplied or locally computed real SHA-256 +# digests; missing/invalid digests are typed inconclusive and never replaced with zeros. +# @INVARIANT Every failed, blocked or inconclusive terminal run emits one idempotent canonical queue +# signal containing immutable run and durable artifact provenance, never agent work. +# @INVARIANT A non-human executor is claimed before adapter I/O. Cancellation/timeout observed +# during that adapter call wins over its returned payload and terminalizes without an +# active worker lease or evidence projection. +# @INVARIANT Every adapter invocation receives an exact persisted ActionExecutionDescriptor; lease +# timeout, idempotency, retry safety and side-effect-key policy derive from it, never +# from tool defaults or caller metadata. +# @REJECTED Registering evidence with an all-zero digest was rejected — it falsifies provenance. +# @REJECTED Treating every non-human tool as retry-safe was rejected because it can replay a +# mutating browser/report/artifact effect after crash or retry. +def _advance_run(db: Session, run: ScenarioRun, registry: ScenarioExecutorRegistry, *, worker_id: str = "api", lease_seconds: int = 30) -> dict[str, Any]: # noqa: C901 + plan = run.runner_plan or {} + try: + validate_pinned_runner_plan(plan) + except ValueError as exc: + return _reject_malformed_plan(db, run, str(exc)) + order: list[str] = plan.get("topological_order", []) + dependencies: list[dict[str, Any]] = plan.get("dependencies", []) + if not order: + run.status = "passed" + run.phase = "completed" + run.started_at = run.started_at or datetime.now(UTC) + run.finished_at = datetime.now(UTC) + db.flush() + return build_result(run, []) + + run.started_at = run.started_at or datetime.now(UTC) + for _ in range(len(order) + 1): + db.refresh(run) + if run.status in {"cancel_requested", "cancelled"}: + from src.services.dashboard_testing.execution.lifecycle import cancel_run + + cancel_run(db, run.id, drain_in_flight=True) + return build_result(run, db.query(ScenarioStepRun).filter(ScenarioStepRun.run_id == run.id).all()) + existing = {step.logical_step_id: step for step in db.query(ScenarioStepRun).filter(ScenarioStepRun.run_id == run.id).all()} + completed = {step_id: (existing[step_id].step_outcome or {"status": existing[step_id].status}) for step_id, step in existing.items() if step.status in {"passed", "failed", "inconclusive", "blocked", "skipped"}} + next_step_id = None + for step_id in order: + if step_id not in existing or existing[step_id].status == "queued": + next_step_id = step_id + break + if next_step_id is None: + run.status = build_result(run, list(existing.values()))["status"] + run.phase = "completed" if run.status in {"passed", "failed", "blocked", "inconclusive"} else run.phase + run.finished_at = datetime.now(UTC) + db.flush() + _record_terminal_side_effects(db, run) + db.flush() + return build_result(run, list(existing.values())) + + step_id = next_step_id + step_meta = next( + step for step in (plan.get("steps") or []) + if str(step.get("logical_step_id", step.get("id", ""))) == step_id + ) + tool = str(step_meta["tool"]) + descriptor = step_meta["action_descriptor"] + step: ScenarioStepRun | None = None + if tool != "human": + if step_id not in existing: + step = ScenarioStepRun( + id=str(uuid.uuid4()), + run_id=run.id, + logical_step_id=step_id, + step_position=order.index(step_id), + attempt=1, + status="queued", + inputs_snapshot={}, + outputs={}, + artifact_refs=[], + progress=0, + step_outcome={}, + ) + db.add(step) + db.flush() + existing[step_id] = step + step = existing[step_id] + step.status = "running" + step.started_at = datetime.now(UTC) + db.flush() + try: + claim_step( + db, + run.id, + step_id, + worker_id=worker_id, + side_effect_key=( + f"{run.id}:{step_id}:{step.attempt}" + if descriptor["side_effect_key_policy"] == "per_attempt" + else None + ), + idempotent=bool(descriptor["idempotent"]), + retry_safe=bool(descriptor["retry_safe"]), + lease_seconds=max(1, min(lease_seconds, int(descriptor["timeout_ms"]) // 1000)), + ) + except ValueError: + step.status = "queued" + db.flush() + return build_result(run, list(existing.values())) + outcome = dispatch_step( + { + "logical_step_id": step_id, + "tool": tool, + "action": step_meta["action"], + "action_descriptor": descriptor, + "step_meta": step_meta, + "scenario_run_id": run.id, + "target_snapshot": run.target_snapshot, + "execution_principal_fingerprint": run.execution_principal_fingerprint, + "live_execution_binding_ref": run.live_execution_binding_ref, + "live_execution_binding_snapshot": run.live_execution_binding_snapshot, + }, + completed=completed, + registry=registry, + edges=dependencies, + ) + db.refresh(run) + if ( + step is not None + and run.status == "inconclusive" + and step.status == "inconclusive" + and step.error_code == "STEP_TIMEOUT" + ): + return build_result(run, db.query(ScenarioStepRun).filter(ScenarioStepRun.run_id == run.id).all()) + if outcome.get("status") == "blocked": + step = step or existing.get(step_id) or ScenarioStepRun( + id=str(uuid.uuid4()), + run_id=run.id, + logical_step_id=step_id, + step_position=order.index(step_id), + attempt=1, + status="blocked", + inputs_snapshot={}, + outputs={}, + artifact_refs=[], + progress=100, + error_code=outcome.get("reason") or "dependency_failed", + step_outcome=outcome, + ) + step.status = "blocked" + step.progress = 100 + step.error_code = outcome.get("reason") or "dependency_failed" + step.step_outcome = outcome + step.finished_at = datetime.now(UTC) + db.add(step) + db.flush() + continue + if tool == "human": + step = existing.get(step_id) or ScenarioStepRun( + id=str(uuid.uuid4()), + run_id=run.id, + logical_step_id=step_id, + step_position=order.index(step_id), + attempt=1, + status="queued", + inputs_snapshot={}, + outputs={}, + artifact_refs=[], + progress=0, + step_outcome={}, + ) + step.status = "waiting_human" + step.progress = 100 + db.add(step) + db.flush() + suspend_for_human(db, run.id, step_id, evidence_refs=[]) + db.flush() + return build_result(run, [*existing.values(), step]) + assert step is not None + + step.step_outcome = outcome + step.status = str(outcome.get("status", "failed")) + step.progress = 100 if step.status in {"passed", "failed", "inconclusive", "blocked"} else 0 + if step.status in {"failed", "inconclusive", "blocked"}: + step.error_code = outcome.get("error_code") or "executor_failed" + if step.status in {"passed", "failed", "inconclusive", "blocked"}: + step.finished_at = datetime.now(UTC) + if outcome.get("output_refs"): + step.outputs = {"refs": outcome["output_refs"]} + if outcome.get("artifact_refs"): + step.artifact_refs = list(outcome["artifact_refs"]) + integrity = register_step_evidence( + db, + run_id=run.id, + logical_step_id=step_id, + artifact_refs=list(outcome["artifact_refs"]), + outcome=outcome, + attempt=step.attempt, + ) + if integrity is not None: + outcome = { + **outcome, + "status": "inconclusive", + "error_code": integrity["reason_code"], + "artifact_integrity": integrity, + } + step.step_outcome = outcome + step.status = "inconclusive" + step.error_code = integrity["reason_code"] + db.flush() + db.refresh(run) + if run.status == "cancel_requested": + from src.services.dashboard_testing.execution.lifecycle import cancel_run + + cancel_run(db, run.id, drain_in_flight=False) + return build_result(run, db.query(ScenarioStepRun).filter(ScenarioStepRun.run_id == run.id).all()) + if step.status in {"passed", "failed", "inconclusive", "blocked"}: + completed[step_id] = outcome + if run.status in {"pending_approval", "queued", "running", "waiting_human"}: + # A step-level inconclusive result is aggregated after all reachable steps; it must not + # make the run terminal before a later human checkpoint can safely suspend it. + run.status = step.status if step.status in {"failed", "blocked"} else run.status + run.phase = "executing" + db.flush() + + run.status = build_result(run, list(existing.values()))["status"] + run.phase = "completed" + run.finished_at = datetime.now(UTC) + _record_terminal_side_effects(db, run) + db.flush() + return build_result(run, list(existing.values())) +# #endregion ScenarioExecution.Runner.Walker + # #endregion ScenarioExecution.Runner diff --git a/backend/src/services/dashboard_testing/execution/runner_plan.py b/backend/src/services/dashboard_testing/execution/runner_plan.py index 83b26e2d9..055df052d 100644 --- a/backend/src/services/dashboard_testing/execution/runner_plan.py +++ b/backend/src/services/dashboard_testing/execution/runner_plan.py @@ -5,6 +5,8 @@ # @INVARIANT Same revision snapshot always yields byte-equivalent plan. # @INVARIANT runner.plan.json in git is a reference artifact, never the runtime source of truth. # @INVARIANT Derivation refuses any revision that is not the scenario's current revision (stale/unsafe). +# @INVARIANT A plan containing a human checkpoint derives manual_run_only=true; the start +# boundary uses this immutable plan fact to reject automation before a run exists. # @REJECTED Reading stored runner.plan.json at runtime was rejected; it is reference-only. # @RATIONALE The derivation is the single source of truth: a stale revision would dispatch steps # from a superseded graph, so the request must fail before a run row is created. @@ -17,6 +19,7 @@ from typing import Any from sqlalchemy.orm import Session from src.models.scenario_registry import ScenarioRegistryEntry, ScenarioRevision +from src.services.dashboard_testing.scenario.templates import validate_action_step def _topological_order(steps: list[dict[str, Any]], edges: list[dict[str, Any]]) -> list[str]: @@ -49,6 +52,12 @@ def _topological_order(steps: list[dict[str, Any]], edges: list[dict[str, Any]]) # @PRE revision_id belongs to scenario and contains an executable graph. # @POST Returns deterministic plan with content hash and plan hash. # @POST Refuses a revision that is not the scenario's current revision (revision mismatch). +# @POST A human checkpoint is represented as immutable manual_run_only=true for the 044 start boundary. +# @INVARIANT Every executable step persists an exact version/hash-pinned 038 ActionExecutionDescriptor; +# missing actions, stale registry identity, invalid I/O shape, and unsafe mutation metadata +# reject before a ScenarioRun/lease/adapter can exist. +# @REJECTED Falling back from a missing action to a tool default was rejected because it bypasses +# immutable ActionRegistry authority and fabricates retry safety. # @TEST_EDGE missing_revision -> reject; revision_mismatch -> reject before run row creation. def derive_runner_plan(db: Session, scenario_id: str, revision_id: str) -> dict[str, Any]: revision = db.query(ScenarioRevision).filter( @@ -67,23 +76,69 @@ def derive_runner_plan(db: Session, scenario_id: str, revision_id: str) -> dict[ f"revision mismatch: requested {revision_id}, current {entry.current_revision_id}" ) graph = dict(revision.graph_snapshot or {}) + registry_version = graph.get("action_registry_version") + registry_hash = graph.get("action_registry_hash") steps = list(graph.get("steps") or []) order = _topological_order(steps, list(graph.get("dependencies") or [])) - by_id = {str(step.get("logical_step_id", step.get("id", index))): step for index, step in enumerate(steps)} - executor_mapping = {step_id: str(by_id[step_id].get("tool", by_id[step_id].get("executor", "assertion"))) for step_id in order} + by_id: dict[str, dict[str, Any]] = {} + for index, source_step in enumerate(steps): + step_id = str(source_step.get("logical_step_id", source_step.get("id", index))) + descriptor = validate_action_step( + source_step, + registry_version=registry_version, + registry_hash=registry_hash, + ) + by_id[step_id] = { + **source_step, + "logical_step_id": step_id, + "action_descriptor": descriptor.snapshot(), + } + pinned_steps = [by_id[str(step.get("logical_step_id", step.get("id", index)))] for index, step in enumerate(steps)] + executor_mapping = { + step_id: by_id[step_id]["action_descriptor"] + for step_id in order + } + human_checkpoints = [step_id for step_id in order if by_id[step_id].get("tool") == "human"] plan = { "scenario_revision_id": revision.revision_id, "scenario_content_hash": revision.content_hash, "verification_program_hash": revision.content_hash, + "action_registry_version": registry_version, + "action_registry_hash": registry_hash, "env_targets": graph.get("environment_ids", []), "resolved_params": graph.get("parameters", {}), "pinned_baselines": graph.get("baselines", {}), "topological_order": order, "executor_mapping": executor_mapping, - "human_checkpoints": [step_id for step_id in order if by_id[step_id].get("tool") == "human"], + "human_checkpoints": human_checkpoints, + "manual_run_only": bool(human_checkpoints), + "dependencies": graph.get("dependencies", []), + "steps": pinned_steps, } plan["plan_hash"] = hashlib.sha256(json.dumps(plan, sort_keys=True, separators=(",", ":")).encode()).hexdigest() return plan # #endregion ScenarioExecution.RunnerPlan.Derive + +# #region ScenarioExecution.RunnerPlan.ValidatePinned [C:4] [TYPE Function] [SEMANTICS scenario,execution,runnerplan,action,preflight] +# @BRIEF Verify a persisted plan still contains exact immutable 038 action descriptors before a worker claim. +# @INVARIANT A legacy/malformed queued plan is blocked before a step lease or adapter call; descriptor +# snapshots must match the current version-pinned registry byte-for-byte. +# @REJECTED Reconstructing a missing descriptor from tool metadata was rejected because mutable or +# unknown effects would otherwise inherit a synthetic retry-safe contract. +def validate_pinned_runner_plan(plan: dict[str, Any]) -> None: + registry_version = plan.get("action_registry_version") + registry_hash = plan.get("action_registry_hash") + for step in plan.get("steps") or []: + if not isinstance(step, dict): + raise ValueError("ACTION_DESCRIPTOR_REQUIRED") + descriptor = validate_action_step( + step, + registry_version=registry_version, + registry_hash=registry_hash, + ) + if step.get("action_descriptor") != descriptor.snapshot(): + raise ValueError("ACTION_DESCRIPTOR_MISMATCH") +# #endregion ScenarioExecution.RunnerPlan.ValidatePinned + # #endregion ScenarioExecution.RunnerPlan diff --git a/backend/src/services/dashboard_testing/execution/worker.py b/backend/src/services/dashboard_testing/execution/worker.py index 387648d0c..46ee65244 100644 --- a/backend/src/services/dashboard_testing/execution/worker.py +++ b/backend/src/services/dashboard_testing/execution/worker.py @@ -10,13 +10,30 @@ from sqlalchemy.orm import Session from src.models.scenario_worker import ScenarioStepLease +# #region ScenarioExecution.Worker.NormalizeTimestamp [C:2] [TYPE Function] [SEMANTICS scenario,execution,worker,lease,time] +# @BRIEF Normalize SQLite's timezone-naive persisted datetimes to UTC before lease comparisons. +def _as_utc(value: datetime) -> datetime: + return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC) +# #endregion ScenarioExecution.Worker.NormalizeTimestamp + + # #region ScenarioExecution.Worker.Claim [C:4] [TYPE Function] [SEMANTICS scenario,execution,worker,claim,lease] +# @INVARIANT The lease row is locked while ownership is evaluated; an expired unsafe side effect +# is never reclaimed without reconciliation. +# @REJECTED Reclaiming every expired lease — an at-least-once retry may duplicate an unsafe +# external effect unless its recorded executor contract is idempotent or retry-safe. def claim_step(db: Session, run_id: str, logical_step_id: str, *, worker_id: str, side_effect_key: str | None, idempotent: bool, retry_safe: bool, lease_seconds: int = 30) -> ScenarioStepLease: + if lease_seconds <= 0: + raise ValueError("lease_seconds must be positive") now = datetime.now(UTC) - existing = db.query(ScenarioStepLease).filter(ScenarioStepLease.run_id == run_id, ScenarioStepLease.logical_step_id == logical_step_id).first() - if existing is not None and existing.expires_at > now and existing.worker_id != worker_id: + existing = db.query(ScenarioStepLease).filter( + ScenarioStepLease.run_id == run_id, + ScenarioStepLease.logical_step_id == logical_step_id, + ).with_for_update().first() + expires_at = _as_utc(existing.expires_at) if existing is not None else None + if existing is not None and expires_at > now and existing.worker_id != worker_id: raise ValueError("step lease held by another worker") - if existing is not None and not (existing.idempotent or existing.retry_safe) and existing.expires_at <= now: + if existing is not None and not (existing.idempotent or existing.retry_safe) and expires_at <= now: raise ValueError("expired non-retry-safe side effect requires reconciliation") lease = existing or ScenarioStepLease(run_id=run_id, logical_step_id=logical_step_id, worker_id=worker_id, side_effect_key=side_effect_key, idempotent=idempotent, retry_safe=retry_safe, expires_at=now) lease.worker_id = worker_id @@ -28,11 +45,18 @@ def claim_step(db: Session, run_id: str, logical_step_id: str, *, worker_id: str # #endregion ScenarioExecution.Worker.Claim # #region ScenarioExecution.Worker.Heartbeat [C:2] [TYPE Function] [SEMANTICS scenario,execution,worker,heartbeat,lease] +# @INVARIANT Worker identity and non-expired lease are checked under the same row lock before renewal. def heartbeat(db: Session, lease_id: str, *, worker_id: str, lease_seconds: int = 30) -> ScenarioStepLease: - lease = db.query(ScenarioStepLease).filter(ScenarioStepLease.id == lease_id, ScenarioStepLease.worker_id == worker_id).first() - if lease is None or lease.expires_at <= datetime.now(UTC): + if lease_seconds <= 0: + raise ValueError("lease_seconds must be positive") + now = datetime.now(UTC) + lease = db.query(ScenarioStepLease).filter( + ScenarioStepLease.id == lease_id, + ScenarioStepLease.worker_id == worker_id, + ).with_for_update().first() + if lease is None or _as_utc(lease.expires_at) <= now: raise ValueError("lease unavailable") - lease.heartbeat_at = datetime.now(UTC) + lease.heartbeat_at = now lease.expires_at = lease.heartbeat_at + timedelta(seconds=lease_seconds) db.flush() return lease diff --git a/backend/src/services/dashboard_testing/registry/staleness.py b/backend/src/services/dashboard_testing/registry/staleness.py index a2ca89858..9f1f97e80 100644 --- a/backend/src/services/dashboard_testing/registry/staleness.py +++ b/backend/src/services/dashboard_testing/registry/staleness.py @@ -22,6 +22,7 @@ from src.models.scenario_registry import ( ScenarioStalenessSeverity, ScenarioStalenessSignal, ) +from src.services.dashboard_testing.analytics.investigation import ingest_investigation_signal def _target_state(kind: str, severity: str) -> str: @@ -97,9 +98,37 @@ def apply_staleness(db: Session, signals: list[dict[str, Any]]) -> dict[str, Any )) if entry.lifecycle_status not in {ScenarioLifecycleState.ARCHIVED, ScenarioLifecycleState.BLOCKED}: entry.lifecycle_status = target_state + ingest_investigation_signal( + db, + { + "fingerprint": f"registry-stale:{scenario_id}:{source_type}:{fingerprint}:{affected_ref}", + "scenario_id": scenario_id, + "severity": severity, + "evidence": { + "source_type": source_type, + "source_fingerprint": fingerprint, + "affected_ref": affected_ref, + "reason": str(signal.get("reason", "upstream change affects scenario")), + }, + }, + ) applied.append({"scenario_id": scenario_id, "state": entry.lifecycle_status, "affected_ref": affected_ref}) db.flush() return {"applied": applied, "skipped": skipped, "applied_count": len(applied), "skipped_count": len(skipped)} + + +def ingest_upstream_staleness_event(db: Session, event: dict[str, Any]) -> dict[str, Any]: + """Production boundary for normalized 037 StructureDiff and 041 blast-radius envelopes.""" + source = str(event.get("source") or event.get("source_type") or "") + if source not in {"structure_diff", "lineage_blast_radius"}: + raise ValueError("unsupported staleness source") + normalized = { + **event, + "source_type": source, + "source_fingerprint": event.get("source_fingerprint") or event.get("fingerprint"), + "affected_ref": event.get("affected_ref") or event.get("reference"), + } + return apply_staleness(db, [normalized]) # #endregion ScenarioRegistry.Staleness.Apply # #endregion ScenarioRegistry.Staleness diff --git a/backend/src/services/dashboard_testing/scenario/templates/__init__.py b/backend/src/services/dashboard_testing/scenario/templates/__init__.py index 1e0bafa62..e1e5215eb 100644 --- a/backend/src/services/dashboard_testing/scenario/templates/__init__.py +++ b/backend/src/services/dashboard_testing/scenario/templates/__init__.py @@ -8,11 +8,15 @@ from __future__ import annotations +from dataclasses import asdict, dataclass +import hashlib +import json from typing import Any PHASES = ("setup", "interact", "observe", "assert", "evidence", "report") TOOLS = ("browser", "superset_api", "xlsx", "assertion", "screenshot", "report", "artifact", "human") RISKS = ("read", "browser_interaction", "draft_write", "human") +ACTION_REGISTRY_VERSION = "038.1.0" # Registered tool/action pairs (action -> phase, risk, required capability) REGISTERED_ACTIONS: dict[str, dict[str, Any]] = { @@ -39,6 +43,108 @@ REGISTERED_ACTIONS: dict[str, dict[str, Any]] = { "human_checkpoint": {"tool": "human", "phase": "assert", "risk": "human", "capability": None}, } + +# #region ScenarioGraph.Templates.ActionRegistry [C:5] [TYPE Class] [SEMANTICS scenario,action,registry,descriptor,side-effect] +# @BRIEF Version-pinned 038 ActionRegistry execution metadata shared with 044 RunnerPlan preflight. +# @INVARIANT A descriptor is selected only by its exact registered {tool, action}, carries its +# retry/side-effect contract, and is fingerprinted with the immutable registry version. +# @RATIONALE 044 must not infer retry safety from a tool: browser mutation and read-only browser +# actions have materially different replay authority. +# @REJECTED A tool-only executor default or universal retry_safe claim was rejected because it can +# replay an unsafe external effect after a lease expiry. +@dataclass(frozen=True) +class ActionExecutionDescriptor: + tool: str + action: str + input_contract: str + output_contract: str + idempotent: bool + retry_safe: bool + side_effect_key_policy: str + timeout_ms: int + risk: str + mutating: bool + + def snapshot(self) -> dict[str, Any]: + return asdict(self) + + +_MUTATING_ACTIONS = frozenset({"row_edit", "bulk_edit", "generate_report", "register_artifact"}) +_BROWSER_ACTIONS = frozenset({ + "open_dashboard", "apply_filters", "text_filter", "table_filter", "pagination", + "row_edit", "bulk_edit", "download_xlsx", "navigate_dashboard", +}) +_TIMEOUTS = { + "download_xlsx": 60000, + "capture_screenshot": 30000, + "execute_metric": 30000, + "dataset_field_assert": 30000, + "dataset_rowset_assert": 30000, +} + + +def _descriptor_for(action: str, entry: dict[str, Any]) -> ActionExecutionDescriptor: + mutating = action in _MUTATING_ACTIONS + browser_interaction = entry["tool"] == "browser" and action in _BROWSER_ACTIONS + if mutating: + return ActionExecutionDescriptor( + tool=entry["tool"], action=action, input_contract="mutation_contract_v1", + output_contract="durable_evidence_v1", idempotent=False, retry_safe=False, + side_effect_key_policy="per_attempt", timeout_ms=_TIMEOUTS.get(action, 30000), + risk=entry["risk"], mutating=True, + ) + return ActionExecutionDescriptor( + tool=entry["tool"], action=action, input_contract="scenario_step_v1", + output_contract="scenario_outcome_v1", idempotent=True, retry_safe=True, + side_effect_key_policy="none", timeout_ms=_TIMEOUTS.get(action, 15000), + risk=entry["risk"], mutating=False, + ) + + +def action_registry_fingerprint() -> str: + payload = { + "version": ACTION_REGISTRY_VERSION, + "descriptors": { + action: _descriptor_for(action, entry).snapshot() + for action, entry in sorted(REGISTERED_ACTIONS.items()) + }, + } + return hashlib.sha256( + json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + +def resolve_action_descriptor( + *, tool: str, action: str, registry_version: str | None, registry_hash: str | None, +) -> ActionExecutionDescriptor: + if registry_version != ACTION_REGISTRY_VERSION: + raise ValueError("ACTION_REGISTRY_VERSION_MISMATCH") + if registry_hash != action_registry_fingerprint(): + raise ValueError("ACTION_REGISTRY_HASH_MISMATCH") + entry = REGISTERED_ACTIONS.get(action) + if entry is None: + raise ValueError("ACTION_DESCRIPTOR_UNKNOWN") + if entry["tool"] != tool: + raise ValueError("ACTION_DESCRIPTOR_TOOL_MISMATCH") + return _descriptor_for(action, entry) + + +def validate_action_step( + step: dict[str, Any], *, registry_version: str | None, registry_hash: str | None, +) -> ActionExecutionDescriptor: + tool, action = step.get("tool"), step.get("action") + if not isinstance(tool, str) or not isinstance(action, str): + raise ValueError("ACTION_DESCRIPTOR_REQUIRED") + descriptor = resolve_action_descriptor( + tool=tool, action=action, registry_version=registry_version, registry_hash=registry_hash, + ) + if any(key in step and not isinstance(step[key], (dict, list)) for key in ("inputs", "outputs")): + raise ValueError("ACTION_DESCRIPTOR_IO_SHAPE_INVALID") + if descriptor.mutating and not isinstance(step.get("mutation_contract"), dict): + raise ValueError("ACTION_MUTATION_CONTRACT_REQUIRED") + return descriptor +# #endregion ScenarioGraph.Templates.ActionRegistry + # Step templates: mapping name -> (action, tool, phase) STEP_TEMPLATES: dict[str, tuple[str, str, str]] = { "browser_apply_observe_assert": ("open_dashboard", "browser", "setup"), diff --git a/backend/tests/api/test_scenario_analytics_api.py b/backend/tests/api/test_scenario_analytics_api.py index 305d104d8..260ffa782 100644 --- a/backend/tests/api/test_scenario_analytics_api.py +++ b/backend/tests/api/test_scenario_analytics_api.py @@ -335,13 +335,13 @@ class TestCaseRoutes: client = analytics_route_env.client() resp = client.post( f"/api/scenario-analytics/cases/{CASE_OPEN}/disposition", - json={"disposition": "confirmed", "expected_version": 1}, + json={"disposition": "resolved", "expected_version": 1, "verification_evidence": {"reconciled": True}}, ) assert resp.status_code == 200, resp.text body = resp.json() assert body["status"] == "resolved" assert body["decision_version"] == 2 - assert body["disposition"] == "confirmed" + assert body["disposition"] == "resolved" def test_disposition_requires_triage_permission(self, analytics_route_env): resp = analytics_route_env.client(user=VIEWER).post( @@ -413,7 +413,7 @@ class TestAnalyticsReads: before = {r.id: r.status for r in env.session_factory().query(ScenarioRun).all()} env.client().post( f"/api/scenario-analytics/cases/{CASE_OPEN}/disposition", - json={"disposition": "confirmed", "expected_version": 1}, + json={"disposition": "resolved", "expected_version": 1, "verification_evidence": {"reconciled": True}}, ) after = {r.id: r.status for r in env.session_factory().query(ScenarioRun).all()} assert before == after diff --git a/backend/tests/api/test_scenario_automation_api.py b/backend/tests/api/test_scenario_automation_api.py index 67cb58fa2..ed5d30762 100644 --- a/backend/tests/api/test_scenario_automation_api.py +++ b/backend/tests/api/test_scenario_automation_api.py @@ -2,10 +2,24 @@ # @BRIEF Verify scenario-automation API surface: schedule/trigger-rule/policy CRUD, metrics, # notifications, RBAC scopes and the direct PROD-gated scenario trigger. # @RELATION BINDS_TO -> [Api.ScenarioAutomation.Routes] +# @RELATION VERIFIES -> [Api.ScenarioAutomation.DirectTrigger] +# @RELATION BINDS_TO -> [ScenarioExecution.Runner.QueuedDispatch] +# @RELATION VERIFIES -> [ScenarioExecution.EnvironmentPolicy.Resolve] +# @RELATION VERIFIES -> [ScenarioExecution.Runner.TriggerSource.RejectAutomatedHuman] # @TEST_EDGE missing_manage_scope -> 403 on schedule mutation # @TEST_EDGE missing_trigger_scope -> 403 on direct trigger # @TEST_EDGE missing_prod_scope -> 403 on PROD direct trigger # @TEST_EDGE idempotency_reuse -> 409 on changed request hash +# @TEST_INVARIANT ScenarioExecution.Runner.Start: The 046 API trigger supplies a server-owned +# automation source; a persisted human graph returns the typed manual-only +# rejection before it creates a run, gate, or notification. +# @TEST_INVARIANT Api.ScenarioAutomation.DirectTrigger: A non-PROD HTTP trigger/replay persists +# only its queued row; its external boundary does not advance until the separate +# 044 dispatcher claims it through status CAS. -> VERIFIED_BY: +# test_direct_trigger_starts_run_with_idempotency +# @TEST_INVARIANT ScenarioExecution.EnvironmentPolicy: A direct target's server-owned PROD class +# requires automation PROD and scenario RUN_PROD before start; client payload has +# no override. -> VERIFIED_BY: test_prod_trigger_requires_prod_scope from __future__ import annotations from collections.abc import Iterator @@ -20,6 +34,11 @@ from src.core.database import SessionLocal from src.dependencies import get_config_manager, get_current_user from src.models.auth import Permission, Role, User from src.models.scenario_registry import ScenarioRegistryEntry +from src.services.dashboard_testing.scenario.templates import ( + ACTION_REGISTRY_VERSION, + action_registry_fingerprint, + resolve_action_descriptor, +) def _make_user_with_permissions(permissions: list[tuple[str, str]]) -> User: @@ -33,6 +52,23 @@ def _make_user_with_permissions(permissions: list[tuple[str, str]]) -> User: return user +def _action_step(step_id: str, tool: str, action: str, **extra) -> dict: + """Hardcoded exact 038 action fixture; no tool-only dispatch is valid.""" + return { + "id": step_id, + "logical_step_id": step_id, + "tool": tool, + "action": action, + "action_descriptor": resolve_action_descriptor( + tool=tool, + action=action, + registry_version=ACTION_REGISTRY_VERSION, + registry_hash=action_registry_fingerprint(), + ).snapshot(), + **extra, + } + + # #region Test.Api.ScenarioAutomation.ConfigManager [C:2] [TYPE Block] [SEMANTICS test,api,scenario,automation,env,fixture] # @BRIEF Server-owned environment classification stub: PROD classification must come from # ConfigManager (stage/is_production), never from the environment_id string. @@ -232,9 +268,9 @@ class TestScenarioAutomationRbac: scenario_id="44444444-4444-4444-8444-444444444444", content_hash="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", graph_snapshot={ - "steps": [ - {"id": "step-1", "logical_step_id": "step-1", "tool": "assertion"} - ], + "action_registry_version": ACTION_REGISTRY_VERSION, + "action_registry_hash": action_registry_fingerprint(), + "steps": [_action_step("step-1", "assertion", "structural_assert")], "dependencies": [], "environment_ids": ["env-preprod-01"], }, @@ -255,6 +291,7 @@ class TestScenarioAutomationRbac: ) assert resp.status_code == 202, resp.text body = resp.json() + # HTTP trigger persists only; no executor runs before the server dispatcher claim. assert body["status"] == "queued" assert body["run_id"] # Same idempotency key returns the same run; a changed request is rejected. @@ -272,5 +309,86 @@ class TestScenarioAutomationRbac: ) assert changed.status_code == 409 assert changed.json()["detail"]["code"] == "IDEMPOTENCY_KEY_REUSED" + + from src.services.dashboard_testing.execution.runner import dispatch_queued_runs + + dispatcher = SessionLocal() + try: + outcomes = dispatch_queued_runs(dispatcher, worker_id="automation-api-dispatch") + dispatcher.commit() + finally: + dispatcher.close() + assert [outcome["status"] for outcome in outcomes] == ["inconclusive"] + + # #region Test.Api.ScenarioAutomation.Rbac.ManualOnly [C:3] [TYPE Function] + # @BRIEF The external automation endpoint refuses a persisted human graph before any run-side effect. + # @TEST_INVARIANT ScenarioExecution.Runner.Start: An API automation request for a human graph + # cannot materialize a manual ScenarioRun or skip its HumanCheckpoint. + def test_direct_trigger_rejects_human_revision_before_run_creation(self): + from src.models.scenario_approval import ActionApprovalGate + from src.models.scenario_automation import ScenarioNotificationEvent + from src.models.scenario_registry import ScenarioRevision + from src.models.scenario_run import ScenarioRun + + scenario_id = "60460000-0000-4000-8000-000000000004" + revision_id = "60460000-0000-4000-8000-000000000014" + setup = SessionLocal() + try: + setup.query(ScenarioRun).filter(ScenarioRun.scenario_id == scenario_id).delete() + setup.query(ScenarioRegistryEntry).filter(ScenarioRegistryEntry.scenario_id == scenario_id).delete() + setup.commit() + setup.add(ScenarioRegistryEntry( + scenario_id=scenario_id, + scenario_key="api-manual-only-044", + name="API manual-only fixture", + dashboard_id=46, + environment_ids=["env-preprod-01"], + owner_id="analyst-046", + owner_username="analyst.046", + lifecycle_status="READY", + validation_status="valid", + current_revision_id=revision_id, + )) + setup.add(ScenarioRevision( + revision_id=revision_id, + scenario_id=scenario_id, + content_hash="6" * 64, + graph_snapshot={ + "action_registry_version": ACTION_REGISTRY_VERSION, + "action_registry_hash": action_registry_fingerprint(), + "steps": [_action_step("human-api-044", "human", "human_checkpoint")], + "dependencies": [], + }, + created_by="analyst-046", + activation_status="current", + )) + setup.commit() + before = ( + setup.query(ActionApprovalGate).count(), + setup.query(ScenarioNotificationEvent).filter(ScenarioNotificationEvent.scenario_id == scenario_id).count(), + ) + finally: + setup.close() + + user = _make_user_with_permissions([("scenario:automation", "TRIGGER")]) + with _client_for(user) as client: + response = client.post( + f"/api/scenario-automation/scenarios/{scenario_id}/trigger", + headers={"Idempotency-Key": "api-human-manual-only-044"}, + json={"environment_id": "env-preprod-01", "revision_id": revision_id}, + ) + + assert response.status_code == 409, response.text + assert response.json()["detail"]["code"] == "AUTOMATION_INELIGIBLE_HUMAN_STEP" + verify = SessionLocal() + try: + assert verify.query(ScenarioRun).filter(ScenarioRun.scenario_id == scenario_id).count() == 0 + assert ( + verify.query(ActionApprovalGate).count(), + verify.query(ScenarioNotificationEvent).filter(ScenarioNotificationEvent.scenario_id == scenario_id).count(), + ) == before + finally: + verify.close() + # #endregion Test.Api.ScenarioAutomation.Rbac.ManualOnly # #endregion Test.Api.ScenarioAutomation.Rbac # #endregion Test.Api.ScenarioAutomation diff --git a/backend/tests/api/test_scenario_runs_api.py b/backend/tests/api/test_scenario_runs_api.py index 1ff4c4b9a..22d21643e 100644 --- a/backend/tests/api/test_scenario_runs_api.py +++ b/backend/tests/api/test_scenario_runs_api.py @@ -1,6 +1,32 @@ # #region Test.ScenarioExecution.Api [C:3] [TYPE Module] [SEMANTICS test,api,scenario,run,start,sse,compare,resume,rbac,prod] # @defgroup ScenarioExecution Test.Api full scenario-runs API (044 T018/T019). # @RELATION VERIFIES -> [Api.ScenarioExecution.Routes] +# @RELATION VERIFIES -> [Api.ScenarioExecution.Routes.Start] +# @RELATION VERIFIES -> [Api.ScenarioExecution.Routes.ApprovalDecision] +# @TEST_CONTRACT: Persisted ScenarioRun frontier + HumanCheckpoint or infrastructure token -> resumed DAG result without re-running completed steps +# @TEST_INVARIANT Api.ScenarioExecution.Routes: Static /compare remains registered before /{run_id}. +# -> VERIFIED_BY: test_compare_is_not_shadowed_by_run_id +# @TEST_INVARIANT Api.ScenarioExecution.Routes.Start: HTTP start/replay is persistence-only: it +# requires scenario:run (and scenario:run:prod for PROD), returns only a durable +# queued/pending row, and creates no step/lease/notification/queue side effect. +# -> VERIFIED_BY: test_start_creates_run_201, test_idempotent_replay_returns_same_run, +# test_rbac_denied_403, test_prod_start_requires_run_prod_when_body_says_false, +# test_prod_start_with_run_prod_creates_pending_approval +# @TEST_INVARIANT ScenarioExecution.EnvironmentPolicy: ConfigManager, never request is_prod, +# classifies PROD before RBAC/gate creation; unknown environments have no run-side +# effect. -> VERIFIED_BY: test_prod_start_requires_run_prod_when_body_says_false, +# test_preprod_true_body_stays_queued, test_prod_replay_ignores_body_classification, +# test_unknown_environment_has_no_run_side_effect +# @TEST_INVARIANT Api.ScenarioExecution.Routes.ApprovalDecision: PROD approval decisions require +# the gate's dedicated scenario:run_prod authority. -> VERIFIED_BY: +# test_approve_prod_gate_queues_run_and_replay_conflicts, +# test_approval_decision_requires_run_prod +# @TEST_INVARIANT ScenarioExecution.Runner.ContinueAfterHuman: A confirmed checkpoint continues +# only dependent saved-frontier steps; completed local deterministic steps stay attempt 1. +# -> VERIFIED_BY: test_human_confirm_resumes_dependents_without_rerunning_completed_steps +# @TEST_INVARIANT ScenarioExecution.Runner.ContinueAfterInfrastructureResume: A valid one-time +# resume token continues only the saved frontier; stale/replayed tokens reject. +# -> VERIFIED_BY: test_infrastructure_resume_continues_saved_frontier_without_rerunning_completed_step # @TEST_EDGE static_compare_before_dynamic_run_id; idempotent_replay; changed_request_409; # stale_revision_409; rbac_403; prod_requires_run_prod; sse_replay_via_last_event_id; # human_checkpoint_resume_409 @@ -22,8 +48,14 @@ from sqlalchemy.pool import StaticPool from src.api.routes.dashboard_testing.scenario_runs import history_router, runs_router from src.core.database import get_db -from src.dependencies import get_current_user +from src.dependencies import get_config_manager, get_current_user from src.models.auth import Permission, Role, User +from src.models.scenario_run import ScenarioRun +from src.services.dashboard_testing.scenario.templates import ( + ACTION_REGISTRY_VERSION, + action_registry_fingerprint, + resolve_action_descriptor, +) _FIXTURE_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "scenario_execution" _SCENARIO = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1" @@ -33,6 +65,47 @@ _RUN_1 = "run-exec-0001" _RUN_2 = "run-exec-0002" +def _action_step(step_id: str, tool: str, action: str, **extra) -> dict: + return { + "logical_step_id": step_id, + "tool": tool, + "action": action, + "action_descriptor": resolve_action_descriptor( + tool=tool, action=action, registry_version=ACTION_REGISTRY_VERSION, + registry_hash=action_registry_fingerprint(), + ).snapshot(), + **extra, + } + + +_HUMAN_CONTINUATION_GRAPH = { + "schema_version": 1, + "action_registry_version": ACTION_REGISTRY_VERSION, + "action_registry_hash": action_registry_fingerprint(), + "environment_ids": ["env-preprod-02"], + "steps": [ + _action_step("before_human", "assertion", "structural_assert", actual=11, expected=11), + _action_step("human_signoff", "human", "human_checkpoint"), + _action_step("after_human", "assertion", "structural_assert", actual=7, expected=7), + ], + "dependencies": [ + {"source": "before_human", "target": "human_signoff"}, + {"source": "human_signoff", "target": "after_human"}, + ], +} +_INFRA_RESUME_GRAPH = { + "schema_version": 1, + "action_registry_version": ACTION_REGISTRY_VERSION, + "action_registry_hash": action_registry_fingerprint(), + "environment_ids": ["env-preprod-02"], + "steps": [ + _action_step("before_resume", "browser", "open_dashboard", session_id="fixture-browser-session"), + _action_step("after_resume", "assertion", "structural_assert", actual=7, expected=7), + ], + "dependencies": [{"source": "before_resume", "target": "after_resume"}], +} + + def _seed(session) -> None: from src.models.scenario_registry import ScenarioRegistryEntry, ScenarioRevision from src.models.scenario_run import ScenarioRun, ScenarioStepRun @@ -119,6 +192,14 @@ class RunRouteEnv: self.session_factory = sessionmaker(bind=self.engine) self.admin = admin self.user = user + self.config_manager = SimpleNamespace( + get_environment=lambda environment_id: { + "env-preprod-02": SimpleNamespace( + stage="PREPROD", is_production=False + ), + "env-prod-01": SimpleNamespace(stage="PROD", is_production=True), + }.get(environment_id) + ) def seed(self) -> None: session = self.session_factory() @@ -149,6 +230,7 @@ class RunRouteEnv: user = SimpleNamespace(id="u1", username="u1", roles=[]) app.dependency_overrides[get_current_user] = lambda: user app.dependency_overrides[get_db] = _override_db + app.dependency_overrides[get_config_manager] = lambda: self.config_manager return TestClient(app) def close(self) -> None: @@ -179,12 +261,24 @@ class TestStart: assert body["runner_plan"]["topological_order"][0] == "s1_setup_filters" def test_idempotent_replay_returns_same_run(self, route_env): + from src.models.scenario_automation import ScenarioNotificationEvent + from src.models.scenario_investigation import InvestigationQueueItem + from src.models.scenario_run import ScenarioStepRun + client = route_env.client() payload = {"scenario_id": _SCENARIO, "revision_id": _REV_CURRENT, "environment_id": "env-preprod-02", "params": {}} first = client.post("/api/scenario-runs", headers={"Idempotency-Key": "start-2"}, json=payload) second = client.post("/api/scenario-runs", headers={"Idempotency-Key": "start-2"}, json=payload) assert first.status_code == 201 and second.status_code == 201 assert first.json()["id"] == second.json()["id"] + session = route_env.session_factory() + try: + run_id = first.json()["id"] + assert session.query(ScenarioStepRun).filter_by(run_id=run_id).count() == 0 + assert session.query(InvestigationQueueItem).filter_by(run_id=run_id).count() == 0 + assert session.query(ScenarioNotificationEvent).filter_by(run_id=run_id).count() == 0 + finally: + session.close() def test_changed_request_same_key_409(self, route_env): client = route_env.client() @@ -219,14 +313,14 @@ class TestStart: finally: env.close() - def test_prod_start_requires_scenario_run_prod(self): + def test_prod_start_requires_run_prod_when_body_says_false(self): env = RunRouteEnv(admin=False, user=_user_with(("scenario", "RUN"))) env.seed() try: resp = env.client().post( "/api/scenario-runs", headers={"Idempotency-Key": "start-prod-1"}, - json={"scenario_id": _SCENARIO, "revision_id": _REV_CURRENT, "environment_id": "env-prod-01", "params": {}, "is_prod": True}, + json={"scenario_id": _SCENARIO, "revision_id": _REV_CURRENT, "environment_id": "env-prod-01", "params": {}, "is_prod": False}, ) assert resp.status_code == 403 finally: @@ -239,12 +333,92 @@ class TestStart: resp = env.client().post( "/api/scenario-runs", headers={"Idempotency-Key": "start-prod-2"}, - json={"scenario_id": _SCENARIO, "revision_id": _REV_CURRENT, "environment_id": "env-prod-01", "params": {}, "is_prod": True}, + json={"scenario_id": _SCENARIO, "revision_id": _REV_CURRENT, "environment_id": "env-prod-01", "params": {}, "is_prod": False}, ) assert resp.status_code == 201, resp.text assert resp.json()["status"] == "pending_approval" finally: env.close() + + def test_preprod_true_body_stays_queued(self, route_env): + response = route_env.client().post( + "/api/scenario-runs", + headers={"Idempotency-Key": "start-preprod-true-044"}, + json={ + "scenario_id": _SCENARIO, + "revision_id": _REV_CURRENT, + "environment_id": "env-preprod-02", + "params": {}, + "is_prod": True, + }, + ) + assert response.status_code == 201, response.text + assert response.json()["status"] == "queued" + + def test_prod_replay_ignores_body_classification(self): + from src.models.scenario_approval import ActionApprovalGate + from src.models.scenario_run import ScenarioStepRun + + env = RunRouteEnv( + admin=False, + user=_user_with(("scenario", "RUN"), ("scenario", "RUN_PROD")), + ) + env.seed() + try: + client = env.client() + payload = { + "scenario_id": _SCENARIO, + "revision_id": _REV_CURRENT, + "environment_id": "env-prod-01", + "params": {"hardcoded": "prod-replay-044"}, + } + first = client.post( + "/api/scenario-runs", + headers={"Idempotency-Key": "start-prod-replay-class-044"}, + json={**payload, "is_prod": False}, + ) + second = client.post( + "/api/scenario-runs", + headers={"Idempotency-Key": "start-prod-replay-class-044"}, + json={**payload, "is_prod": True}, + ) + assert first.status_code == 201, first.text + assert second.status_code == 201, second.text + assert first.json()["id"] == second.json()["id"] + assert second.json()["status"] == "pending_approval" + session = env.session_factory() + try: + run_id = first.json()["id"] + assert session.query(ActionApprovalGate).filter_by(owner_id=run_id).count() == 1 + assert session.query(ScenarioStepRun).filter_by(run_id=run_id).count() == 0 + finally: + session.close() + finally: + env.close() + + def test_unknown_environment_has_no_run_side_effect(self, route_env): + session = route_env.session_factory() + try: + before = session.query(ScenarioRun).count() + finally: + session.close() + response = route_env.client().post( + "/api/scenario-runs", + headers={"Idempotency-Key": "start-unknown-environment-044"}, + json={ + "scenario_id": _SCENARIO, + "revision_id": _REV_CURRENT, + "environment_id": "env-unknown-044", + "params": {}, + }, + ) + assert response.status_code == 422 + assert response.json()["detail"]["code"] == "ENVIRONMENT_NOT_CONFIGURED" + session = route_env.session_factory() + try: + assert session.query(ScenarioRun).count() == before + finally: + session.close() # #endregion Test.ScenarioExecution.Api.TestStart @@ -414,6 +588,145 @@ class TestSse: # #region Test.ScenarioExecution.Api.TestLifecycleRoutes [C:2] [TYPE Class] class TestLifecycleRoutes: + def test_infrastructure_resume_continues_saved_frontier_without_rerunning_completed_step(self, route_env): + from src.models.scenario_registry import ScenarioRevision + from src.models.scenario_run import ScenarioStepRun + from src.services.dashboard_testing.execution.lifecycle import pause_for_infrastructure + from src.services.dashboard_testing.execution.runner import start_run + + session = route_env.session_factory() + try: + revision = session.query(ScenarioRevision).filter_by(revision_id=_REV_CURRENT).one() + revision.graph_snapshot = _INFRA_RESUME_GRAPH + session.flush() + run = start_run( + session, _SCENARIO, _REV_CURRENT, {}, "env-preprod-02", + actor="infra-continuation", idempotency_key="infra-continuation-1", + config_manager=route_env.config_manager, + auto_advance=False, + ) + session.add(ScenarioStepRun( + run_id=run.id, logical_step_id="before_resume", step_position=0, + attempt=1, status="passed", progress=100, + step_outcome={"logical_step_id": "before_resume", "status": "passed"}, + )) + _, token = pause_for_infrastructure(session, run.id) + session.commit() + run_id = run.id + finally: + session.close() + + client = route_env.client() + invalid = client.post( + f"/api/scenario-runs/{run_id}/resume", + json={"resume_token": "stale-token", "resume_reason": "worker_recovered"}, + ) + assert invalid.status_code == 409 + + resumed = client.post( + f"/api/scenario-runs/{run_id}/resume", + json={"resume_token": token, "resume_reason": "worker_recovered"}, + ) + assert resumed.status_code == 200, resumed.text + assert resumed.json()["status"] == "passed" + result = client.get(f"/api/scenario-runs/{run_id}/result") + steps = {step["logical_step_id"]: step for step in result.json()["snapshot"]["steps"]} + assert result.json()["status"] == "passed" + assert steps["before_resume"]["status"] == "passed" + assert steps["before_resume"]["attempt"] == 1 + assert steps["after_resume"]["status"] == "passed" + assert steps["after_resume"]["attempt"] == 1 + + replay = client.post( + f"/api/scenario-runs/{run_id}/resume", + json={"resume_token": token, "resume_reason": "worker_recovered"}, + ) + assert replay.status_code == 409 + + def test_human_confirm_resumes_dependents_without_rerunning_completed_steps(self, route_env): + from src.models.scenario_registry import ScenarioRevision + from src.models.scenario_run import ScenarioStepRun + from src.services.dashboard_testing.execution.runner import dispatch_queued_runs, start_run + + session = route_env.session_factory() + try: + revision = session.query(ScenarioRevision).filter_by(revision_id=_REV_CURRENT).one() + revision.graph_snapshot = _HUMAN_CONTINUATION_GRAPH + session.flush() + run = start_run( + session, _SCENARIO, _REV_CURRENT, {}, "env-preprod-02", + actor="human-continuation", idempotency_key="human-continuation-1", + config_manager=route_env.config_manager, + ) + session.commit() + run_id = run.id + assert session.query(ScenarioStepRun).filter_by(run_id=run_id).count() == 0 + dispatch_queued_runs(session, worker_id="api-human-continuation") + session.commit() + before = session.query(ScenarioStepRun).filter_by( + run_id=run_id, logical_step_id="before_human" + ).one() + assert before.status == "passed" + assert before.attempt == 1 + finally: + session.close() + + decision = route_env.client().post( + f"/api/scenario-runs/{run_id}/human/decision", + json={"disposition": "confirm", "expected_version": 1, "comment": "fixture sign-off"}, + ) + + assert decision.status_code == 200, decision.text + result = route_env.client().get(f"/api/scenario-runs/{run_id}/result") + assert result.status_code == 200, result.text + assert result.json()["status"] == "passed" + steps = {step["logical_step_id"]: step for step in result.json()["snapshot"]["steps"]} + assert steps["before_human"]["status"] == "passed" + assert steps["before_human"]["attempt"] == 1 + assert steps["human_signoff"]["status"] == "passed" + assert steps["after_human"]["status"] == "passed" + + @pytest.mark.parametrize( + ("disposition", "error_code"), + [("false_positive", "HUMAN_FALSE_POSITIVE"), ("inconclusive", "HUMAN_INCONCLUSIVE")], + ) + def test_human_nonpass_dispositions_resume_with_explicit_inconclusive_outcome( + self, route_env, disposition, error_code, + ): + from src.models.scenario_registry import ScenarioRevision + from src.services.dashboard_testing.execution.runner import dispatch_queued_runs, start_run + + session = route_env.session_factory() + try: + revision = session.query(ScenarioRevision).filter_by(revision_id=_REV_CURRENT).one() + revision.graph_snapshot = _HUMAN_CONTINUATION_GRAPH + session.flush() + run = start_run( + session, _SCENARIO, _REV_CURRENT, {}, "env-preprod-02", + actor="human-nonpass", idempotency_key=f"human-nonpass-{disposition}", + config_manager=route_env.config_manager, + ) + session.commit() + run_id = run.id + dispatch_queued_runs(session, worker_id=f"api-human-nonpass-{disposition}") + session.commit() + finally: + session.close() + + decision = route_env.client().post( + f"/api/scenario-runs/{run_id}/human/decision", + json={"disposition": disposition, "expected_version": 1}, + ) + + assert decision.status_code == 200, decision.text + result = route_env.client().get(f"/api/scenario-runs/{run_id}/result") + steps = {step["logical_step_id"]: step for step in result.json()["snapshot"]["steps"]} + assert result.json()["status"] == "inconclusive" + assert steps["human_signoff"]["status"] == "inconclusive" + assert steps["human_signoff"]["error_code"] == error_code + assert steps["human_signoff"]["step_outcome"]["disposition"] == disposition + assert steps["after_human"]["status"] == "passed" + def test_cancel_run(self, route_env): client = route_env.client() created = client.post( @@ -446,7 +759,8 @@ class TestLifecycleRoutes: json={"resume_token": token, "resume_reason": "worker_recovered"}, ) assert resp.status_code == 200, resp.text - assert resp.json()["phase"] == "executing" + assert resp.json()["status"] == "waiting_human" + assert resp.json()["phase"] == "waiting_human" def test_resume_human_checkpoint_409(self, route_env): from src.services.dashboard_testing.execution.lifecycle import suspend_for_human @@ -470,11 +784,11 @@ class TestLifecycleRoutes: assert resp.status_code == 409 assert "human checkpoint" in resp.json()["detail"]["detail"] - def test_retry_step_invalidates_closure(self, route_env): + def test_retry_step_rejects_legacy_unsafe_plan(self, route_env): resp = route_env.client().post(f"/api/scenario-runs/{_RUN_1}/steps/s3_export_xlsx/retry") - assert resp.status_code == 200, resp.text - affected = resp.json() - assert any(step["logical_step_id"] == "s3_export_xlsx" and step["attempt"] == 2 for step in affected) + assert resp.status_code == 409, resp.text + assert resp.json()["detail"]["code"] == "RETRY_CONFLICT" + assert "non-retry-safe action descriptor" in resp.json()["detail"]["detail"] def test_retry_missing_step_409(self, route_env): resp = route_env.client().post(f"/api/scenario-runs/{_RUN_1}/steps/nope/retry") diff --git a/backend/tests/fixtures/scenario_execution/graph.json b/backend/tests/fixtures/scenario_execution/graph.json index 5a37dc263..edfc4627d 100644 --- a/backend/tests/fixtures/scenario_execution/graph.json +++ b/backend/tests/fixtures/scenario_execution/graph.json @@ -1,6 +1,8 @@ { "schema_version": 1, "compiler_version": "038.1.0", + "action_registry_version": "038.1.0", + "action_registry_hash": "fd369209425dd46219eb857bc9ddcb837b533a69f71d81b014207983fad129e9", "scenario_id": "fi-0080_verify-filters-metric-xlsx", "revision_hash": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "objective": { @@ -28,7 +30,7 @@ "logical_step_id": "s2_query_metric", "id": "step-uuid-2", "tool": "superset_api", - "action": "execute_query", + "action": "execute_metric", "title": "Query revenue metric", "timeout_ms": 30000 }, @@ -36,7 +38,7 @@ "logical_step_id": "s3_export_xlsx", "id": "step-uuid-3", "tool": "xlsx", - "action": "export", + "action": "parse_xlsx", "title": "Export XLSX", "timeout_ms": 60000 }, @@ -44,7 +46,7 @@ "logical_step_id": "s4_assert_revenue", "id": "step-uuid-4", "tool": "assertion", - "action": "compare_baseline", + "action": "compare_to_baseline", "title": "Assert revenue vs baseline", "timeout_ms": 15000 }, @@ -52,6 +54,7 @@ "logical_step_id": "s5_review_human", "id": "step-uuid-5", "tool": "human", + "action": "human_checkpoint", "title": "Human review of export", "timeout_ms": 0 }, @@ -59,7 +62,7 @@ "logical_step_id": "s6_screenshot", "id": "step-uuid-6", "tool": "screenshot", - "action": "capture", + "action": "capture_screenshot", "title": "Capture evidence screenshot", "timeout_ms": 30000 } diff --git a/backend/tests/services/dashboard_testing/registry/conftest.py b/backend/tests/services/dashboard_testing/registry/conftest.py index b9579c0eb..d853ee1f2 100644 --- a/backend/tests/services/dashboard_testing/registry/conftest.py +++ b/backend/tests/services/dashboard_testing/registry/conftest.py @@ -13,6 +13,7 @@ from datetime import datetime import json from pathlib import Path import pytest +from types import SimpleNamespace from sqlalchemy import create_engine, event from sqlalchemy.orm import Session, sessionmaker @@ -33,6 +34,32 @@ _CONSUME_COMMIT = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b" _TEST_SOURCE_RESPONSE_HASH = "abc123def456abc123def456abc123def456abc123def456abc123def456abcf" +# #region Test.ScenarioRegistry.Conftest.EnvironmentPolicy [C:2] [TYPE Function] [SEMANTICS test,scenario,execution,environment,prod,fixture] +# @BRIEF Provide only explicit server-owned environment records to 044 persisted fixtures. +# @TEST_FIXTURE preprod/prod/env-preprod-02/env-preprod-044 -> INLINE_CONFIG_MANAGER +# @TEST_INVARIANT ScenarioExecution.EnvironmentPolicy: fixture run bodies cannot classify an +# environment; only this hardcoded ConfigManager mapping may select PROD. +@pytest.fixture(autouse=True) +def configured_scenario_execution_environments(monkeypatch): + environments = { + "preprod": SimpleNamespace(stage="PREPROD", is_production=False), + "prod": SimpleNamespace(stage="PROD", is_production=True), + "env-preprod-02": SimpleNamespace(stage="PREPROD", is_production=False), + "env-preprod-044": SimpleNamespace(stage="PREPROD", is_production=False), + "env-preprod-manual-only": SimpleNamespace( + stage="PREPROD", is_production=False + ), + } + config_manager = SimpleNamespace( + get_environment=lambda environment_id: environments.get(environment_id) + ) + monkeypatch.setattr( + "src.dependencies.get_config_manager", lambda: config_manager + ) + return config_manager +# #endregion Test.ScenarioRegistry.Conftest.EnvironmentPolicy + + def _make_request_with_capture(agent_run_id: str, capture_artifact_id: str) -> CandidateRequest: return CandidateRequest.model_construct( environment_id="ss-preprod", diff --git a/backend/tests/services/dashboard_testing/registry/test_live_execution_binding.py b/backend/tests/services/dashboard_testing/registry/test_live_execution_binding.py new file mode 100644 index 000000000..318457dbf --- /dev/null +++ b/backend/tests/services/dashboard_testing/registry/test_live_execution_binding.py @@ -0,0 +1,567 @@ +# #region Test.ScenarioExecution.LiveBinding [C:3] [TYPE Module] [SEMANTICS test,scenario,execution,live-binding,superset] +# @BRIEF Verify persisted live-binding composition only authorizes exact 037 query execution. +# @RELATION BINDS_TO -> [ScenarioExecution.LiveBinding] +# @RELATION VERIFIES -> [ScenarioExecution.Runner.DefaultRegistry] +# @RELATION VERIFIES -> [ScenarioExecution.LiveBinding.Execute] +# @RELATION VERIFIES -> [ScenarioExecution.LiveBinding.Evidence] +# @RELATION VERIFIES -> [ScenarioExecution.Runner.Start] +# @RELATION VERIFIES -> [ScenarioExecution.LiveCompositionRoot] +# @RELATION VERIFIES -> [Core.ConfigModels.ScenarioLiveExecutionBinding] +# @TEST_CONTRACT: persisted LiveExecutionBinding + resolver -> authorized 037 Superset adapter outcome +# @TEST_FIXTURE: binding_044 + query_model_044 + chart_data_044 -> INLINE_JSON hardcoded identity and response +# @TEST_EDGE: missing_binding -> inconclusive/no client call; resolver_mismatch -> inconclusive/no client call; external_failure -> failed; external_timeout -> inconclusive +# @TEST_INVARIANT ScenarioExecution.LiveBinding: A live query executes only when a resolver returns the exact persisted binding and pinned model/principal fingerprints. -> VERIFIED_BY: missing_binding, resolver_mismatch, matching_binding, external_failure, external_timeout, start_persists_binding +# @TEST_INVARIANT ScenarioExecution.LiveBinding.Evidence: Exact raw 037 bytes produce the stored opaque ref and SHA-256; invalid evidence cannot pass. -> VERIFIED_BY: matching_binding, start_persists_binding +# @TEST_INVARIANT ScenarioExecution.Runner.Start: The validated immutable identity snapshot persists with its binding ref. -> VERIFIED_BY: start_persists_binding +# @TEST_INVARIANT Core.ConfigModels.ScenarioLiveExecutionBinding: An enabled trusted setting enters +# composition only through the exact LiveExecutionBinding identity allowlist; it +# cannot turn an environment label into runtime authority. -> VERIFIED_BY: +# lifespan_bootstrap_configured_binding_dispatches_queued_run +# @TEST_INVARIANT ScenarioExecution.LiveCompositionRoot: Startup-owned providers execute only for an +# exact binding; unavailable/mismatched identities make no external call. -> VERIFIED_BY: +# composition_root_default_registry_invokes_037, composition_root_mismatch_and_unavailable_fail_closed +# @TEST_INVARIANT ScenarioExecution.LiveCompositionRoot: Browser recovery requires its pinned safe +# checkpoint, PROD mutation is rejected, and Screenshot PASS carries provider-issued +# durable evidence. -> VERIFIED_BY: composition_root_browser_checkpoint_and_screenshot_evidence +# @TEST_INVARIANT ScenarioExecution.LiveCompositionRoot.Bootstrap: No configured Browser/Screenshot +# provider exists by default; both tools remain typed unavailable and make no live +# client call until server-side provider registration. -> VERIFIED_BY: +# lifespan_bootstrap_mismatch_and_unavailable_are_no_call +# @TEST_INVARIANT App.AppModule.LiveExecutionCompositionBootstrap: Trusted configuration registers +# exact 037 authority before queued dispatch; mismatch/unavailable paths make no I/O call. -> VERIFIED_BY: +# lifespan_bootstrap_configured_binding_dispatches_queued_run, +# lifespan_bootstrap_mismatch_and_unavailable_are_no_call +from __future__ import annotations + +import asyncio +from hashlib import sha256 +from types import SimpleNamespace +from unittest.mock import AsyncMock + +from src.core.config_models import ScenarioLiveExecutionBindingConfig +from src.core.superset_client._chart_data import ChartDataResponse +from src.core.utils.network import SupersetAPIError +from src.models.scenario_artifact import ScenarioArtifact +from src.models.scenario_run import ScenarioStepRun +from src.schemas.dashboard_testing import ( + ChartQueryModel, + DashboardQueryModel, + MetricDescriptor, + VizType, +) +from src.services.dashboard_testing.execution.live_adapter import LiveAdapterResult +from src.services.dashboard_testing.execution.live_binding import ( + LiveExecutionBinding, + ResolvedLiveExecutionBinding, + superset_adapter_from, +) +from src.services.dashboard_testing.execution.live_composition import LiveExecutionCompositionRoot +from src.services.dashboard_testing.execution.runner import ( + _advance_run, + _build_default_registry, + dispatch_queued_runs, + start_run, +) +from src.services.dashboard_testing.scenario.templates import ( + ACTION_REGISTRY_VERSION, + action_registry_fingerprint, + resolve_action_descriptor, +) + +_RAW_RESPONSE_044 = b'{"result":[{"data":{"revenue":4400}}],"query_id":"q-044"}' +_RAW_RESPONSE_SHA_044 = "f2c10974f67a9dd9177e48c1d26982797eb929f8b0f8f385c5ae3b25ff1d35b8" +_BINDING_044 = { + "binding_ref": "live-binding-044", + "environment_id": "env-preprod-044", + "dashboard_release_id": "release-044", + "release_fingerprint": "1" * 64, + "dashboard_id": 44, + "query_model_fingerprint": "sha256:model-044", + "execution_principal_fingerprint": "5079d4e113c33372e1d69c9dc7eece02c0a915c25a67bf550f214d4551d54113", + "rls_security_fingerprint": "3" * 64, + "browser_safe_checkpoint_ref": None, + "browser_action_binding_ref": None, + "evidence_owner_type": "scenario_run", + "evidence_ref_policy": "draft_storage_raw_response", +} +_STEP_044 = { + "logical_step_id": "superset-metric-044", + "scenario_run_id": "run-044", + "execution_principal_fingerprint": "5079d4e113c33372e1d69c9dc7eece02c0a915c25a67bf550f214d4551d54113", + "target_snapshot": {"environment_id": "env-preprod-044", "dashboard_release_id": "release-044"}, + "live_execution_binding_ref": "live-binding-044", + "live_execution_binding_snapshot": _BINDING_044, + "step_meta": { + "environment_id": "env-preprod-044", + "dashboard_id": 44, + "chart_id": 440, + "result_key": "revenue", + "normalized_filters": {"schema_version": 1, "filters": [], "filters_hash": "sha256:filters-044"}, + }, +} +_BROWSER_BINDING_044 = { + **_BINDING_044, + "browser_safe_checkpoint_ref": "checkpoint:044", + "browser_action_binding_ref": "browser-action:044", +} + + +def _superset_plan() -> dict: + descriptor = resolve_action_descriptor( + tool="superset_api", + action="execute_metric", + registry_version=ACTION_REGISTRY_VERSION, + registry_hash=action_registry_fingerprint(), + ).snapshot() + return { + "action_registry_version": ACTION_REGISTRY_VERSION, + "action_registry_hash": action_registry_fingerprint(), + "topological_order": ["superset-metric-044"], + "dependencies": [], + "steps": [_STEP_044["step_meta"] | { + "logical_step_id": "superset-metric-044", + "tool": "superset_api", + "action": "execute_metric", + "action_descriptor": descriptor, + }], + } + + +def _resolve(registry, tool: str, action: str): + return registry.resolve(resolve_action_descriptor( + tool=tool, + action=action, + registry_version=ACTION_REGISTRY_VERSION, + registry_hash=action_registry_fingerprint(), + ).snapshot()) + + +# #region Test.ScenarioExecution.LiveBinding.Fakes [C:1] [TYPE Class] +class _EvidenceStore: + def __init__(self) -> None: + self.stored: list[tuple[str, str, bytes]] = [] + + def store(self, run_id: str, digest: str, data: bytes) -> str: + self.stored.append((run_id, digest, data)) + return f"draft:{run_id}:{digest}" + + +class _Resolver: + def __init__(self, resolved: ResolvedLiveExecutionBinding | None) -> None: + self.resolved = resolved + self.references: list[str] = [] + + def resolve(self, binding_ref: str) -> ResolvedLiveExecutionBinding | None: + self.references.append(binding_ref) + return self.resolved +# #endregion Test.ScenarioExecution.LiveBinding.Fakes + + +# #region Test.ScenarioExecution.LiveBinding.QueryModel [C:1] [TYPE Function] +def _query_model() -> DashboardQueryModel: + return DashboardQueryModel( + environment_id="env-preprod-044", + dashboard_id=44, + title="Revenue 044", + query_model_fingerprint="sha256:model-044", + charts=[ + ChartQueryModel( + chart_id=440, + slice_name="Revenue", + viz_type=VizType.TABLE, + dataset_id=441, + dataset_name="sales", + metrics=[MetricDescriptor(metric_name="revenue", label="Revenue", expression_type="SIMPLE")], + ) + ], + ) +# #endregion Test.ScenarioExecution.LiveBinding.QueryModel + + +# #region Test.ScenarioExecution.LiveBinding.Resolved [C:1] [TYPE Function] +def _resolved(binding: LiveExecutionBinding, client: AsyncMock, evidence: _EvidenceStore) -> ResolvedLiveExecutionBinding: + return ResolvedLiveExecutionBinding( + binding=binding, + superset_client=client, + query_model=_query_model(), + evidence_storage=evidence, + run_async=asyncio.run, + ) +# #endregion Test.ScenarioExecution.LiveBinding.Resolved + + +# #region Test.ScenarioExecution.LiveBinding.CompositionRoot [C:1] [TYPE Function] +def _install_composition_root(monkeypatch) -> LiveExecutionCompositionRoot: + import src.dependencies as dependencies + + root = LiveExecutionCompositionRoot() + monkeypatch.setattr(dependencies, "live_execution_composition_root", root) + return root +# #endregion Test.ScenarioExecution.LiveBinding.CompositionRoot + + +# #region Test.ScenarioExecution.LiveBinding.Missing [C:2] [TYPE Function] +# @BRIEF Missing persisted binding fails closed without querying the external client. +def test_missing_binding_is_inconclusive_and_does_not_call_client(): + client = AsyncMock() + resolver = _Resolver(None) + outcome = superset_adapter_from(resolver)({**_STEP_044, "live_execution_binding_ref": None, "live_execution_binding_snapshot": None}, {}) + + assert outcome.status == "inconclusive" + assert outcome.reason_code == "SUPERSET_BINDING_MISSING" + assert resolver.references == [] + client.execute_chart_data_raw.assert_not_awaited() +# #endregion Test.ScenarioExecution.LiveBinding.Missing + + +# #region Test.ScenarioExecution.LiveBinding.Mismatch [C:2] [TYPE Function] +# @BRIEF Resolver identity mismatch fails closed before the 037 client receives a query. +def test_resolver_mismatch_is_inconclusive_and_does_not_call_client(): + client = AsyncMock() + expected = LiveExecutionBinding.from_snapshot(_BINDING_044) + mismatched = expected.with_query_model_fingerprint("sha256:other-model-044") + resolver = _Resolver(_resolved(mismatched, client, _EvidenceStore())) + + outcome = superset_adapter_from(resolver)(_STEP_044, {}) + + assert outcome.status == "inconclusive" + assert outcome.reason_code == "SUPERSET_BINDING_MISMATCH" + assert resolver.references == ["live-binding-044"] + client.execute_chart_data_raw.assert_not_awaited() +# #endregion Test.ScenarioExecution.LiveBinding.Mismatch + + +# #region Test.ScenarioExecution.LiveBinding.Success [C:2] [TYPE Function] +# @BRIEF Exact resolver identity invokes the existing 037 envelope and forwards verified evidence. +def test_matching_binding_invokes_query_executor_and_forwards_real_evidence_digest(): + client = AsyncMock() + client.execute_chart_data_raw.return_value = ChartDataResponse( + parsed={"result": [{"data": {"revenue": 4400}}], "query_id": "q-044"}, + raw_bytes=_RAW_RESPONSE_044, + source_response_hash=_RAW_RESPONSE_SHA_044, + ) + binding = LiveExecutionBinding.from_snapshot(_BINDING_044) + evidence = _EvidenceStore() + resolver = _Resolver(_resolved(binding, client, evidence)) + + outcome = superset_adapter_from(resolver)(_STEP_044, {}) + + assert sha256(_RAW_RESPONSE_044).hexdigest() == _RAW_RESPONSE_SHA_044 + assert outcome.status == "passed" + assert outcome.reason_code == "SUPERSET_QUERY_EXECUTED" + assert outcome.details["source_response_hash"] == _RAW_RESPONSE_SHA_044 + assert outcome.artifact_refs == [f"draft:run-044:{_RAW_RESPONSE_SHA_044}"] + assert outcome.artifact_digests == {f"draft:run-044:{_RAW_RESPONSE_SHA_044}": _RAW_RESPONSE_SHA_044} + assert evidence.stored == [("run-044", _RAW_RESPONSE_SHA_044, _RAW_RESPONSE_044)] + client.execute_chart_data_raw.assert_awaited_once() +# #endregion Test.ScenarioExecution.LiveBinding.Success + + +# #region Test.ScenarioExecution.LiveBinding.Failure [C:2] [TYPE Function] +# @BRIEF A typed 037 API failure remains non-pass after an exact resolver match. +def test_matching_binding_maps_query_failure_to_non_pass(): + client = AsyncMock() + client.execute_chart_data_raw.side_effect = SupersetAPIError("preprod denied") + binding = LiveExecutionBinding.from_snapshot(_BINDING_044) + resolver = _Resolver(_resolved(binding, client, _EvidenceStore())) + + outcome = _resolve(_build_default_registry(resolver), "superset_api", "execute_metric")(_STEP_044, {}) + + assert outcome["status"] == "failed" + assert outcome["error_code"] == "SUPERSET_QUERY_FAILED" + client.execute_chart_data_raw.assert_awaited_once() +# #endregion Test.ScenarioExecution.LiveBinding.Failure + + +# #region Test.ScenarioExecution.LiveBinding.Timeout [C:2] [TYPE Function] +# @BRIEF A timeout at the authorized external client remains typed inconclusive. +def test_matching_binding_maps_query_timeout_to_typed_inconclusive(): + client = AsyncMock() + client.execute_chart_data_raw.side_effect = TimeoutError("chart-data timed out") + binding = LiveExecutionBinding.from_snapshot(_BINDING_044) + resolver = _Resolver(_resolved(binding, client, _EvidenceStore())) + + outcome = _resolve(_build_default_registry(resolver), "superset_api", "execute_metric")(_STEP_044, {}) + + assert outcome["status"] == "inconclusive" + assert outcome["error_code"] == "SUPERSET_ADAPTER_TIMEOUT" + client.execute_chart_data_raw.assert_awaited_once() +# #endregion Test.ScenarioExecution.LiveBinding.Timeout + + +# #region Test.ScenarioExecution.LiveBinding.Persistence [C:2] [TYPE Function] +# @BRIEF Start persists the identity snapshot and walker forwards it to the authorized 037 adapter. +def test_start_persists_binding_and_walker_registers_exact_evidence(seeded_registry): + client = AsyncMock() + client.execute_chart_data_raw.return_value = ChartDataResponse( + parsed={"result": [{"data": {"revenue": 4400}}], "query_id": "q-044"}, + raw_bytes=_RAW_RESPONSE_044, + source_response_hash=_RAW_RESPONSE_SHA_044, + ) + binding = LiveExecutionBinding.from_snapshot(_BINDING_044) + resolver = _Resolver(_resolved(binding, client, _EvidenceStore())) + run = start_run( + seeded_registry, + "11111111-1111-4111-8111-111111111111", + "22222222-2222-4222-8222-222222222222", + {}, + "env-preprod-044", + actor="binding-actor-044", + idempotency_key="live-binding-start-044", + dashboard_release_id="release-044", + live_execution_binding=binding, + auto_advance=False, + ) + assert run.live_execution_binding_ref == "live-binding-044" + assert run.live_execution_binding_snapshot == _BINDING_044 + assert run.target_snapshot["dashboard_release_id"] == "release-044" + run.runner_plan = _superset_plan() + seeded_registry.flush() + + result = _advance_run( + seeded_registry, + run, + _build_default_registry(resolver), + worker_id="binding-actor-044", + ) + + step = seeded_registry.query(ScenarioStepRun).filter_by(run_id=run.id).one() + artifact = seeded_registry.query(ScenarioArtifact).filter_by(owner_id=run.id).one() + assert result["status"] == "passed" + assert step.status == "passed" + assert artifact.content_ref == f"draft:{run.id}:{_RAW_RESPONSE_SHA_044}" + assert artifact.sha256 == _RAW_RESPONSE_SHA_044 + client.execute_chart_data_raw.assert_awaited_once() +# #endregion Test.ScenarioExecution.LiveBinding.Persistence + + +# #region Test.ScenarioExecution.LiveBinding.CompositionStartup [C:2] [TYPE Function] +# @BRIEF Application dependency exposes the one startup-owned root used by default runner composition. +def test_composition_root_is_registered_through_application_dependency(monkeypatch): + import src.dependencies as dependencies + + root = _install_composition_root(monkeypatch) + + assert dependencies.get_live_execution_composition_root() is root +# #endregion Test.ScenarioExecution.LiveBinding.CompositionStartup + + +# #region Test.ScenarioExecution.LiveBinding.Composition037 [C:2] [TYPE Function] +# @BRIEF An application composition registration is discovered by the default runner registry and invokes 037. +def test_composition_root_default_registry_invokes_037(monkeypatch): + client = AsyncMock() + client.execute_chart_data_raw.return_value = ChartDataResponse( + parsed={"result": [{"data": {"revenue": 4400}}], "query_id": "q-044-root"}, + raw_bytes=_RAW_RESPONSE_044, + source_response_hash=_RAW_RESPONSE_SHA_044, + ) + binding = LiveExecutionBinding.from_snapshot(_BINDING_044) + evidence = _EvidenceStore() + root = _install_composition_root(monkeypatch) + root.register_superset(_resolved(binding, client, evidence)) + + outcome = _resolve(_build_default_registry(), "superset_api", "execute_metric")(_STEP_044, {}) + + assert outcome["status"] == "passed" + assert outcome["artifact_refs"] == [f"draft:run-044:{_RAW_RESPONSE_SHA_044}"] + assert outcome["step_outcome"]["artifact_digests"] == { + f"draft:run-044:{_RAW_RESPONSE_SHA_044}": _RAW_RESPONSE_SHA_044 + } + client.execute_chart_data_raw.assert_awaited_once() +# #endregion Test.ScenarioExecution.LiveBinding.Composition037 + + +# #region Test.ScenarioExecution.LiveBinding.CompositionFailClosed [C:2] [TYPE Function] +# @BRIEF Configured-but-unavailable and wrong-bound identity paths make no 037 client call. +def test_composition_root_mismatch_and_unavailable_fail_closed(monkeypatch): + client = AsyncMock() + binding = LiveExecutionBinding.from_snapshot(_BINDING_044) + root = _install_composition_root(monkeypatch) + root.register_superset(_resolved(binding, client, _EvidenceStore())) + wrong_snapshot = {**_BINDING_044, "release_fingerprint": "9" * 64} + + mismatch = _resolve(_build_default_registry(), "superset_api", "execute_metric")( + {**_STEP_044, "live_execution_binding_snapshot": wrong_snapshot}, {} + ) + root = _install_composition_root(monkeypatch) + unavailable_binding = LiveExecutionBinding.from_snapshot( + {**_BROWSER_BINDING_044, "binding_ref": "configured-other-044"} + ) + root.register_screenshot( + unavailable_binding, + lambda _context: LiveAdapterResult(status="passed", reason_code="unused"), + ) + unavailable = _resolve(_build_default_registry(), "screenshot", "capture_screenshot")( + {**_STEP_044, "live_execution_binding_snapshot": _BROWSER_BINDING_044}, {} + ) + + assert mismatch["status"] == "inconclusive" + assert mismatch["error_code"] == "SUPERSET_BINDING_MISMATCH" + assert unavailable["status"] == "inconclusive" + assert unavailable["error_code"] == "SCREENSHOT_BINDING_UNAVAILABLE" + client.execute_chart_data_raw.assert_not_awaited() +# #endregion Test.ScenarioExecution.LiveBinding.CompositionFailClosed + + +# #region Test.ScenarioExecution.LiveBinding.CompositionBrowserScreenshot [C:2] [TYPE Function] +# @BRIEF Browser recovery blocks before provider I/O without its checkpoint; a registered capture returns durable evidence. +def test_composition_root_browser_checkpoint_and_screenshot_evidence(monkeypatch): + binding = LiveExecutionBinding.from_snapshot(_BROWSER_BINDING_044) + root = _install_composition_root(monkeypatch) + browser_calls: list[str] = [] + root.register_browser( + binding, + lambda context: browser_calls.append(context.binding.binding_ref) or LiveAdapterResult(status="passed", reason_code="BROWSER_ACTION_EXECUTED"), + ) + root.register_screenshot( + binding, + lambda context: LiveAdapterResult( + status="passed", + reason_code="SCREENSHOT_CAPTURED", + details={"sha256": _RAW_RESPONSE_SHA_044}, + output_refs=[f"draft:{context.step['scenario_run_id']}:{_RAW_RESPONSE_SHA_044}"], + artifact_refs=[f"draft:{context.step['scenario_run_id']}:{_RAW_RESPONSE_SHA_044}"], + artifact_digests={f"draft:{context.step['scenario_run_id']}:{_RAW_RESPONSE_SHA_044}": _RAW_RESPONSE_SHA_044}, + ), + ) + browser_step = { + **_STEP_044, + "live_execution_binding_ref": binding.binding_ref, + "live_execution_binding_snapshot": binding.snapshot(), + "step_meta": {"recovery_mode": True, "browser_safe_checkpoint_ref": "wrong-checkpoint"}, + } + screenshot_step = {**_STEP_044, "live_execution_binding_snapshot": binding.snapshot()} + + browser_outcome = _resolve(_build_default_registry(), "browser", "open_dashboard")(browser_step, {}) + screenshot_outcome = _resolve(_build_default_registry(), "screenshot", "capture_screenshot")(screenshot_step, {}) + prod_mutation_step = { + **browser_step, + "step_meta": { + "mutation_contract": {"safe_test_fixture_id": "fixture-044"}, + "action_registry_fingerprint": "registry-044", + }, + "target_snapshot": { + "environment_id": "env-preprod-044", + "dashboard_release_id": "release-044", + "environment_class": "PROD", + }, + } + prod_mutation = _resolve(_build_default_registry(), "browser", "row_edit")(prod_mutation_step, {}) + + assert browser_outcome["status"] == "inconclusive" + assert browser_outcome["error_code"] == "BROWSER_SAFE_CHECKPOINT_REQUIRED" + assert browser_calls == [] + assert screenshot_outcome["status"] == "passed" + assert screenshot_outcome["artifact_refs"] == [f"draft:run-044:{_RAW_RESPONSE_SHA_044}"] + assert prod_mutation["status"] == "failed" + assert prod_mutation["error_code"] == "BROWSER_MUTATION_PROD_REJECTED" + assert browser_calls == [] +# #endregion Test.ScenarioExecution.LiveBinding.CompositionBrowserScreenshot + + +# #region Test.ScenarioExecution.LiveBinding.LifespanBootstrap [C:3] [TYPE Function] +# @BRIEF App startup bootstrap registers trusted config before the default queued dispatcher reaches 037. +def test_lifespan_bootstrap_configured_binding_dispatches_queued_run(seeded_registry, monkeypatch): + from src import app as application + from src.services.dashboard_testing.execution import live_composition + + client = AsyncMock() + client.execute_chart_data_raw.return_value = ChartDataResponse( + parsed={"result": [{"data": {"revenue": 4400}}], "query_id": "q-044-lifespan"}, + raw_bytes=_RAW_RESPONSE_044, + source_response_hash=_RAW_RESPONSE_SHA_044, + ) + binding = LiveExecutionBinding.from_snapshot(_BINDING_044) + evidence = _EvidenceStore() + configuration = SimpleNamespace( + get_config=lambda: SimpleNamespace(settings=SimpleNamespace( + scenario_live_execution_bindings=[ScenarioLiveExecutionBindingConfig( + enabled=True, + binding_snapshot=binding.snapshot(), + query_model_snapshot=_query_model().model_dump(mode="json"), + )], + )), + get_environment=lambda environment_id: ( + SimpleNamespace(id=environment_id) if environment_id == "env-preprod-044" else None + ), + ) + root = _install_composition_root(monkeypatch) + monkeypatch.setattr(application, "get_config_manager", lambda: configuration) + monkeypatch.setattr(application, "get_live_execution_composition_root", lambda: root) + monkeypatch.setattr(application, "get_async_job_runner", lambda: SimpleNamespace(run=asyncio.run)) + monkeypatch.setattr(live_composition, "SupersetClient", lambda _environment: client) + monkeypatch.setattr(live_composition, "get_draft_storage", lambda: evidence) + + assert application.initialize_live_execution_composition() == 1 + run = start_run( + seeded_registry, + "11111111-1111-4111-8111-111111111111", + "22222222-2222-4222-8222-222222222222", + {}, + "env-preprod-044", + actor="binding-actor-044", + idempotency_key="lifespan-bootstrap-044", + dashboard_release_id="release-044", + live_execution_binding=binding, + ) + run.runner_plan = _superset_plan() + seeded_registry.flush() + + outcomes = dispatch_queued_runs(seeded_registry, worker_id="lifespan-044") + artifact = seeded_registry.query(ScenarioArtifact).filter_by(owner_id=run.id).one() + + assert outcomes[0]["status"] == "passed" + assert artifact.content_ref == f"draft:{run.id}:{_RAW_RESPONSE_SHA_044}" + assert artifact.sha256 == _RAW_RESPONSE_SHA_044 + client.execute_chart_data_raw.assert_awaited_once() +# #endregion Test.ScenarioExecution.LiveBinding.LifespanBootstrap + + +# #region Test.ScenarioExecution.LiveBinding.LifespanFailClosed [C:3] [TYPE Function] +# @BRIEF Bootstrap retains configured unavailable and mismatched bindings as no-I/O typed outcomes. +def test_lifespan_bootstrap_mismatch_and_unavailable_are_no_call(monkeypatch): + from src import app as application + from src.services.dashboard_testing.execution import live_composition + + binding = LiveExecutionBinding.from_snapshot(_BINDING_044) + client = AsyncMock() + root = _install_composition_root(monkeypatch) + configuration = SimpleNamespace( + get_config=lambda: SimpleNamespace(settings=SimpleNamespace( + scenario_live_execution_bindings=[ScenarioLiveExecutionBindingConfig( + enabled=True, + binding_snapshot=binding.snapshot(), + query_model_snapshot=_query_model().model_dump(mode="json"), + )], + )), + get_environment=lambda _environment_id: SimpleNamespace(id="env-preprod-044"), + ) + monkeypatch.setattr(application, "get_config_manager", lambda: configuration) + monkeypatch.setattr(application, "get_live_execution_composition_root", lambda: root) + monkeypatch.setattr(application, "get_async_job_runner", lambda: SimpleNamespace(run=asyncio.run)) + monkeypatch.setattr(live_composition, "SupersetClient", lambda _environment: client) + monkeypatch.setattr(live_composition, "get_draft_storage", _EvidenceStore) + + assert application.initialize_live_execution_composition() == 1 + mismatch = _resolve(_build_default_registry(), "superset_api", "execute_metric")( + {**_STEP_044, "live_execution_binding_snapshot": {**_BINDING_044, "rls_security_fingerprint": "7" * 64}}, + {}, + ) + unavailable_binding = {**_BINDING_044, "binding_ref": "unavailable-044"} + unavailable = _resolve(_build_default_registry(), "superset_api", "execute_metric")( + {**_STEP_044, "live_execution_binding_ref": "unavailable-044", "live_execution_binding_snapshot": unavailable_binding}, + {}, + ) + browser_unavailable = _resolve(_build_default_registry(), "browser", "open_dashboard")(_STEP_044, {}) + screenshot_unavailable = _resolve(_build_default_registry(), "screenshot", "capture_screenshot")(_STEP_044, {}) + + assert mismatch["status"] == "inconclusive" + assert mismatch["error_code"] == "SUPERSET_BINDING_MISMATCH" + assert unavailable["status"] == "inconclusive" + assert unavailable["error_code"] == "SUPERSET_BINDING_UNAVAILABLE" + assert browser_unavailable["error_code"] == "BROWSER_BINDING_UNAVAILABLE" + assert screenshot_unavailable["error_code"] == "SCREENSHOT_BINDING_UNAVAILABLE" + client.execute_chart_data_raw.assert_not_awaited() +# #endregion Test.ScenarioExecution.LiveBinding.LifespanFailClosed + +# #endregion Test.ScenarioExecution.LiveBinding diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_auto_queue.py b/backend/tests/services/dashboard_testing/registry/test_scenario_auto_queue.py new file mode 100644 index 000000000..0046a73ad --- /dev/null +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_auto_queue.py @@ -0,0 +1,30 @@ +# #region Test.ScenarioExecution.AutoQueue [C:3] [TYPE Module] [SEMANTICS test,scenario,execution,autoqueue,failed,investigation] +# @defgroup ScenarioExecution Auto-queue failed runs into investigation queue. +# @LAYER Test +# @RELATION BINDS_TO -> [ScenarioAnalytics.Investigation.AutoQueue] +# @TEST_EDGE failed_run_same_env -> duplicate fingerprint increments count; passed_run_no_queue -> no item +from __future__ import annotations + +from src.models.scenario_run import ScenarioRun +from src.services.dashboard_testing.analytics.investigation import auto_queue_failed_run + + +def test_failed_run_auto_queues_investigation(seeded_execution): + run = seeded_execution.query(ScenarioRun).filter(ScenarioRun.scenario_id == "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1").first() + run.status = "failed" + run.phase = "terminal" + run.finished_at = __import__("datetime").datetime.now(__import__("datetime").UTC) + seeded_execution.flush() + auto_queue_failed_run( + seeded_execution, + scenario_id=run.scenario_id, + run_id=run.id, + environment_id=run.environment_id, + ) + seeded_execution.commit() + from src.models.scenario_investigation import InvestigationQueueItem + item = seeded_execution.query(InvestigationQueueItem).filter(InvestigationQueueItem.scenario_id == run.scenario_id).first() + assert item is not None + assert item.fingerprint == f"{run.scenario_id}:{run.environment_id}" + assert item.run_id == run.id +# #endregion Test.ScenarioExecution.AutoQueue diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_automation_trigger.py b/backend/tests/services/dashboard_testing/registry/test_scenario_automation_trigger.py index 3c23b1535..4a3d95fc8 100644 --- a/backend/tests/services/dashboard_testing/registry/test_scenario_automation_trigger.py +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_automation_trigger.py @@ -1,17 +1,26 @@ # #region Test.ScenarioAutomation.Trigger [C:3] [TYPE Module] [SEMANTICS test,scenario,automation,trigger,event] # @BRIEF Verify 046 trigger semantics (T003/T005): typed events map to pinned run # candidates, non-matching/disabled rules are skipped, and policy gates -# (capacity / PROD / dedup) reject candidates before they are returned. +# (capacity / dedup) reject candidates before they are returned; 044 resolves PROD from +# ConfigManager at the durable start boundary. # @RELATION BINDS_TO -> [ScenarioAutomation.Trigger] +# @RELATION VERIFIES -> [ScenarioAutomation.Trigger.Dispatch] +# @RELATION VERIFIES -> [ScenarioExecution.EnvironmentPolicy.Resolve] # @TEST_EDGE: event->run -> matched rule yields pinned scenario/revision/env candidate # @TEST_EDGE: release_create->run -> release_create event maps to a run candidate # @TEST_EDGE: ETL->run -> etl_completed event maps to a run candidate # @TEST_EDGE: disabled_rule_skipped -> rule with enabled=False produces no candidate # @TEST_EDGE: event_type_mismatch_skipped -> non-matching event_type produces no candidate # @TEST_EDGE: capacity_blocked -> active run in env rejects candidate via max_concurrent_per_env -# @TEST_EDGE: prod_gate_blocked -> PROD-class rule never auto-starts without approval +# @TEST_EDGE: stale_rule_environment_class -> rule metadata cannot classify PROD # @TEST_EDGE: dedup_blocked -> recent duplicate fingerprint rejects candidate -from src.services.dashboard_testing.automation.trigger import handle_trigger_event +# @TEST_INVARIANT ScenarioAutomation.Trigger: A real event source is passed to 044 before +# ScenarioRun creation; it is never post-create provenance mutation. +# @TEST_INVARIANT ScenarioExecution.EnvironmentPolicy: Event/rule data carries target identity, +# never PROD authority; 044 resolves the configured class at its common durable +# start boundary. -> VERIFIED_BY: test_trigger_rule_environment_class_does_not_decide_prod_authority, +# test_dispatch_trigger_event_calls_run_boundary_with_pinned_current_revision +from src.services.dashboard_testing.automation.trigger import dispatch_trigger_event, handle_trigger_event # #region Test.ScenarioAutomation.Trigger.Mapping [C:2] [TYPE Function] [SEMANTICS test,scenario,automation,trigger] @@ -34,7 +43,7 @@ def test_trigger_event_maps_matched_rule_to_pinned_candidate(): assert candidate["environment_id"] == "env-preprod-01" assert candidate["trigger_source"] == "release_create" assert candidate["dedup_fingerprint"] == "rel-42" - assert candidate["is_prod"] is False + assert "is_prod" not in candidate def test_trigger_event_etl_completed_maps_candidate_with_current_revision(): @@ -118,7 +127,7 @@ def test_trigger_blocks_candidate_at_capacity(): assert handle_trigger_event(event, [rule], active, {"max_concurrent_per_env": 1}) == [] -def test_trigger_blocks_prod_rule_without_approval(): +def test_trigger_rule_environment_class_does_not_decide_prod_authority(): event = {"type": "release_create", "fingerprint": "rel-77"} rule = { "enabled": True, @@ -129,7 +138,8 @@ def test_trigger_blocks_prod_rule_without_approval(): "environment_class": "PROD", } candidates = handle_trigger_event(event, [rule], [], {"max_concurrent_per_env": 1}) - assert candidates == [] # PROD candidates require the approval path; never auto-started + assert len(candidates) == 1 + assert "is_prod" not in candidates[0] def test_trigger_blocks_duplicate_fingerprint_in_window(): @@ -143,5 +153,36 @@ def test_trigger_blocks_duplicate_fingerprint_in_window(): } recent = [{"environment_id": "env-preprod-01", "status": "passed", "dedup_fingerprint": "rel-42"}] assert handle_trigger_event(event, [rule], recent, {"max_concurrent_per_env": 5, "dedup_window_seconds": 0}) == [] + + +def test_dispatch_trigger_event_calls_run_boundary_with_pinned_current_revision(seeded_registry): + from src.models.scenario_automation import ScenarioTriggerRule + from src.models.scenario_registry import ScenarioRegistryEntry + + entry = seeded_registry.query(ScenarioRegistryEntry).filter_by( + scenario_id="11111111-1111-4111-8111-111111111111" + ).one() + entry.current_revision_id = "22222222-2222-4222-8222-222222222222" + + seeded_registry.add(ScenarioTriggerRule( + id="trigger-rule-1", + trigger="release_created", + scenario_id="11111111-1111-4111-8111-111111111111", + revision_policy="current", + environment_id="preprod", + enabled=True, + )) + calls = [] + + def start_stub(_db, **kwargs): + calls.append(kwargs) + return type("Run", (), {"id": "created-run", "trigger_source": "manual"})() + + created = dispatch_trigger_event(seeded_registry, {"type": "release_created", "fingerprint": "release-1"}, start_run_fn=start_stub) + assert created == ["created-run"] + assert calls[0]["revision_id"] == "22222222-2222-4222-8222-222222222222" + assert calls[0]["params"]["automation_fingerprint"] == "release-1" + assert calls[0]["trigger_source"] == "release_created" + assert calls[0]["config_manager"].get_environment("preprod").stage == "PREPROD" # #endregion Test.ScenarioAutomation.Trigger.Policy # #endregion Test.ScenarioAutomation.Trigger diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_cancel_timeout.py b/backend/tests/services/dashboard_testing/registry/test_scenario_cancel_timeout.py new file mode 100644 index 000000000..0cf280950 --- /dev/null +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_cancel_timeout.py @@ -0,0 +1,437 @@ +# #region Test.ScenarioExecution.CancelTimeout [C:5] [TYPE Module] [SEMANTICS test,scenario,execution,cancel,timeout,lease,artifact,signal] +# @BRIEF Prove real persisted cancel/drain and timeout transitions leave no active work stranded. +# @RELATION BINDS_TO -> [ScenarioExecution.Lifecycle.Cancel] +# @RELATION BINDS_TO -> [ScenarioExecution.Lifecycle.Timeout] +# @RELATION BINDS_TO -> [ScenarioExecution.Lifecycle.TimeoutClosure] +# @RELATION BINDS_TO -> [ScenarioExecution.Runner.Walker] +# @TEST_CONTRACT: persisted running step + controlled external adapter callback -> terminal closure +# @TEST_FIXTURE: cancel_timeout_044 -> INLINE_JSON hardcoded DAG, evidence refs, and digests +# @TEST_INVARIANT ScenarioExecution.Lifecycle.Cancel: A cancelled terminal has no queued/running +# step, active lease, or active evidence projection; immutable rows remain audit +# history. -> VERIFIED_BY: test_cancel_before_dispatch_retires_active_projection, +# test_cancel_during_claimed_adapter_drains_then_terminalizes, +# test_cancel_expires_prod_gate_and_cancels_human_checkpoint, +# test_cancel_deadline_finalizes_unresponsive_claim +# @TEST_INVARIANT ScenarioExecution.Lifecycle.Timeout: A timeout during claimed adapter I/O wins +# over returned payload, blocks descendants, expires leases, retires partial +# evidence, and emits exactly one typed inconclusive terminal signal. -> VERIFIED_BY: +# test_timeout_during_claimed_adapter_terminalizes_without_stranded_state, +# test_timeout_materializes_lazy_descendant_and_walker_never_dispatches +# @TEST_INVARIANT ScenarioExecution.Lifecycle.TimeoutClosure: Each descendant absent from the +# persisted step rows is materialized from the pinned plan as blocked before a +# later walker observes the terminal run; its executor is never dispatched. -> VERIFIED_BY: +# test_timeout_materializes_lazy_descendant_and_walker_never_dispatches +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +import pytest + +from src.models.scenario_approval import ActionApprovalGate +from src.models.scenario_artifact import ScenarioArtifact +from src.models.scenario_checkpoint import HumanCheckpoint +from src.models.scenario_investigation import InvestigationQueueItem +from src.models.scenario_run import ScenarioRun, ScenarioStepRun +from src.models.scenario_worker import ScenarioStepLease +from src.services.dashboard_testing.execution.approval import decide_approval_gate +from src.services.dashboard_testing.execution.artifacts import register_artifact +from src.services.dashboard_testing.execution.executor_registry import ScenarioExecutorRegistry +from src.services.dashboard_testing.execution.lifecycle import ( + apply_step_timeout, + cancel_run, + decide_checkpoint, + finalize_expired_cancellations, +) +from src.services.dashboard_testing.execution.result import build_result +from src.services.dashboard_testing.execution.runner import _advance_run +from src.services.dashboard_testing.execution.worker import claim_step +from src.services.dashboard_testing.scenario.templates import ( + ACTION_REGISTRY_VERSION, + action_registry_fingerprint, + resolve_action_descriptor, +) + +_SCENARIO_ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1" +_REVISION_ID = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1" +_CANCEL_RUN = "80440000-0000-4000-8000-000000000004" +_TIMEOUT_RUN = "80440000-0000-4000-8000-000000000005" +_CANCEL_SHA = "c" * 64 +_TIMEOUT_SHA = "d" * 64 + + +def _valid_plan(step_ids: list[str], dependencies: list[dict]) -> dict: + descriptor = resolve_action_descriptor( + tool="assertion", action="structural_assert", registry_version=ACTION_REGISTRY_VERSION, + registry_hash=action_registry_fingerprint(), + ).snapshot() + return { + "action_registry_version": ACTION_REGISTRY_VERSION, + "action_registry_hash": action_registry_fingerprint(), + "topological_order": step_ids, + "dependencies": dependencies, + "steps": [{ + "logical_step_id": step_id, "tool": "assertion", "action": "structural_assert", + "action_descriptor": descriptor, + } for step_id in step_ids], + } + + +def _run(run_id: str, plan: dict) -> ScenarioRun: + return ScenarioRun( + id=run_id, + scenario_id=_SCENARIO_ID, + scenario_revision_id=_REVISION_ID, + scenario_content_hash="d" * 64, + environment_id="env-cancel-timeout-044", + status="queued", + phase="executing", + parameter_bindings={"fixture": "cancel-timeout-044"}, + target_snapshot={"environment_id": "env-cancel-timeout-044"}, + trigger_source="manual", + idempotency_key=run_id, + runner_plan=plan, + execution_principal_fingerprint="e" * 64, + ) + + +def _expired(lease: ScenarioStepLease) -> bool: + expires_at = lease.expires_at + expires_at = expires_at.replace(tzinfo=UTC) if expires_at.tzinfo is None else expires_at.astimezone(UTC) + return expires_at <= datetime.now(UTC) + + +# #region Test.ScenarioExecution.CancelTimeout.BeforeDispatch [C:4] [TYPE Function] +# @BRIEF Cancelling queued work terminalizes immediately and retires its current evidence projection. +def test_cancel_before_dispatch_retires_active_projection(seeded_execution): + run = _run(_CANCEL_RUN, {"topological_order": ["queued-cancel"], "dependencies": [], "steps": []}) + step = ScenarioStepRun( + run_id=run.id, + logical_step_id="queued-cancel", + step_position=0, + attempt=1, + status="queued", + artifact_refs=["evidence:queued-cancel"], + ) + seeded_execution.add_all([run, step]) + seeded_execution.flush() + artifact = register_artifact( + seeded_execution, + owner_type="scenario_run", + owner_id=run.id, + kind="evidence", + name="queued-cancel-evidence", + content_ref="evidence:queued-cancel", + sha256=_CANCEL_SHA, + logical_step_id=step.logical_step_id, + attempt=1, + ) + + cancelled = cancel_run(seeded_execution, run.id) + assert cancelled.status == "cancelled" + assert cancelled.phase == "terminal" + assert step.status == "skipped" + assert step.artifact_refs == [] + assert step.outputs["terminal_history"][0]["artifact_refs"] == ["evidence:queued-cancel"] + assert artifact.is_active is False + assert artifact.invalidated_at is not None + assert build_result(cancelled, [step])["status"] == "cancelled" + assert cancel_run(seeded_execution, run.id).id == run.id +# #endregion Test.ScenarioExecution.CancelTimeout.BeforeDispatch + + +# #region Test.ScenarioExecution.CancelTimeout.Deadline [C:5] [TYPE Function] +# @BRIEF A persisted deadline closes a genuinely unresponsive claimed step without relying on adapter return. +# @TEST_INVARIANT ScenarioExecution.Lifecycle.CancelFinalizer: A bounded drain cannot strand a +# running lease or active evidence after the persisted deadline. -> VERIFIED_BY: +# test_cancel_deadline_finalizes_unresponsive_claim +def test_cancel_deadline_finalizes_unresponsive_claim(seeded_execution): + run = _run( + "80440000-0000-4000-8000-000000000008", + {"topological_order": ["unresponsive-044"], "dependencies": [], "steps": []}, + ) + step = ScenarioStepRun( + run_id=run.id, + logical_step_id="unresponsive-044", + step_position=0, + attempt=1, + status="running", + artifact_refs=["evidence:unresponsive"], + ) + seeded_execution.add_all([run, step]) + seeded_execution.flush() + artifact = register_artifact( + seeded_execution, + owner_type="scenario_run", + owner_id=run.id, + kind="evidence", + name="unresponsive-evidence", + content_ref="evidence:unresponsive", + sha256=_CANCEL_SHA, + logical_step_id=step.logical_step_id, + attempt=1, + ) + lease = claim_step( + seeded_execution, + run.id, + step.logical_step_id, + worker_id="unresponsive-worker", + side_effect_key=None, + idempotent=True, + retry_safe=True, + ) + + requested = cancel_run(seeded_execution, run.id, drain_seconds=1) + assert requested.status == "cancel_requested" + assert requested.cancel_drain_deadline_at is not None + assert step.status == "running" + assert not _expired(lease) + + terminalized = finalize_expired_cancellations( + seeded_execution, + now=requested.cancel_drain_deadline_at + timedelta(seconds=1), + ) + assert [item.id for item in terminalized] == [run.id] + assert run.status == "cancelled" + assert step.status == "cancelled" + assert step.artifact_refs == [] + assert artifact.is_active is False + assert _expired(lease) + assert finalize_expired_cancellations( + seeded_execution, + now=requested.cancel_drain_deadline_at + timedelta(seconds=2), + ) == [] +# #endregion Test.ScenarioExecution.CancelTimeout.Deadline + + +# #region Test.ScenarioExecution.CancelTimeout.Controls [C:4] [TYPE Function] +# @BRIEF Cancellation retires, rather than decides, the distinct human observation and PROD authorization controls. +# @TEST_INVARIANT ScenarioExecution.Lifecycle.Cancel: A cancelled run cannot later be revived by a +# stale checkpoint decision or pending PROD approval gate. -> VERIFIED_BY: +# test_cancel_expires_prod_gate_and_cancels_human_checkpoint +def test_cancel_expires_prod_gate_and_cancels_human_checkpoint(seeded_execution): + human_run = _run( + "80440000-0000-4000-8000-000000000006", + {"topological_order": [], "dependencies": [], "steps": []}, + ) + human_run.status = "waiting_human" + human_run.phase = "waiting_human" + checkpoint = HumanCheckpoint( + run_id=human_run.id, + logical_step_id="human-cancel-044", + evidence_refs=["evidence:human-cancel"], + ) + prod_run = _run( + "80440000-0000-4000-8000-000000000007", + {"topological_order": [], "dependencies": [], "steps": []}, + ) + prod_run.status = "pending_approval" + prod_run.phase = "preflight" + gate = ActionApprovalGate( + owner_type="scenario_run", + owner_id=prod_run.id, + request_hash="f" * 64, + status="pending", + ) + seeded_execution.add_all([human_run, prod_run]) + seeded_execution.flush() + seeded_execution.add_all([checkpoint, gate]) + seeded_execution.flush() + + assert cancel_run(seeded_execution, human_run.id).status == "cancelled" + assert checkpoint.status == "cancelled" + with pytest.raises(ValueError, match="stale checkpoint decision"): + decide_checkpoint( + seeded_execution, + checkpoint.id, + disposition="confirm", + expected_version=checkpoint.decision_version, + actor_id="analyst-cancel-044", + ) + + assert cancel_run(seeded_execution, prod_run.id).status == "cancelled" + assert gate.status == "expired" + with pytest.raises(ValueError, match="already decided"): + decide_approval_gate( + seeded_execution, + gate.id, + decision="approve", + actor_id="approver-cancel-044", + ) +# #endregion Test.ScenarioExecution.CancelTimeout.Controls + + +# #region Test.ScenarioExecution.CancelTimeout.Drain [C:5] [TYPE Function] +# @BRIEF A controlled adapter requests cancel while holding a real worker lease; its one returned +# completion drains, then the runner terminalizes all remaining pending work as cancelled. +def test_cancel_during_claimed_adapter_drains_then_terminalizes(seeded_execution): + plan = _valid_plan( + ["inflight-cancel", "queued-after-cancel"], + [{"source": "inflight-cancel", "target": "queued-after-cancel"}], + ) + run = _run(_CANCEL_RUN, plan) + queued = ScenarioStepRun( + run_id=run.id, + logical_step_id="queued-after-cancel", + step_position=1, + attempt=1, + status="queued", + ) + seeded_execution.add_all([run, queued]) + seeded_execution.flush() + registry = ScenarioExecutorRegistry() + + def adapter(_step, _completed): + lease = seeded_execution.query(ScenarioStepLease).filter_by( + run_id=run.id, logical_step_id="inflight-cancel", + ).one() + assert not _expired(lease) + assert cancel_run(seeded_execution, run.id, drain_in_flight=True).status == "cancel_requested" + return { + "status": "passed", + "step_outcome": {"status": "passed", "artifact_digests": {"evidence:drained": _CANCEL_SHA}}, + "artifact_refs": ["evidence:drained"], + } + + registry.register("assertion", adapter, action="structural_assert") + result = _advance_run(seeded_execution, run, registry, worker_id="cancel-drain-worker") + steps = { + row.logical_step_id: row + for row in seeded_execution.query(ScenarioStepRun).filter_by(run_id=run.id).all() + } + lease = seeded_execution.query(ScenarioStepLease).filter_by(run_id=run.id, logical_step_id="inflight-cancel").one() + artifacts = seeded_execution.query(ScenarioArtifact).filter_by(owner_id=run.id).all() + assert result["status"] == "cancelled" + assert run.status == "cancelled" + assert steps["inflight-cancel"].status == "passed" + assert steps["inflight-cancel"].step_outcome == {} + assert steps["inflight-cancel"].outputs["terminal_history"][0]["step_outcome"]["status"] == "passed" + assert steps["inflight-cancel"].artifact_refs == [] + assert steps["queued-after-cancel"].status == "skipped" + assert _expired(lease) + assert len(artifacts) == 1 and artifacts[0].is_active is False + assert seeded_execution.query(InvestigationQueueItem).filter_by(run_id=run.id).count() == 0 +# #endregion Test.ScenarioExecution.CancelTimeout.Drain + + +# #region Test.ScenarioExecution.CancelTimeout.AdapterTimeout [C:5] [TYPE Function] +# @BRIEF A real claimed adapter invokes the timeout boundary; its later PASS payload cannot overwrite timeout truth. +def test_timeout_during_claimed_adapter_terminalizes_without_stranded_state(seeded_execution): + plan = _valid_plan( + ["inflight-timeout", "blocked-after-timeout"], + [{"source": "inflight-timeout", "target": "blocked-after-timeout"}], + ) + run = _run(_TIMEOUT_RUN, plan) + timeout_step = ScenarioStepRun( + run_id=run.id, + logical_step_id="inflight-timeout", + step_position=0, + attempt=1, + status="queued", + artifact_refs=["evidence:partial-timeout"], + ) + descendant = ScenarioStepRun( + run_id=run.id, + logical_step_id="blocked-after-timeout", + step_position=1, + attempt=1, + status="queued", + ) + seeded_execution.add_all([run, timeout_step, descendant]) + seeded_execution.flush() + partial = register_artifact( + seeded_execution, + owner_type="scenario_run", + owner_id=run.id, + kind="evidence", + name="partial-timeout-evidence", + content_ref="evidence:partial-timeout", + sha256=_TIMEOUT_SHA, + logical_step_id=timeout_step.logical_step_id, + attempt=1, + ) + registry = ScenarioExecutorRegistry() + + def adapter(step, _completed): + lease = seeded_execution.query(ScenarioStepLease).filter_by( + run_id=run.id, logical_step_id="inflight-timeout", + ).one() + assert not _expired(lease) + apply_step_timeout(seeded_execution, run.id, step["logical_step_id"], timeout_ms=5) + return {"status": "passed", "step_outcome": {"status": "passed"}, "artifact_refs": []} + + registry.register("assertion", adapter, action="structural_assert") + result = _advance_run(seeded_execution, run, registry, worker_id="timeout-worker") + lease = seeded_execution.query(ScenarioStepLease).filter_by(run_id=run.id, logical_step_id="inflight-timeout").one() + signals = seeded_execution.query(InvestigationQueueItem).filter_by(run_id=run.id).all() + assert result["status"] == "inconclusive" + assert run.status == "inconclusive" + assert timeout_step.status == "inconclusive" + assert timeout_step.error_code == "STEP_TIMEOUT" + assert timeout_step.artifact_refs == [] + assert descendant.status == "blocked" + assert descendant.error_code == "UPSTREAM_TIMEOUT" + assert partial.is_active is False + assert _expired(lease) + assert len(signals) == 1 + assert signals[0].evidence_snapshot["evidence"] == [] + assert build_result(run, [timeout_step, descendant])["status"] == "inconclusive" + + assert apply_step_timeout(seeded_execution, run.id, timeout_step.logical_step_id, timeout_ms=5).id == timeout_step.id + assert seeded_execution.query(InvestigationQueueItem).filter_by(run_id=run.id).count() == 1 +# #endregion Test.ScenarioExecution.CancelTimeout.AdapterTimeout + + +# #region Test.ScenarioExecution.CancelTimeout.LazyDescendant [C:5] [TYPE Function] +# @BRIEF A timeout persists an absent declared descendant as blocked before any later walker can materialize it. +# @TEST_INVARIANT ScenarioExecution.Lifecycle.Timeout: Every declared timeout descendant has a +# pinned-plan blocked row, so recovery/continuation cannot lazy-dispatch the +# downstream executor. -> VERIFIED_BY: +# test_timeout_materializes_lazy_descendant_and_walker_never_dispatches +def test_timeout_materializes_lazy_descendant_and_walker_never_dispatches(seeded_execution): + plan = _valid_plan( + ["timeout-source-044", "lazy-descendant-044"], + [{"source": "timeout-source-044", "target": "lazy-descendant-044"}], + ) + run = _run("80440000-0000-4000-8000-000000000009", plan) + source = ScenarioStepRun( + run_id=run.id, + logical_step_id="timeout-source-044", + step_position=0, + attempt=1, + status="running", + ) + seeded_execution.add_all([run, source]) + seeded_execution.flush() + assert seeded_execution.query(ScenarioStepRun).filter_by( + run_id=run.id, + logical_step_id="lazy-descendant-044", + ).first() is None + + apply_step_timeout(seeded_execution, run.id, source.logical_step_id, timeout_ms=5) + descendant = seeded_execution.query(ScenarioStepRun).filter_by( + run_id=run.id, + logical_step_id="lazy-descendant-044", + ).one() + assert descendant.step_position == 1 + assert descendant.attempt == 1 + assert descendant.status == "blocked" + assert descendant.error_code == "UPSTREAM_TIMEOUT" + assert descendant.step_outcome == { + "status": "blocked", + "error_code": "UPSTREAM_TIMEOUT", + "blocked_by": "timeout-source-044", + } + assert descendant.inputs_snapshot["runner_plan_step"]["tool"] == "assertion" + + calls: list[str] = [] + registry = ScenarioExecutorRegistry() + + def must_not_dispatch(step, _completed): + calls.append(step["logical_step_id"]) + return {"status": "passed", "step_outcome": {"status": "passed"}, "artifact_refs": []} + + registry.register("assertion", must_not_dispatch, action="structural_assert") + result = _advance_run(seeded_execution, run, registry, worker_id="timeout-lazy-continuation") + assert result["status"] == "inconclusive" + assert calls == [] +# #endregion Test.ScenarioExecution.CancelTimeout.LazyDescendant +# #endregion Test.ScenarioExecution.CancelTimeout diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_crash_recovery.py b/backend/tests/services/dashboard_testing/registry/test_scenario_crash_recovery.py new file mode 100644 index 000000000..7310497d1 --- /dev/null +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_crash_recovery.py @@ -0,0 +1,221 @@ +# #region Test.ScenarioExecution.CrashRecovery [C:5] [TYPE Module] [SEMANTICS test,scenario,execution,recovery,crash,lease,browser,artifact] +# @BRIEF Prove persisted server-driven crash recovery resumes only safe frontier work from the pinned plan. +# @RELATION BINDS_TO -> [ScenarioExecution.Runner.CrashRecovery] +# @RELATION BINDS_TO -> [ScenarioExecution.Worker.Claim] +# @RELATION BINDS_TO -> [ScenarioExecution.Artifacts.InvalidateStepEvidence] +# @RELATION BINDS_TO -> [ScenarioExecution.Runner.TerminalSignal] +# @TEST_CONTRACT: expired persisted worker claim + pinned ScenarioRun -> safe recovery or typed non-pass +# @TEST_FIXTURE: crash_recovery_044 -> INLINE SQLite hardcoded DAG, leases, and evidence refs +# @TEST_INVARIANT ScenarioExecution.Runner.CrashRecovery: Completed ancestors are never rerun; +# only an expired idempotent/retry-safe claim creates one new frontier attempt. +# -> VERIFIED_BY: test_safe_claim_recovery_replays_only_frontier_once +# @TEST_INVARIANT ScenarioExecution.Runner.CrashRecovery: Unsafe effects require reconciliation and +# browser work without a pinned safe checkpoint is non-pass without adapter I/O. +# -> VERIFIED_BY: test_unsafe_claim_requires_reconciliation, +# test_browser_recovery_requires_declared_safe_checkpoint +# @TEST_INVARIANT ScenarioExecution.Runner.CrashRecovery.ArchiveAttempt: A safe replay archives +# the abandoned evidence and retires only its active projection before attempt two. +# -> VERIFIED_BY: test_safe_claim_recovery_replays_only_frontier_once +# @TEST_INVARIANT ScenarioExecution.Runner.CrashRecovery.Reject: Repeating an identical unsafe +# recovery rejection reuses one immutable terminal signal and never dispatches it. +# -> VERIFIED_BY: test_unsafe_claim_requires_reconciliation +# @TEST_INVARIANT ScenarioExecution.Runner.CrashRecovery.BrowserCheckpoint: Browser work without +# a pinned safe checkpoint is inconclusive before any browser adapter call. -> VERIFIED_BY: +# test_browser_recovery_requires_declared_safe_checkpoint +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from src.models.scenario_investigation import InvestigationQueueItem +from src.models.scenario_run import ScenarioRun, ScenarioStepRun +from src.services.dashboard_testing.execution.artifacts import register_artifact +from src.services.dashboard_testing.execution.executor_registry import ScenarioExecutorRegistry +from src.services.dashboard_testing.execution.runner import recover_run +from src.services.dashboard_testing.execution.worker import claim_step +from src.services.dashboard_testing.scenario.templates import ( + ACTION_REGISTRY_VERSION, + action_registry_fingerprint, + resolve_action_descriptor, +) + +_SCENARIO_ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1" +_REVISION_ID = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1" +_RECOVERY_SHA = "a" * 64 + + +def _step(step_id: str, tool: str, action: str, **extra) -> dict: + return { + "logical_step_id": step_id, "tool": tool, "action": action, + "action_descriptor": resolve_action_descriptor( + tool=tool, action=action, registry_version=ACTION_REGISTRY_VERSION, + registry_hash=action_registry_fingerprint(), + ).snapshot(), + **extra, + } + + +def _plan(steps: list[dict], dependencies: list[dict]) -> dict: + return { + "action_registry_version": ACTION_REGISTRY_VERSION, + "action_registry_hash": action_registry_fingerprint(), + "topological_order": [step["logical_step_id"] for step in steps], + "dependencies": dependencies, + "steps": steps, + } + + +def _run(run_id: str, plan: dict) -> ScenarioRun: + return ScenarioRun( + id=run_id, + scenario_id=_SCENARIO_ID, + scenario_revision_id=_REVISION_ID, + scenario_content_hash="b" * 64, + environment_id="env-crash-recovery-044", + status="running", + phase="executing", + parameter_bindings={"fixture": "crash-recovery-044"}, + target_snapshot={"environment_id": "env-crash-recovery-044"}, + trigger_source="manual", + idempotency_key=run_id, + runner_plan=plan, + execution_principal_fingerprint="c" * 64, + ) + + +def _expire(lease) -> None: + lease.expires_at = datetime.now(UTC) - timedelta(seconds=1) + + +# #region Test.ScenarioExecution.CrashRecovery.Safe [C:5] [TYPE Function] +# @BRIEF An expired retry-safe claim is archived, evidence-retired, and dispatched exactly once as attempt two. +def test_safe_claim_recovery_replays_only_frontier_once(seeded_execution): + plan = _plan( + [ + _step("completed-ancestor-044", "assertion", "structural_assert"), + _step("safe-frontier-044", "assertion", "structural_assert"), + ], + [{"source": "completed-ancestor-044", "target": "safe-frontier-044"}], + ) + run = _run("80440000-0000-4000-8000-000000000010", plan) + completed = ScenarioStepRun( + run_id=run.id, + logical_step_id="completed-ancestor-044", + step_position=0, + attempt=1, + status="passed", + progress=100, + step_outcome={"status": "passed"}, + ) + frontier = ScenarioStepRun( + run_id=run.id, + logical_step_id="safe-frontier-044", + step_position=1, + attempt=1, + status="running", + artifact_refs=["evidence:crash-partial"], + ) + seeded_execution.add_all([run, completed, frontier]) + seeded_execution.flush() + historical = register_artifact( + seeded_execution, + owner_type="scenario_run", + owner_id=run.id, + kind="evidence", + name="crash-partial", + content_ref="evidence:crash-partial", + sha256=_RECOVERY_SHA, + logical_step_id=frontier.logical_step_id, + attempt=1, + ) + lease = claim_step( + seeded_execution, + run.id, + frontier.logical_step_id, + worker_id="crashed-worker-044", + side_effect_key="safe-effect-044", + idempotent=True, + retry_safe=True, + ) + _expire(lease) + seeded_execution.flush() + calls: list[str] = [] + registry = ScenarioExecutorRegistry() + + def safe_adapter(step, _completed): + calls.append(step["logical_step_id"]) + return {"status": "passed", "step_outcome": {"status": "passed"}, "artifact_refs": []} + + registry.register("assertion", safe_adapter, action="structural_assert") + result = recover_run(seeded_execution, run.id, worker_id="recovery-worker-044", registry=registry) + + assert result["status"] == "passed" + assert calls == ["safe-frontier-044"] + assert completed.attempt == 1 + assert completed.status == "passed" + assert frontier.attempt == 2 + assert frontier.status == "passed" + assert frontier.outputs["recovery_history"][0]["artifact_refs"] == ["evidence:crash-partial"] + assert frontier.artifact_refs == [] + assert historical.is_active is False + assert seeded_execution.query(InvestigationQueueItem).filter_by(run_id=run.id).count() == 0 + + assert recover_run(seeded_execution, run.id, worker_id="recovery-worker-044", registry=registry)["status"] == "passed" + assert calls == ["safe-frontier-044"] +# #endregion Test.ScenarioExecution.CrashRecovery.Safe + + +# #region Test.ScenarioExecution.CrashRecovery.Unsafe [C:4] [TYPE Function] +# @BRIEF An expired unsafe effect becomes an auditable blocked reconciliation requirement, never a replay. +def test_unsafe_claim_requires_reconciliation(seeded_execution): + plan = _plan( + [_step("unsafe-frontier-044", "artifact", "register_artifact", mutation_contract={"fixture": "unsafe"})], + [], + ) + run = _run("80440000-0000-4000-8000-000000000011", plan) + frontier = ScenarioStepRun(run_id=run.id, logical_step_id="unsafe-frontier-044", step_position=0, attempt=1, status="running") + seeded_execution.add_all([run, frontier]) + seeded_execution.flush() + lease = claim_step( + seeded_execution, run.id, frontier.logical_step_id, worker_id="crashed-unsafe-044", + side_effect_key="unsafe-effect-044", idempotent=False, retry_safe=False, + ) + _expire(lease) + registry = ScenarioExecutorRegistry() + calls: list[str] = [] + registry.register("artifact", lambda step, _completed: calls.append(step["logical_step_id"]), action="register_artifact") + + result = recover_run(seeded_execution, run.id, worker_id="recovery-unsafe-044", registry=registry) + assert result["status"] == "blocked" + assert frontier.attempt == 1 + assert frontier.error_code == "RECOVERY_RECONCILIATION_REQUIRED" + assert calls == [] + assert seeded_execution.query(InvestigationQueueItem).filter_by(run_id=run.id).count() == 1 + assert recover_run(seeded_execution, run.id, worker_id="recovery-unsafe-044", registry=registry)["status"] == "blocked" + assert seeded_execution.query(InvestigationQueueItem).filter_by(run_id=run.id).count() == 1 +# #endregion Test.ScenarioExecution.CrashRecovery.Unsafe + + +# #region Test.ScenarioExecution.CrashRecovery.Browser [C:4] [TYPE Function] +# @BRIEF A dead browser context cannot recover without a declared pinned browser-safe checkpoint. +def test_browser_recovery_requires_declared_safe_checkpoint(seeded_execution): + plan = _plan([_step("browser-frontier-044", "browser", "open_dashboard")], []) + run = _run("80440000-0000-4000-8000-000000000012", plan) + frontier = ScenarioStepRun(run_id=run.id, logical_step_id="browser-frontier-044", step_position=0, attempt=1, status="running") + seeded_execution.add_all([run, frontier]) + seeded_execution.flush() + lease = claim_step( + seeded_execution, run.id, frontier.logical_step_id, worker_id="crashed-browser-044", + side_effect_key="browser-effect-044", idempotent=True, retry_safe=True, + ) + _expire(lease) + calls: list[str] = [] + registry = ScenarioExecutorRegistry() + registry.register("browser", lambda step, _completed: calls.append(step["logical_step_id"])) + + result = recover_run(seeded_execution, run.id, worker_id="recovery-browser-044", registry=registry) + assert result["status"] == "inconclusive" + assert frontier.error_code == "BROWSER_RECOVERY_CHECKPOINT_REQUIRED" + assert calls == [] + assert seeded_execution.query(InvestigationQueueItem).filter_by(run_id=run.id).count() == 1 +# #endregion Test.ScenarioExecution.CrashRecovery.Browser +# #endregion Test.ScenarioExecution.CrashRecovery diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_dispatch.py b/backend/tests/services/dashboard_testing/registry/test_scenario_dispatch.py index 8ad5d739d..fb8dac7fc 100644 --- a/backend/tests/services/dashboard_testing/registry/test_scenario_dispatch.py +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_dispatch.py @@ -1,23 +1,36 @@ # #region Test.ScenarioExecution.Dispatch [C:3] [TYPE Module] [SEMANTICS test,scenario,execution,dispatch,dependency] # @RELATION BINDS_TO -> [ScenarioExecution.Dispatch.Step] -import pytest - from src.services.dashboard_testing.execution.dispatch import dispatch_step from src.services.dashboard_testing.execution.executor_registry import ScenarioExecutorRegistry +from src.services.dashboard_testing.scenario.templates import ( + ACTION_REGISTRY_VERSION, + action_registry_fingerprint, + resolve_action_descriptor, +) + + +def _step(step_id: str, tool: str, action: str) -> dict: + return { + "logical_step_id": step_id, "tool": tool, "action": action, + "action_descriptor": resolve_action_descriptor( + tool=tool, action=action, registry_version=ACTION_REGISTRY_VERSION, + registry_hash=action_registry_fingerprint(), + ).snapshot(), + } def test_dispatch_rejects_unknown_tool_before_io(): registry = ScenarioExecutorRegistry() - with pytest.raises(ValueError, match="unknown executor"): - dispatch_step({"logical_step_id": "a", "tool": "unknown"}, completed={}, registry=registry, edges=[]) + result = dispatch_step({"logical_step_id": "a", "tool": "unknown"}, completed={}, registry=registry, edges=[]) + assert result == {"status": "blocked", "reason": "ACTION_DESCRIPTOR_REQUIRED", "logical_step_id": "a"} def test_dispatch_blocks_descendant_after_failed_dependency(): registry = ScenarioExecutorRegistry() registry.register("assertion", lambda _step, _completed: {"status": "passed"}) - result = dispatch_step({"logical_step_id": "b", "tool": "assertion"}, completed={"a": {"status": "failed"}}, registry=registry, edges=[{"source": "a", "target": "b"}]) + result = dispatch_step(_step("b", "assertion", "structural_assert"), completed={"a": {"status": "failed"}}, registry=registry, edges=[{"source": "a", "target": "b"}]) assert result["status"] == "blocked" def test_human_is_waiting_control_not_executor(): - result = dispatch_step({"logical_step_id": "human-1", "tool": "human"}, completed={}, registry=ScenarioExecutorRegistry(), edges=[]) + result = dispatch_step(_step("human-1", "human", "human_checkpoint"), completed={}, registry=ScenarioExecutorRegistry(), edges=[]) assert result["status"] == "waiting_human" # #endregion Test.ScenarioExecution.Dispatch diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_executors.py b/backend/tests/services/dashboard_testing/registry/test_scenario_executors.py new file mode 100644 index 000000000..f00fa5246 --- /dev/null +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_executors.py @@ -0,0 +1,472 @@ +# #region Test.ScenarioExecution.Executors [C:3] [TYPE Module] [SEMANTICS test,scenario,execution,executor,browser] +# @BRIEF Verify typed executor outcomes and fail-safe browser adapter dispatch. +# @RELATION BINDS_TO -> [ScenarioExecution.Executors] +# @RELATION VERIFIES -> [ScenarioExecution.Executors.Browser] +# @RELATION VERIFIES -> [ScenarioExecution.Executors.SupersetApi] +# @RELATION VERIFIES -> [ScenarioExecution.Executors.Screenshot] +# @TEST_CONTRACT: Browser/Superset/ScreenshotStep + typed external adapter -> typed ScenarioStep outcome +# @TEST_CONTRACT: ArtifactStep + canonical non-zero SHA-256 -> typed non-forged ScenarioStep outcome +# @TEST_FIXTURE: browser_step_044 + superset_step_044 + screenshot_step_044 -> INLINE_JSON hardcoded live-I/O context +# @TEST_EDGE: metadata_without_adapter -> inconclusive; invalid_adapter_status -> inconclusive; missing_evidence -> inconclusive; adapter_failure -> failed; adapter_timeout -> inconclusive; forged_or_mismatched_digest -> non-pass +# @TEST_INVARIANT ScenarioExecution.Executors.Browser: Browser PASS is materialized only from an explicit BrowserExecutionAdapter success, never from session_id or page_ref metadata. -> VERIFIED_BY: ids_without_adapter, adapter_success, adapter_failure, adapter_timeout +# @TEST_INVARIANT ScenarioExecution.Executors.SupersetApi: Superset API PASS is materialized only from an explicit SupersetExecutionAdapter success, never from query_result metadata. -> VERIFIED_BY: query_metadata_without_adapter, superset_adapter_success, superset_adapter_failure, superset_adapter_timeout +# @TEST_INVARIANT ScenarioExecution.Executors.Screenshot: Screenshot PASS is materialized only from explicit ScreenshotExecutionAdapter success with durable evidence refs, never from capture_bytes metadata. -> VERIFIED_BY: capture_bytes_without_adapter, screenshot_adapter_success, screenshot_success_without_evidence, screenshot_adapter_failure, screenshot_adapter_timeout +# @TEST_INVARIANT ScenarioExecution.Executors: Unavailable live I/O returns typed inconclusive. -> VERIFIED_BY: ids_without_adapter, query_metadata_without_adapter, capture_bytes_without_adapter, browser_adapter_timeout, superset_adapter_timeout, screenshot_adapter_timeout +# @TEST_INVARIANT ScenarioExecution.LiveAdapter: Runtime adapter status is exactly passed, failed, or inconclusive; untrusted values fail closed. -> VERIFIED_BY: invalid_adapter_status +# @TEST_INVARIANT ScenarioExecution.Executors.Artifact: Artifact PASS requires a canonical, non-zero SHA-256 digest for the declared ref. -> VERIFIED_BY: artifact_rejects_forged_digest +# @TEST_INVARIANT ScenarioExecution.Executors.Screenshot: Screenshot PASS requires non-zero SHA-256 values agreeing with every forwarded artifact digest. -> VERIFIED_BY: screenshot_invalid_or_mismatched_digest, screenshot_multiple_evidence_refs +from io import BytesIO +import pytest + +from src.services.dashboard_testing.execution.executor_registry import ScenarioExecutorRegistry +from src.services.dashboard_testing.execution.executors import ( + BrowserAdapterResult, + ScreenshotAdapterResult, + SupersetAdapterResult, + _register_default_executors, +) +from src.services.dashboard_testing.scenario.templates import ( + ACTION_REGISTRY_VERSION, + action_registry_fingerprint, + resolve_action_descriptor, +) + + +# #region Test.ScenarioExecution.Executors.Registry [C:1] [TYPE Function] +def _registry(*, browser_adapter=None, superset_adapter=None, screenshot_adapter=None) -> ScenarioExecutorRegistry: + registry = ScenarioExecutorRegistry() + _register_default_executors( + registry, + browser_adapter=browser_adapter, + superset_adapter=superset_adapter, + screenshot_adapter=screenshot_adapter, + ) + return registry + + +def _resolve(registry: ScenarioExecutorRegistry, tool: str, action: str): + """Resolve only a hardcoded immutable descriptor, never a tool fallback.""" + return registry.resolve(resolve_action_descriptor( + tool=tool, + action=action, + registry_version=ACTION_REGISTRY_VERSION, + registry_hash=action_registry_fingerprint(), + ).snapshot()) +# #endregion Test.ScenarioExecution.Executors.Registry + + +_BROWSER_STEP_044 = { + "logical_step_id": "browser-filter-044", + "session_id": "persisted-session-044", + "page_ref": "persisted-page-044", + "step_meta": {"action": "apply_filters", "selector": "[data-test=region-filter]"}, +} +_COMPLETED_044 = {"setup-044": {"status": "passed", "output_refs": ["setup:044"]}} +_SUPERSET_STEP_044 = { + "logical_step_id": "metric-044", + "step_meta": { + "environment_id": "preprod-044", + "dashboard_id": "dashboard-044", + "chart_id": 44, + "result_key": "revenue", + "query_result": {"revenue": 4400}, + }, +} +_SCREENSHOT_STEP_044 = { + "logical_step_id": "screenshot-044", + "step_meta": { + "environment_id": "preprod-044", + "dashboard_id": "dashboard-044", + "capture_profile_id": "profile-044", + "capture_bytes": b"metadata-only-capture-044", + }, +} + + +# #region Test.ScenarioExecution.Executors.BrowserUnavailable [C:2] [TYPE Function] +# @BRIEF IDs without an executable browser adapter are inconclusive, never synthetic PASS. +def test_browser_ids_without_adapter_are_inconclusive(): + outcome = _resolve(_registry(), "browser", "apply_filters")(_BROWSER_STEP_044, _COMPLETED_044) + assert outcome["status"] == "inconclusive" + assert outcome["error_code"] == "BROWSER_ADAPTER_UNAVAILABLE" + assert outcome["step_outcome"]["session_id"] == "persisted-session-044" + assert outcome["step_outcome"]["page_ref"] == "persisted-page-044" +# #endregion Test.ScenarioExecution.Executors.BrowserUnavailable + + +# #region Test.ScenarioExecution.Executors.BrowserSuccess [C:2] [TYPE Function] +# @BRIEF Explicit external adapter success maps to passed while retaining execution context. +def test_browser_adapter_success_maps_to_passed_and_receives_context(): + observed: dict[str, object] = {} + + def successful_transport(step, completed): + observed["step_id"] = step["logical_step_id"] + observed["action"] = step["step_meta"]["action"] + observed["completed"] = completed["setup-044"]["status"] + return BrowserAdapterResult( + status="passed", + reason_code="BROWSER_ACTION_APPLIED", + details={"observed_text": "EMEA"}, + output_refs=["browser-output:044"], + artifact_refs=["browser-evidence:044"], + ) + + outcome = _resolve(_registry(browser_adapter=successful_transport), "browser", "apply_filters")( + _BROWSER_STEP_044, + _COMPLETED_044, + ) + + assert observed == {"step_id": "browser-filter-044", "action": "apply_filters", "completed": "passed"} + assert outcome["status"] == "passed" + assert outcome["error_code"] is None + assert outcome["step_outcome"]["reason_code"] == "BROWSER_ACTION_APPLIED" + assert outcome["step_outcome"]["observed_text"] == "EMEA" + assert outcome["step_outcome"]["session_id"] == "persisted-session-044" + assert outcome["step_outcome"]["page_ref"] == "persisted-page-044" + assert outcome["output_refs"] == ["browser-output:044"] + assert outcome["artifact_refs"] == ["browser-evidence:044"] +# #endregion Test.ScenarioExecution.Executors.BrowserSuccess + + +# #region Test.ScenarioExecution.Executors.BrowserFailure [C:2] [TYPE Function] +# @BRIEF Explicit adapter failure is a failed step, not a browser PASS. +def test_browser_adapter_failure_maps_to_explicit_failed_outcome(): + def failing_transport(_step, _completed): + return BrowserAdapterResult(status="failed", reason_code="BROWSER_SELECTOR_NOT_FOUND") + + outcome = _resolve(_registry(browser_adapter=failing_transport), "browser", "apply_filters")( + _BROWSER_STEP_044, + _COMPLETED_044, + ) + + assert outcome["status"] == "failed" + assert outcome["error_code"] == "BROWSER_SELECTOR_NOT_FOUND" +# #endregion Test.ScenarioExecution.Executors.BrowserFailure + + +# #region Test.ScenarioExecution.Executors.BrowserTimeout [C:2] [TYPE Function] +# @BRIEF External adapter timeout remains explicitly inconclusive and never manufactures PASS. +def test_browser_adapter_timeout_maps_to_explicit_inconclusive_outcome(): + def timeout_transport(_step, _completed): + raise TimeoutError("browser action exceeded 30s") + + outcome = _resolve(_registry(browser_adapter=timeout_transport), "browser", "apply_filters")( + _BROWSER_STEP_044, + _COMPLETED_044, + ) + + assert outcome["status"] == "inconclusive" + assert outcome["error_code"] == "BROWSER_ADAPTER_TIMEOUT" +# #endregion Test.ScenarioExecution.Executors.BrowserTimeout + + +# #region Test.ScenarioExecution.Executors.AdapterInvalidStatus [C:2] [TYPE Function] +# @BRIEF An external adapter's untrusted runtime status fails closed before result aggregation. +def test_browser_adapter_invalid_status_is_typed_inconclusive(): + def invalid_status_transport(_step, _completed): + return BrowserAdapterResult(status="untrusted", reason_code="UNTRUSTED_STATUS") + + outcome = _resolve(_registry(browser_adapter=invalid_status_transport), "browser", "apply_filters")( + _BROWSER_STEP_044, + _COMPLETED_044, + ) + + assert outcome["status"] == "inconclusive" + assert outcome["error_code"] == "BROWSER_ADAPTER_INVALID_RESULT" +# #endregion Test.ScenarioExecution.Executors.AdapterInvalidStatus + + +# #region Test.ScenarioExecution.Executors.SupersetUnavailable [C:2] [TYPE Function] +# @BRIEF Static query metadata without a live adapter is inconclusive, never synthetic PASS. +def test_superset_query_metadata_without_adapter_is_inconclusive(): + outcome = _resolve(_registry(), "superset_api", "execute_metric")(_SUPERSET_STEP_044, _COMPLETED_044) + + assert outcome["status"] == "inconclusive" + assert outcome["error_code"] == "SUPERSET_ADAPTER_UNAVAILABLE" + assert outcome["step_outcome"]["dashboard_id"] == "dashboard-044" + assert outcome["step_outcome"]["result_key"] == "revenue" + assert "actual" not in outcome["step_outcome"] +# #endregion Test.ScenarioExecution.Executors.SupersetUnavailable + + +# #region Test.ScenarioExecution.Executors.SupersetSuccess [C:2] [TYPE Function] +# @BRIEF Explicit 037 adapter success maps to passed and receives full execution context. +def test_superset_adapter_success_maps_to_passed_and_receives_context(): + observed: dict[str, object] = {} + + def successful_transport(step, completed): + observed["step_id"] = step["logical_step_id"] + observed["query_metadata"] = step["step_meta"]["query_result"] + observed["completed"] = completed["setup-044"]["status"] + return SupersetAdapterResult( + status="passed", + reason_code="SUPERSET_CHART_DATA_EXECUTED", + details={"actual": {"revenue": 4400}, "source_response_hash": "a" * 64}, + output_refs=["superset-output:044"], + artifact_refs=["superset-evidence:044"], + ) + + outcome = _resolve(_registry(superset_adapter=successful_transport), "superset_api", "execute_metric")( + _SUPERSET_STEP_044, + _COMPLETED_044, + ) + + assert observed == { + "step_id": "metric-044", + "query_metadata": {"revenue": 4400}, + "completed": "passed", + } + assert outcome["status"] == "passed" + assert outcome["error_code"] is None + assert outcome["step_outcome"]["reason_code"] == "SUPERSET_CHART_DATA_EXECUTED" + assert outcome["step_outcome"]["actual"] == {"revenue": 4400} + assert outcome["step_outcome"]["source_response_hash"] == "a" * 64 + assert outcome["output_refs"] == ["superset-output:044"] + assert outcome["artifact_refs"] == ["superset-evidence:044"] +# #endregion Test.ScenarioExecution.Executors.SupersetSuccess + + +# #region Test.ScenarioExecution.Executors.SupersetFailure [C:2] [TYPE Function] +# @BRIEF Explicit adapter failure remains failed and never turns query metadata into PASS. +def test_superset_adapter_failure_maps_to_explicit_failed_outcome(): + def failing_transport(_step, _completed): + return SupersetAdapterResult(status="failed", reason_code="SUPERSET_FORBIDDEN") + + outcome = _resolve(_registry(superset_adapter=failing_transport), "superset_api", "execute_metric")( + _SUPERSET_STEP_044, + _COMPLETED_044, + ) + + assert outcome["status"] == "failed" + assert outcome["error_code"] == "SUPERSET_FORBIDDEN" +# #endregion Test.ScenarioExecution.Executors.SupersetFailure + + +# #region Test.ScenarioExecution.Executors.SupersetTimeout [C:2] [TYPE Function] +# @BRIEF Explicit adapter timeout remains inconclusive and never manufactures PASS. +def test_superset_adapter_timeout_maps_to_explicit_inconclusive_outcome(): + def timeout_transport(_step, _completed): + raise TimeoutError("chart-data request exceeded 30s") + + outcome = _resolve(_registry(superset_adapter=timeout_transport), "superset_api", "execute_metric")( + _SUPERSET_STEP_044, + _COMPLETED_044, + ) + + assert outcome["status"] == "inconclusive" + assert outcome["error_code"] == "SUPERSET_ADAPTER_TIMEOUT" +# #endregion Test.ScenarioExecution.Executors.SupersetTimeout + + +# #region Test.ScenarioExecution.Executors.ScreenshotUnavailable [C:2] [TYPE Function] +# @BRIEF Raw capture bytes without the ScreenshotService/036 adapter are inconclusive, never PASS. +def test_screenshot_capture_bytes_without_adapter_are_inconclusive(): + outcome = _resolve(_registry(), "screenshot", "capture_screenshot")(_SCREENSHOT_STEP_044, _COMPLETED_044) + + assert outcome["status"] == "inconclusive" + assert outcome["error_code"] == "SCREENSHOT_ADAPTER_UNAVAILABLE" + assert outcome["step_outcome"]["dashboard_id"] == "dashboard-044" + assert "sha256" not in outcome["step_outcome"] + assert outcome["artifact_refs"] == [] +# #endregion Test.ScenarioExecution.Executors.ScreenshotUnavailable + + +# #region Test.ScenarioExecution.Executors.ScreenshotSuccess [C:2] [TYPE Function] +# @BRIEF Explicit ScreenshotService/036 adapter success carries durable evidence and passes. +def test_screenshot_adapter_success_maps_to_passed_with_evidence_ref(): + observed: dict[str, object] = {} + + def successful_transport(step, completed): + observed["step_id"] = step["logical_step_id"] + observed["capture_bytes"] = step["step_meta"]["capture_bytes"] + observed["completed"] = completed["setup-044"]["status"] + return ScreenshotAdapterResult( + status="passed", + reason_code="SCREENSHOT_EVIDENCE_REGISTERED", + details={"sha256": "b" * 64, "capture_method": "screenshot-service"}, + output_refs=["evidence:screenshot-044"], + artifact_refs=["draft-screenshot-044"], + artifact_digests={"draft-screenshot-044": "b" * 64}, + ) + + outcome = _resolve(_registry(screenshot_adapter=successful_transport), "screenshot", "capture_screenshot")( + _SCREENSHOT_STEP_044, + _COMPLETED_044, + ) + + assert observed == { + "step_id": "screenshot-044", + "capture_bytes": b"metadata-only-capture-044", + "completed": "passed", + } + assert outcome["status"] == "passed" + assert outcome["error_code"] is None + assert outcome["step_outcome"]["reason_code"] == "SCREENSHOT_EVIDENCE_REGISTERED" + assert outcome["step_outcome"]["sha256"] == "b" * 64 + assert outcome["step_outcome"]["artifact_digests"] == {"draft-screenshot-044": "b" * 64} + assert outcome["output_refs"] == ["evidence:screenshot-044"] + assert outcome["artifact_refs"] == ["draft-screenshot-044"] +# #endregion Test.ScenarioExecution.Executors.ScreenshotSuccess + + +# #region Test.ScenarioExecution.Executors.ScreenshotMultipleEvidence [C:2] [TYPE Function] +# @BRIEF Multiple durable screenshot artifacts pass only when every ref has a matching digest. +def test_screenshot_adapter_multiple_evidence_refs_map_to_passed(): + digests = {"draft:screenshot-044:one": "b" * 64, "draft:screenshot-044:two": "c" * 64} + + def successful_transport(_step, _completed): + return ScreenshotAdapterResult( + status="passed", + reason_code="SCREENSHOT_EVIDENCE_REGISTERED", + details={"sha256": digests["draft:screenshot-044:one"]}, + artifact_refs=list(digests), + artifact_digests=digests, + ) + + outcome = _resolve(_registry(screenshot_adapter=successful_transport), "screenshot", "capture_screenshot")( + _SCREENSHOT_STEP_044, + _COMPLETED_044, + ) + + assert outcome["status"] == "passed" + assert outcome["artifact_refs"] == list(digests) +# #endregion Test.ScenarioExecution.Executors.ScreenshotMultipleEvidence + + +# #region Test.ScenarioExecution.Executors.ScreenshotMissingEvidence [C:2] [TYPE Function] +# @BRIEF A typed success without durable digest and evidence refs remains inconclusive. +def test_screenshot_adapter_success_without_evidence_is_inconclusive(): + def incomplete_transport(_step, _completed): + return ScreenshotAdapterResult(status="passed", reason_code="SCREENSHOT_CAPTURED") + + outcome = _resolve(_registry(screenshot_adapter=incomplete_transport), "screenshot", "capture_screenshot")( + _SCREENSHOT_STEP_044, + _COMPLETED_044, + ) + + assert outcome["status"] == "inconclusive" + assert outcome["error_code"] == "SCREENSHOT_EVIDENCE_REQUIRED" + assert outcome["artifact_refs"] == [] +# #endregion Test.ScenarioExecution.Executors.ScreenshotMissingEvidence + + +# #region Test.ScenarioExecution.Executors.ScreenshotDigestIntegrity [C:2] [TYPE Function] +# @BRIEF Invalid or mismatched screenshot digests cannot turn a live adapter response into PASS. +@pytest.mark.parametrize( + ("declared_digest", "forwarded_digest"), + [("0" * 64, "0" * 64), ("c" * 64, "d" * 64)], +) +def test_screenshot_adapter_invalid_or_mismatched_digest_is_nonpass( + declared_digest, forwarded_digest, +): + def forged_transport(_step, _completed): + return ScreenshotAdapterResult( + status="passed", + reason_code="SCREENSHOT_EVIDENCE_REGISTERED", + details={"sha256": declared_digest}, + artifact_refs=["screenshot-evidence-integrity-044"], + artifact_digests={"screenshot-evidence-integrity-044": forwarded_digest}, + ) + + outcome = _resolve(_registry(screenshot_adapter=forged_transport), "screenshot", "capture_screenshot")( + _SCREENSHOT_STEP_044, + _COMPLETED_044, + ) + + assert outcome["status"] == "inconclusive" + assert outcome["error_code"] == "SCREENSHOT_EVIDENCE_INVALID" + assert outcome["artifact_refs"] == [] +# #endregion Test.ScenarioExecution.Executors.ScreenshotDigestIntegrity + + +# #region Test.ScenarioExecution.Executors.ScreenshotFailure [C:2] [TYPE Function] +# @BRIEF Explicit capture failure remains failed and cannot turn metadata bytes into PASS. +def test_screenshot_adapter_failure_maps_to_explicit_failed_outcome(): + def failing_transport(_step, _completed): + return ScreenshotAdapterResult(status="failed", reason_code="SCREENSHOT_CAPTURE_FAILED") + + outcome = _resolve(_registry(screenshot_adapter=failing_transport), "screenshot", "capture_screenshot")( + _SCREENSHOT_STEP_044, + _COMPLETED_044, + ) + + assert outcome["status"] == "failed" + assert outcome["error_code"] == "SCREENSHOT_CAPTURE_FAILED" +# #endregion Test.ScenarioExecution.Executors.ScreenshotFailure + + +# #region Test.ScenarioExecution.Executors.ScreenshotTimeout [C:2] [TYPE Function] +# @BRIEF Explicit capture timeout remains inconclusive and never manufactures evidence. +def test_screenshot_adapter_timeout_maps_to_explicit_inconclusive_outcome(): + def timeout_transport(_step, _completed): + raise TimeoutError("screenshot capture exceeded 30s") + + outcome = _resolve(_registry(screenshot_adapter=timeout_transport), "screenshot", "capture_screenshot")( + _SCREENSHOT_STEP_044, + _COMPLETED_044, + ) + + assert outcome["status"] == "inconclusive" + assert outcome["error_code"] == "SCREENSHOT_ADAPTER_TIMEOUT" +# #endregion Test.ScenarioExecution.Executors.ScreenshotTimeout + + +def test_assertion_uses_037_compare_values_and_can_fail(): + step = { + "logical_step_id": "assert-1", + "actual": {"kind": "integer", "canonical_value": "10"}, + "expected": {"kind": "integer", "canonical_value": "11"}, + "policy": {"type": "exact"}, + } + outcome = _resolve(_registry(), "assertion", "structural_assert")(step, {}) + assert outcome["status"] == "failed" + assert outcome["step_outcome"]["comparison_status"] == "fail" + + +def test_assertion_pass_is_derived_from_compare_values(): + step = { + "logical_step_id": "assert-2", + "actual": {"kind": "string", "canonical_value": "emea"}, + "expected": {"kind": "string", "canonical_value": "emea"}, + "policy": {"type": "exact"}, + } + outcome = _resolve(_registry(), "assertion", "structural_assert")(step, {}) + assert outcome["status"] == "passed" + assert outcome["error_code"] is None + + +def test_xlsx_parses_workbook_bytes(): + from openpyxl import Workbook + + workbook = Workbook() + workbook.active.append(["region", "amount"]) + workbook.active.append(["emea", 10]) + buffer = BytesIO() + workbook.save(buffer) + outcome = _resolve(_registry(), "xlsx", "parse_xlsx")({"logical_step_id": "xlsx-1", "xlsx_bytes": buffer.getvalue()}, {}) + assert outcome["status"] == "passed" + assert outcome["step_outcome"]["row_count"] == 2 + assert outcome["artifact_refs"][0].startswith("xlsx:") + + +def test_screenshot_without_adapter_is_inconclusive(): + outcome = _resolve(_registry(), "screenshot", "capture_screenshot")({"logical_step_id": "shot-1"}, {}) + assert outcome["status"] == "inconclusive" + assert outcome["error_code"] == "SCREENSHOT_ADAPTER_UNAVAILABLE" + + +# #region Test.ScenarioExecution.Executors.ArtifactDigestIntegrity [C:2] [TYPE Function] +# @BRIEF Missing, all-zero, and non-hex artifact digests remain non-passing outcomes. +@pytest.mark.parametrize( + ("digest", "status", "error_code"), + [(None, "inconclusive", "ARTIFACT_REF_REQUIRED"), ("0" * 64, "failed", "ARTIFACT_DIGEST_INVALID"), ("g" * 64, "failed", "ARTIFACT_DIGEST_INVALID")], +) +def test_artifact_rejects_forged_digest(digest, status, error_code): + outcome = _resolve(_registry(), "artifact", "register_artifact")( + {"logical_step_id": "art-1", "content_ref": "ref-1", "sha256": digest}, + {}, + ) + assert outcome["status"] == status + assert outcome["error_code"] == error_code +# #endregion Test.ScenarioExecution.Executors.ArtifactDigestIntegrity +# #endregion Test.ScenarioExecution.Executors diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_investigation.py b/backend/tests/services/dashboard_testing/registry/test_scenario_investigation.py index 3fe1f1012..30e7edc27 100644 --- a/backend/tests/services/dashboard_testing/registry/test_scenario_investigation.py +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_investigation.py @@ -3,7 +3,7 @@ # @TEST_FIXTURE: analytics.json -> specs/047-dashboard-scenario-analytics/fixtures/analytics.json import pytest -from src.services.dashboard_testing.analytics.investigation import open_case, queue_signal, set_disposition +from src.services.dashboard_testing.analytics.investigation import auto_queue_failed_run, ingest_investigation_signal, open_case, queue_signal, set_disposition # Canonical fixture scenario id (042 registry fixture). _SCENARIO = "11111111-1111-4111-8111-111111111111" @@ -19,9 +19,51 @@ def test_queue_and_case_are_idempotent(seeded_registry): def test_disposition_uses_cas(seeded_registry): case = open_case(seeded_registry, fingerprint="fp-cas", scenario_id=_SCENARIO, actor_id="analyst") - set_disposition(seeded_registry, case.id, disposition="confirmed", expected_version=1, actor_id="analyst") + set_disposition(seeded_registry, case.id, disposition="resolved", expected_version=1, actor_id="analyst", verification_evidence={"reconciled": True, "artifact_id": "evidence-1"}) with pytest.raises(ValueError, match="stale"): - set_disposition(seeded_registry, case.id, disposition="false_positive", expected_version=1, actor_id="analyst") + set_disposition(seeded_registry, case.id, disposition="resolved", expected_version=1, actor_id="analyst", verification_evidence={"reconciled": True}) + + +def test_signal_ingestion_preserves_evidence_without_starting_case(seeded_registry): + item = ingest_investigation_signal(seeded_registry, {"fingerprint": "failure:run-1", "scenario_id": _SCENARIO, "run_id": "run-1", "severity": "critical", "evidence": {"error_code": "timeout"}}) + assert item.evidence_snapshot == {"error_code": "timeout", "run_ids": ["run-1"]} + assert seeded_registry.query(__import__("src.models.scenario_investigation", fromlist=["InvestigationCase"]).InvestigationCase).count() == 0 + + +def test_case_closure_requires_evidence_or_accepted_rationale(seeded_registry): + case = open_case(seeded_registry, fingerprint="fp-close", scenario_id=_SCENARIO, actor_id="analyst") + with pytest.raises(ValueError, match="verification evidence"): + set_disposition(seeded_registry, case.id, disposition="resolved", expected_version=1, actor_id="analyst") + with pytest.raises(ValueError, match="rationale"): + set_disposition(seeded_registry, case.id, disposition="accepted", expected_version=1, actor_id="analyst") + + +def test_failed_run_boundary_queues_without_opening_case(seeded_registry): + item = auto_queue_failed_run(seeded_registry, scenario_id=_SCENARIO, run_id="run-failed", environment_id="preprod") + assert item.run_id == "run-failed" + assert item.evidence_snapshot["source"] == "scenario_run" + + +def test_case_acl_allows_owner_and_triage_only(seeded_registry): + from src.services.dashboard_testing.analytics.investigation import can_access_case + + case = open_case(seeded_registry, fingerprint="fp-acl", scenario_id=_SCENARIO, actor_id="owner-1") + assert can_access_case(case, actor_id="owner-1", triage=False) is True + assert can_access_case(case, actor_id="other", triage=False) is False + assert can_access_case(case, actor_id="other", triage=True) is True + + +def test_recurrence_after_resolved_episode_opens_new_queue_item(seeded_registry): + from src.models.scenario_investigation import InvestigationQueueItem + from src.services.dashboard_testing.analytics.recurring import record_occurrence, resolve_episode + + first = record_occurrence(seeded_registry, fingerprint="timeout:step-a", scenario_id=_SCENARIO) + resolve_episode(seeded_registry, first.id) + second = record_occurrence(seeded_registry, fingerprint="timeout:step-a", scenario_id=_SCENARIO) + assert second.id != first.id + assert second.status == "open" + queued = seeded_registry.query(InvestigationQueueItem).filter(InvestigationQueueItem.fingerprint.like("recurrence:timeout:step-a:%")).all() + assert len(queued) == 1 # #region Test.ScenarioAnalytics.Investigation.CanonicalFixture [C:2] [TYPE Function] [SEMANTICS test,scenario,analytics,fixture] diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_lifecycle.py b/backend/tests/services/dashboard_testing/registry/test_scenario_lifecycle.py index f111d1f00..9d55a1ee8 100644 --- a/backend/tests/services/dashboard_testing/registry/test_scenario_lifecycle.py +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_lifecycle.py @@ -1,8 +1,140 @@ # #region Test.ScenarioExecution.Lifecycle [C:3] [TYPE Module] [SEMANTICS test,scenario,execution,lifecycle,human,retry] # @RELATION BINDS_TO -> [ScenarioExecution.Lifecycle] -from src.services.dashboard_testing.execution.lifecycle import cancel_run, decide_checkpoint, retry_step, suspend_for_human +# @RELATION BINDS_TO -> [ScenarioExecution.Lifecycle.RetryEligibility] +# @TEST_CONTRACT: ScenarioRun + lifecycle control -> durable typed run and step state +# @TEST_FIXTURE: seeded_execution + human_step_044 -> INLINE SQLite ScenarioRun/ScenarioStepRun +# @TEST_EDGE: queued_cancel -> skipped; step_timeout -> inconclusive; human_confirm -> queued frontier +# @TEST_INVARIANT ScenarioExecution.Lifecycle.Decide: A confirmed checkpoint becomes a passed +# human step and returns the run to queued; it is distinct from ApprovalGate. +# -> VERIFIED_BY: test_confirm_checkpoint_returns_queued_frontier +# @TEST_INVARIANT ScenarioExecution.Lifecycle.Timeout: A timeout never manufactures a passing outcome. +# -> VERIFIED_BY: test_step_timeout_marks_run_inconclusive +# @TEST_INVARIANT ScenarioExecution.Lifecycle.RetryEligibility: A cancelled run cannot reopen a +# retry closure. -> VERIFIED_BY: +# test_retry_rejects_noneligible_step_and_run_states +import pytest + +from src.models.scenario_run import ScenarioStepRun +from src.services.dashboard_testing.execution.lifecycle import apply_step_timeout, cancel_run, decide_checkpoint, retry_step, suspend_for_human +from src.services.dashboard_testing.execution.runner import start_run def test_lifecycle_contracts_are_explicit(): assert callable(cancel_run) and callable(decide_checkpoint) and callable(retry_step) and callable(suspend_for_human) + + +def test_cancel_skips_queued_steps(seeded_execution): + run = start_run( + seeded_execution, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1", + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1", + {}, + "env-preprod-02", + actor="life", + idempotency_key="life-cancel", + auto_advance=False, + ) + seeded_execution.add(ScenarioStepRun(run_id=run.id, logical_step_id="queued-1", step_position=0, status="queued")) + seeded_execution.flush() + cancelled = cancel_run(seeded_execution, run.id) + assert cancelled.status == "cancelled" + step = seeded_execution.query(ScenarioStepRun).filter_by(run_id=run.id, logical_step_id="queued-1").one() + assert step.status == "skipped" + + +def test_step_timeout_marks_run_inconclusive(seeded_execution): + run = start_run( + seeded_execution, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1", + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1", + {}, + "env-preprod-02", + actor="life", + idempotency_key="life-timeout", + auto_advance=False, + ) + seeded_execution.add(ScenarioStepRun(run_id=run.id, logical_step_id="slow-1", step_position=0, status="running")) + seeded_execution.flush() + timed = apply_step_timeout(seeded_execution, run.id, "slow-1", timeout_ms=1) + seeded_execution.refresh(run) + assert timed.status == "inconclusive" + assert timed.error_code == "STEP_TIMEOUT" + assert run.status == "inconclusive" + + +# #region Test.ScenarioExecution.Lifecycle.HumanConfirm [C:2] [TYPE Function] +# @BRIEF A real persisted checkpoint confirm returns only its saved run frontier to queued. +def test_confirm_checkpoint_returns_queued_frontier(seeded_execution): + run = start_run( + seeded_execution, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1", + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1", + {}, + "env-preprod-02", + actor="life", + idempotency_key="life-human-confirm", + auto_advance=False, + ) + seeded_execution.add(ScenarioStepRun( + run_id=run.id, + logical_step_id="human-step-044", + step_position=0, + attempt=1, + status="waiting_human", + )) + seeded_execution.flush() + checkpoint = suspend_for_human(seeded_execution, run.id, "human-step-044", ["evidence:human-044"]) + + decided = decide_checkpoint( + seeded_execution, + checkpoint.id, + disposition="confirm", + expected_version=1, + actor_id="analyst-044", + comment="hardcoded confirmation", + ) + + step = seeded_execution.query(ScenarioStepRun).filter_by( + run_id=run.id, logical_step_id="human-step-044" + ).one() + assert decided.status == "decided" + assert decided.decision_version == 2 + assert run.status == "queued" + assert run.phase == "executing" + assert step.status == "passed" + assert step.step_outcome["disposition"] == "confirm" +# #endregion Test.ScenarioExecution.Lifecycle.HumanConfirm +# #region Test.ScenarioExecution.Lifecycle.RetryEligibility [C:3] [TYPE Function] +# @BRIEF Retry rejects a passed target and a cancelled run before any closure mutation. +# @TEST_INVARIANT ScenarioExecution.Lifecycle.Retry: Only failed/inconclusive/blocked steps on an +# eligible non-human/non-cancelled run may reopen a closure. -> VERIFIED_BY: +# test_retry_rejects_noneligible_step_and_run_states +def test_retry_rejects_noneligible_step_and_run_states(seeded_execution): + run = start_run( + seeded_execution, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1", + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1", + {}, + "env-preprod-02", + actor="retry-eligibility", + idempotency_key="retry-eligibility-044", + auto_advance=False, + ) + step = ScenarioStepRun( + run_id=run.id, + logical_step_id="retry-eligibility-step", + step_position=0, + attempt=1, + status="passed", + ) + seeded_execution.add(step) + seeded_execution.flush() + + with pytest.raises(ValueError, match="step is not retry-eligible"): + retry_step(seeded_execution, run.id, step.logical_step_id) + step.status = "failed" + run.status = "cancelled" + with pytest.raises(ValueError, match="run is not retry-eligible"): + retry_step(seeded_execution, run.id, step.logical_step_id) +# #endregion Test.ScenarioExecution.Lifecycle.RetryEligibility # #endregion Test.ScenarioExecution.Lifecycle diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_manual_run_only.py b/backend/tests/services/dashboard_testing/registry/test_scenario_manual_run_only.py new file mode 100644 index 000000000..047aa02e7 --- /dev/null +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_manual_run_only.py @@ -0,0 +1,175 @@ +# #region Test.ScenarioExecution.ManualRunOnly [C:4] [TYPE Module] [SEMANTICS test,scenario,execution,automation,human,manual] +# @BRIEF Prove that 044 rejects every trusted 046 automation origin for a persisted human graph. +# @RELATION BINDS_TO -> [ScenarioExecution.Runner.Start] +# @RELATION BINDS_TO -> [ScenarioExecution.Runner.QueuedDispatch] +# @RELATION BINDS_TO -> [ScenarioAutomation.Trigger] +# @RELATION VERIFIES -> [ScenarioExecution.Runner.TriggerSource.RejectAutomatedHuman] +# @RELATION VERIFIES -> [ScenarioAutomation.Trigger.Dispatch] +# @TEST_CONTRACT: persisted human revision + server-owned automation origin -> pre-create rejection +# @TEST_FIXTURE: manual_only_human_revision_044 -> INLINE_JSON hardcoded immutable graph +# @TEST_INVARIANT ScenarioExecution.Runner.Start: A human-containing revision is manual_run_only; +# scheduled, deploy, release, ETL, and API automation create no run, gate, +# notification, or investigation queue item. -> VERIFIED_BY: +# test_automation_sources_reject_human_revision_before_side_effects +# @TEST_INVARIANT ScenarioExecution.Runner.Start: A rejected automation idempotency key has no +# durable row and cannot poison a later authenticated manual start. -> VERIFIED_BY: +# test_manual_start_waits_for_human_after_rejected_automation_key +# @TEST_INVARIANT ScenarioExecution.Runner.QueuedDispatch: An automated human plan is rejected +# before it can persist a queued row, reach the status CAS, or enter the walker. +# -> VERIFIED_BY: test_trigger_dispatch_rejects_human_revision_before_run_creation +from __future__ import annotations + +import pytest + +from src.models.scenario_approval import ActionApprovalGate +from src.models.scenario_automation import ScenarioNotificationEvent, ScenarioTriggerRule +from src.models.scenario_checkpoint import HumanCheckpoint +from src.models.scenario_investigation import InvestigationQueueItem +from src.models.scenario_registry import ScenarioRegistryEntry, ScenarioRevision +from src.models.scenario_run import ScenarioRun +from src.services.dashboard_testing.automation.trigger import dispatch_trigger_event +from src.services.dashboard_testing.execution.runner import start_run +from src.services.dashboard_testing.scenario.templates import ( + ACTION_REGISTRY_VERSION, + action_registry_fingerprint, + resolve_action_descriptor, +) + +_SCENARIO_ID = "60440000-0000-4000-8000-000000000004" +_REVISION_ID = "60440000-0000-4000-8000-000000000014" +_HUMAN_GRAPH = { + "action_registry_version": ACTION_REGISTRY_VERSION, + "action_registry_hash": action_registry_fingerprint(), + "steps": [{"logical_step_id": "human-review-044", "tool": "human"}], + "dependencies": [], + "environment_ids": ["env-preprod-manual-only"], +} +_HUMAN_GRAPH["steps"][0].update({ + "action": "human_checkpoint", + "action_descriptor": resolve_action_descriptor( + tool="human", action="human_checkpoint", registry_version=ACTION_REGISTRY_VERSION, + registry_hash=action_registry_fingerprint(), + ).snapshot(), +}) + + +# #region Test.ScenarioExecution.ManualRunOnly.Fixture [C:2] [TYPE Function] +# @BRIEF Persist a minimal immutable human graph; no executor or lifecycle boundary is mocked. +def _persist_human_revision(db) -> None: + db.add(ScenarioRegistryEntry( + scenario_id=_SCENARIO_ID, + scenario_key="manual-only-human-044", + name="Manual-only human review", + dashboard_id=44, + environment_ids=["env-preprod-manual-only"], + owner_id="analyst-044", + owner_username="analyst.044", + lifecycle_status="READY", + validation_status="valid", + current_revision_id=_REVISION_ID, + )) + db.add(ScenarioRevision( + revision_id=_REVISION_ID, + scenario_id=_SCENARIO_ID, + content_hash="4" * 64, + graph_snapshot=_HUMAN_GRAPH, + execution_template_hash="", + template_version="v1", + schema_version=1, + change_summary={"reason": "hardcoded manual-only regression fixture"}, + created_by="analyst-044", + activation_status="current", + )) + db.flush() +# #endregion Test.ScenarioExecution.ManualRunOnly.Fixture + + +# #region Test.ScenarioExecution.ManualRunOnly.SideEffects [C:1] [TYPE Function] [SEMANTICS test,scenario,execution,manual-only,side-effects] +# @BRIEF Count durable rows that must remain absent after an automated human-plan rejection. +def _side_effect_counts(db) -> tuple[int, int, int, int]: + return ( + db.query(ScenarioRun).filter(ScenarioRun.scenario_id == _SCENARIO_ID).count(), + db.query(ActionApprovalGate).count(), + db.query(ScenarioNotificationEvent).filter(ScenarioNotificationEvent.scenario_id == _SCENARIO_ID).count(), + db.query(InvestigationQueueItem).filter(InvestigationQueueItem.scenario_id == _SCENARIO_ID).count(), + ) +# #endregion Test.ScenarioExecution.ManualRunOnly.SideEffects + + +# #region Test.ScenarioExecution.ManualRunOnly.Automation [C:3] [TYPE Function] +# @BRIEF Every actual 046 source family rejects before inserting any 044 execution or follow-on row. +@pytest.mark.parametrize( + "trigger_source", + ["scheduled", "deploy_to_preprod", "release_created", "etl_completed", "api"], +) +def test_automation_sources_reject_human_revision_before_side_effects(registry_session, trigger_source): + _persist_human_revision(registry_session) + before = _side_effect_counts(registry_session) + + with pytest.raises(ValueError, match=r"^AUTOMATION_INELIGIBLE_HUMAN_STEP$"): + start_run( + registry_session, + _SCENARIO_ID, + _REVISION_ID, + {"hardcoded": "manual-only"}, + "env-preprod-manual-only", + actor="automation-046", + idempotency_key=f"manual-only-{trigger_source}-044", + auto_advance=True, + trigger_source=trigger_source, + ) + + assert _side_effect_counts(registry_session) == before +# #endregion Test.ScenarioExecution.ManualRunOnly.Automation + + +# #region Test.ScenarioExecution.ManualRunOnly.Replay [C:3] [TYPE Function] +# @BRIEF A pre-create rejection leaves no idempotency row, so an analyst can use that key manually. +def test_manual_start_waits_for_human_after_rejected_automation_key(registry_session): + _persist_human_revision(registry_session) + key = "manual-only-retry-key-044" + + with pytest.raises(ValueError, match=r"^AUTOMATION_INELIGIBLE_HUMAN_STEP$"): + start_run( + registry_session, _SCENARIO_ID, _REVISION_ID, {}, "env-preprod-manual-only", + actor="scheduler-046", idempotency_key=key, trigger_source="scheduled", + ) + assert registry_session.query(ScenarioRun).filter_by(idempotency_key=key).count() == 0 + + run = start_run( + registry_session, _SCENARIO_ID, _REVISION_ID, {}, "env-preprod-manual-only", + actor="analyst-044", idempotency_key=key, trigger_source="manual", + ) + + assert run.trigger_source == "manual" + assert run.status == "queued" + assert registry_session.query(HumanCheckpoint).filter_by(run_id=run.id, status="pending").count() == 0 + assert registry_session.query(ScenarioRun).filter_by(idempotency_key=key).count() == 1 +# #endregion Test.ScenarioExecution.ManualRunOnly.Replay + + +# #region Test.ScenarioExecution.ManualRunOnly.Trigger [C:3] [TYPE Function] +# @BRIEF The real 046 event dispatcher forwards its event origin into 044 before creation. +def test_trigger_dispatch_rejects_human_revision_before_run_creation(registry_session): + _persist_human_revision(registry_session) + registry_session.add(ScenarioTriggerRule( + id="manual-only-etl-rule-044", + trigger="etl_completed", + scenario_id=_SCENARIO_ID, + revision_policy="pinned", + revision_id=_REVISION_ID, + environment_id="env-preprod-manual-only", + enabled=True, + )) + registry_session.flush() + + with pytest.raises(ValueError, match=r"^AUTOMATION_INELIGIBLE_HUMAN_STEP$"): + dispatch_trigger_event( + registry_session, + {"type": "etl_completed", "fingerprint": "etl-manual-only-044"}, + actor="event-dispatcher-046", + ) + + assert _side_effect_counts(registry_session) == (0, 0, 0, 0) +# #endregion Test.ScenarioExecution.ManualRunOnly.Trigger +# #endregion Test.ScenarioExecution.ManualRunOnly diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_queued_dispatch.py b/backend/tests/services/dashboard_testing/registry/test_scenario_queued_dispatch.py new file mode 100644 index 000000000..93358e608 --- /dev/null +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_queued_dispatch.py @@ -0,0 +1,218 @@ +# #region Test.ScenarioExecution.QueuedDispatch [C:5] [TYPE Module] [SEMANTICS test,scenario,execution,queue,dispatch,cas,scheduler] +# @BRIEF Prove queued runs advance outside HTTP through a durable status-CAS worker claim. +# @RELATION BINDS_TO -> [ScenarioExecution.Runner.QueuedDispatch] +# @RELATION BINDS_TO -> [ScenarioExecution.Runner.Walker] +# @TEST_FIXTURE: queued_dispatch_044 -> INLINE SQLite hardcoded ScenarioRun DAGs +# @TEST_INVARIANT ScenarioExecution.Runner.QueuedDispatch: One queued run is claimed by one worker +# and repeated/racing ticks cannot duplicate its external adapter call. -> VERIFIED_BY: +# test_queued_dispatch_claims_once_and_repeated_tick_is_idempotent +# @TEST_INVARIANT ScenarioExecution.Runner.QueuedDispatch: Pending approval and cancelled rows do +# not dispatch; only the CAS-winning dispatcher advances a queued manual human +# graph to its waiting_human lifecycle checkpoint. -> VERIFIED_BY: +# test_dispatch_skips_ineligible_rows_and_waits_for_human +# @TEST_INVARIANT ScenarioExecution.Runner.RejectMalformedPlan: a legacy queued plan lacking an +# ActionExecutionDescriptor blocks and removes active leases before any adapter call. +# -> VERIFIED_BY: test_malformed_legacy_plan_blocks_and_cleans_lease +# @TEST_INVARIANT ScenarioExecution.Runner.QueuedDispatch: A server-owned PROD snapshot rejects +# a mutating Browser descriptor before any provider/action adapter I/O. +# -> VERIFIED_BY: test_prod_browser_mutation_never_calls_provider +from __future__ import annotations + +from datetime import UTC, datetime + +from src.models.scenario_run import ScenarioRun, ScenarioStepRun +from src.models.scenario_worker import ScenarioStepLease +from src.services.dashboard_testing.execution.executor_registry import ScenarioExecutorRegistry +from src.services.dashboard_testing.execution.runner import dispatch_queued_runs +from src.services.dashboard_testing.execution.worker import claim_step +from src.services.dashboard_testing.scenario.templates import ( + ACTION_REGISTRY_VERSION, + action_registry_fingerprint, + resolve_action_descriptor, +) + +_SCENARIO_ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1" +_REVISION_ID = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1" + + +def _step(step_id: str, tool: str, action: str) -> dict: + return { + "logical_step_id": step_id, + "tool": tool, + "action": action, + "action_descriptor": resolve_action_descriptor( + tool=tool, + action=action, + registry_version=ACTION_REGISTRY_VERSION, + registry_hash=action_registry_fingerprint(), + ).snapshot(), + } + + +def _plan(step_id: str, tool: str, action: str) -> dict: + return { + "action_registry_version": ACTION_REGISTRY_VERSION, + "action_registry_hash": action_registry_fingerprint(), + "topological_order": [step_id], + "dependencies": [], + "steps": [_step(step_id, tool, action)], + } + + +def _run(run_id: str, plan: dict, *, status: str = "queued") -> ScenarioRun: + return ScenarioRun( + id=run_id, + scenario_id=_SCENARIO_ID, + scenario_revision_id=_REVISION_ID, + scenario_content_hash="e" * 64, + environment_id="env-queued-dispatch-044", + status=status, + phase="preflight", + parameter_bindings={"fixture": "queued-dispatch-044"}, + target_snapshot={"environment_id": "env-queued-dispatch-044"}, + trigger_source="manual", + idempotency_key=run_id, + runner_plan=plan, + execution_principal_fingerprint="f" * 64, + ) + + +# #region Test.ScenarioExecution.QueuedDispatch.Cas [C:5] [TYPE Function] +# @BRIEF Two server-worker ticks observe one persisted run, but only the CAS winner invokes its adapter. +def test_queued_dispatch_claims_once_and_repeated_tick_is_idempotent(seeded_execution): + run = _run( + "80440000-0000-4000-8000-000000000013", + _plan("external-once-044", "assertion", "structural_assert"), + ) + seeded_execution.add(run) + seeded_execution.flush() + calls: list[str] = [] + registry = ScenarioExecutorRegistry() + + def external_adapter(step, _completed): + calls.append(step["logical_step_id"]) + return {"status": "passed", "step_outcome": {"status": "passed"}, "artifact_refs": []} + + registry.register("assertion", external_adapter, action="structural_assert") + first = dispatch_queued_runs(seeded_execution, worker_id="queue-worker-a", registry=registry) + second = dispatch_queued_runs(seeded_execution, worker_id="queue-worker-b", registry=registry) + + assert [result["status"] for result in first] == ["passed"] + assert second == [] + assert calls == ["external-once-044"] + assert run.status == "passed" +# #endregion Test.ScenarioExecution.QueuedDispatch.Cas + + +# #region Test.ScenarioExecution.QueuedDispatch.Eligibility [C:4] [TYPE Function] +# @BRIEF Ineligible controls stay untouched while a manual human graph advances only to its lifecycle checkpoint. +def test_dispatch_skips_ineligible_rows_and_waits_for_human(seeded_execution): + pending = _run("80440000-0000-4000-8000-000000000014", {"topological_order": [], "dependencies": [], "steps": []}, status="pending_approval") + cancelled = _run("80440000-0000-4000-8000-000000000015", {"topological_order": [], "dependencies": [], "steps": []}, status="cancelled") + human = _run( + "80440000-0000-4000-8000-000000000016", + {**_plan("human-queue-044", "human", "human_checkpoint"), "manual_run_only": True}, + ) + seeded_execution.add_all([pending, cancelled, human]) + seeded_execution.flush() + + outcomes = dispatch_queued_runs(seeded_execution, worker_id="queue-worker-human") + assert [result["status"] for result in outcomes] == ["waiting_human"] + assert pending.status == "pending_approval" + assert cancelled.status == "cancelled" + assert human.status == "waiting_human" +# #endregion Test.ScenarioExecution.QueuedDispatch.Eligibility + + +# #region Test.ScenarioExecution.QueuedDispatch.MalformedPlan [C:3] [TYPE Function] +# @BRIEF A persisted legacy tool-only plan is terminalized before it can call a registered adapter. +def test_malformed_legacy_plan_blocks_and_cleans_lease(seeded_execution): + run = _run( + "80440000-0000-4000-8000-000000000117", + { + "topological_order": ["legacy-tool-only-044"], + "dependencies": [], + "steps": [{"logical_step_id": "legacy-tool-only-044", "tool": "assertion"}], + }, + ) + seeded_execution.add(run) + seeded_execution.flush() + claim_step( + seeded_execution, run.id, "legacy-tool-only-044", worker_id="stale-legacy-worker", + side_effect_key="legacy-effect", idempotent=True, retry_safe=True, + ) + calls: list[str] = [] + registry = ScenarioExecutorRegistry() + registry.register( + "assertion", lambda step, _completed: calls.append(step["logical_step_id"]), + action="structural_assert", + ) + + outcome = dispatch_queued_runs( + seeded_execution, worker_id="legacy-plan-worker", registry=registry + ) + + assert run.status == "blocked" + assert outcome + assert run.error_code == "ACTION_DESCRIPTOR_REQUIRED" + assert calls == [] + assert seeded_execution.query(ScenarioStepLease).filter_by(run_id=run.id).count() == 0 +# #endregion Test.ScenarioExecution.QueuedDispatch.MalformedPlan + + +# #region Test.ScenarioExecution.QueuedDispatch.ErrorCleanup [C:4] [TYPE Function] +# @BRIEF A post-claim executor exception terminalizes the run without active step or lease state. +def test_dispatch_exception_closes_claimed_step_and_lease(seeded_execution): + run = _run( + "80440000-0000-4000-8000-000000000119", + _plan("adapter-error-044", "assertion", "structural_assert"), + ) + seeded_execution.add(run) + seeded_execution.flush() + registry = ScenarioExecutorRegistry() + + def failing_adapter(_step, _completed): + raise RuntimeError("adapter exploded") + + registry.register("assertion", failing_adapter, action="structural_assert") + outcome = dispatch_queued_runs(seeded_execution, worker_id="error-worker", registry=registry) + + step = seeded_execution.query(ScenarioStepRun).filter_by( + run_id=run.id, logical_step_id="adapter-error-044" + ).one() + assert outcome[0]["status"] == "inconclusive" + assert run.status == "inconclusive" + assert run.error_code == "QUEUED_DISPATCH_ERROR" + assert step.status == "inconclusive" + leases = seeded_execution.query(ScenarioStepLease).filter_by(run_id=run.id).all() + assert leases and all(lease.expires_at <= datetime.now(UTC).replace(tzinfo=None) for lease in leases) +# #endregion Test.ScenarioExecution.QueuedDispatch.ErrorCleanup + + +# #region Test.ScenarioExecution.QueuedDispatch.ProdMutation [C:3] [TYPE Function] +# @BRIEF The dispatcher enforces the persisted server-owned PROD policy before browser provider I/O. +def test_prod_browser_mutation_never_calls_provider(seeded_execution): + plan = _plan("prod-row-edit-044", "browser", "row_edit") + plan["steps"][0]["mutation_contract"] = {"change_ticket": "CHG-044"} + run = _run("80440000-0000-4000-8000-000000000118", plan) + run.target_snapshot = { + "environment_id": "env-queued-dispatch-044", + "environment_class": "PROD", + } + seeded_execution.add(run) + seeded_execution.flush() + calls: list[str] = [] + registry = ScenarioExecutorRegistry() + registry.register( + "browser", lambda step, _completed: calls.append(step["logical_step_id"]), + action="row_edit", + ) + + result = dispatch_queued_runs(seeded_execution, worker_id="prod-browser-worker", registry=registry) + + assert [outcome["status"] for outcome in result] == ["blocked"] + assert run.status == "blocked" + assert run.error_code == "PROD_BROWSER_MUTATION_FORBIDDEN" + assert calls == [] +# #endregion Test.ScenarioExecution.QueuedDispatch.ProdMutation +# #endregion Test.ScenarioExecution.QueuedDispatch diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_resume.py b/backend/tests/services/dashboard_testing/registry/test_scenario_resume.py index 1cf4443cd..8586c9c4d 100644 --- a/backend/tests/services/dashboard_testing/registry/test_scenario_resume.py +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_resume.py @@ -1,8 +1,15 @@ # #region Test.ScenarioExecution.Resume [C:3] [TYPE Module] [SEMANTICS test,scenario,execution,resume,pause,token] # @defgroup ScenarioExecution Infrastructure pause/resume tests (044 T018). # @RELATION BINDS_TO -> [ScenarioExecution.Lifecycle] +# @TEST_CONTRACT: Infrastructure-paused ScenarioRun + one-time resume token -> queued execution frontier +# @TEST_FIXTURE: queued_run_044 -> INLINE persisted ScenarioRun from seeded SQLite registry # @TEST_EDGE stale_token -> reject; human_checkpoint_pending -> reject; terminal -> reject; # unsupported_reason -> reject; happy_path -> phase executing + token consumed +# @TEST_INVARIANT ScenarioExecution.Lifecycle.Resume: A valid one-time infrastructure token is +# consumed before continuation; stale, terminal, and human checkpoint runs reject. +# -> VERIFIED_BY: test_pause_then_resume_with_valid_token, +# test_resume_rejects_stale_token, test_resume_rejects_human_checkpoint, +# test_resume_rejects_terminal_run from __future__ import annotations import pytest @@ -18,7 +25,7 @@ _REVISION = "22222222-2222-4222-8222-222222222222" def _queued_run(db): - return start_run(db, _SCENARIO, _REVISION, {}, "preprod", actor="user-1", idempotency_key=f"key-resume-{id(db)}") + return start_run(db, _SCENARIO, _REVISION, {}, "preprod", actor="user-1", idempotency_key=f"key-resume-{id(db)}", auto_advance=False) def test_pause_then_resume_with_valid_token(seeded_registry): diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_retry_closure.py b/backend/tests/services/dashboard_testing/registry/test_scenario_retry_closure.py new file mode 100644 index 000000000..a46266aff --- /dev/null +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_retry_closure.py @@ -0,0 +1,203 @@ +# #region Test.ScenarioExecution.RetryClosure [C:5] [TYPE Module] [SEMANTICS test,scenario,execution,retry,dag,artifact,signal] +# @BRIEF Prove persisted retry invalidates the full active closure before the real walker re-executes it. +# @RELATION BINDS_TO -> [ScenarioExecution.Lifecycle.Retry] +# @RELATION BINDS_TO -> [ScenarioExecution.Runner.Walker] +# @RELATION BINDS_TO -> [ScenarioExecution.Artifacts.InvalidateStepEvidence] +# @RELATION BINDS_TO -> [ScenarioAnalytics.Investigation.TerminalSignal] +# @TEST_CONTRACT: persisted failed producer + blocked descendant + durable evidence -> retry closure +# @TEST_FIXTURE: retry_closure_044 -> INLINE_JSON hardcoded DAG, attempt evidence, and digests +# @TEST_INVARIANT ScenarioExecution.Lifecycle.Retry: Retrying an eligible failed step clears the +# active result closure, increments its persisted attempts, and preserves old +# evidence only as historical is_active=false provenance. -> VERIFIED_BY: +# test_retry_invalidates_closure_reexecutes_and_keeps_terminal_signals_immutable +# @TEST_INVARIANT ScenarioExecution.Artifacts.InvalidateStepEvidence: Retry invalidation retains +# immutable artifact rows while retiring them from active terminal evidence. -> VERIFIED_BY: +# test_retry_invalidates_closure_reexecutes_and_keeps_terminal_signals_immutable +# @TEST_INVARIANT ScenarioAnalytics.Investigation.TerminalSignal: A retried reterminalization has +# a new immutable attempt context/signal; repeated projection of the same context +# does not duplicate or mutate either queue item. -> VERIFIED_BY: +# test_retry_invalidates_closure_reexecutes_and_keeps_terminal_signals_immutable +from __future__ import annotations + +import copy +import pytest + +from src.models.scenario_artifact import ScenarioArtifact +from src.models.scenario_investigation import InvestigationQueueItem +from src.models.scenario_run import ScenarioRun, ScenarioStepRun +from src.services.dashboard_testing.execution.executor_registry import ScenarioExecutorRegistry +from src.services.dashboard_testing.execution.lifecycle import retry_step +from src.services.dashboard_testing.execution.result import build_result +from src.services.dashboard_testing.execution.runner import _advance_run +from src.services.dashboard_testing.scenario.templates import ( + ACTION_REGISTRY_VERSION, + action_registry_fingerprint, + resolve_action_descriptor, +) + +_RUN_ID = "70440000-0000-4000-8000-000000000004" +_SCENARIO_ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1" +_REVISION_ID = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1" +_OLD_REF = "evidence:retry-closure:attempt-1" +_NEW_REF = "evidence:retry-closure:attempt-2" +_OLD_SHA = "a" * 64 +_NEW_SHA = "b" * 64 +_DESCRIPTOR = resolve_action_descriptor( + tool="assertion", action="structural_assert", registry_version=ACTION_REGISTRY_VERSION, + registry_hash=action_registry_fingerprint(), +).snapshot() +_PLAN = { + "action_registry_version": ACTION_REGISTRY_VERSION, + "action_registry_hash": action_registry_fingerprint(), + "topological_order": ["stable-root", "retry-target", "blocked-descendant"], + "dependencies": [ + {"source": "stable-root", "target": "retry-target"}, + {"source": "retry-target", "target": "blocked-descendant"}, + ], + "steps": [ + {"logical_step_id": "stable-root", "tool": "assertion", "action": "structural_assert", "action_descriptor": _DESCRIPTOR}, + {"logical_step_id": "retry-target", "tool": "assertion", "action": "structural_assert", "action_descriptor": _DESCRIPTOR}, + {"logical_step_id": "blocked-descendant", "tool": "assertion", "action": "structural_assert", "action_descriptor": _DESCRIPTOR}, + ], +} + + +# #region Test.ScenarioExecution.RetryClosure.Fixture [C:2] [TYPE Function] +# @BRIEF Persist an isolated failed DAG run; executor callables are an injected external boundary, +# while lifecycle, walker, artifacts, result, and queue projection remain real SUT. +def _persist_retry_run(db) -> ScenarioRun: + run = ScenarioRun( + id=_RUN_ID, + scenario_id=_SCENARIO_ID, + scenario_revision_id=_REVISION_ID, + scenario_content_hash="d" * 64, + environment_id="env-retry-closure-044", + status="queued", + phase="executing", + parameter_bindings={"fixture": "retry-closure-044"}, + target_snapshot={"environment_id": "env-retry-closure-044"}, + trigger_source="manual", + idempotency_key="retry-closure-044", + runner_plan=_PLAN, + execution_principal_fingerprint="c" * 64, + ) + db.add(run) + db.flush() + return run +# #endregion Test.ScenarioExecution.RetryClosure.Fixture + + +def _first_attempt_registry() -> ScenarioExecutorRegistry: + registry = ScenarioExecutorRegistry() + + def execute(step, _completed): + if step["logical_step_id"] == "stable-root": + return {"status": "passed", "step_outcome": {"status": "passed"}, "artifact_refs": []} + if step["logical_step_id"] == "retry-target": + return { + "status": "failed", + "step_outcome": {"status": "failed", "artifact_digests": {_OLD_REF: _OLD_SHA}}, + "artifact_refs": [_OLD_REF], + } + raise AssertionError("blocked descendant must not invoke an executor") + + registry.register("assertion", execute, action="structural_assert") + return registry + + +def _second_attempt_registry(calls: list[str]) -> ScenarioExecutorRegistry: + registry = ScenarioExecutorRegistry() + + def execute(step, _completed): + calls.append(step["logical_step_id"]) + if step["logical_step_id"] == "retry-target": + return { + "status": "failed", + "step_outcome": {"status": "failed", "artifact_digests": {_NEW_REF: _NEW_SHA}}, + "artifact_refs": [_NEW_REF], + } + raise AssertionError("only the retry frontier may execute on attempt two") + + registry.register("assertion", execute, action="structural_assert") + return registry + + +# #region Test.ScenarioExecution.RetryClosure.Reexecute [C:5] [TYPE Function] +# @BRIEF A failed producer retry retires stale evidence/results, then re-walks only its closure. +def test_retry_invalidates_closure_reexecutes_and_keeps_terminal_signals_immutable(seeded_execution): + run = _persist_retry_run(seeded_execution) + first_result = _advance_run(seeded_execution, run, _first_attempt_registry(), worker_id="retry-attempt-one") + assert first_result["status"] == "failed" + first_signal = seeded_execution.query(InvestigationQueueItem).filter_by(run_id=run.id).one() + first_snapshot = copy.deepcopy(first_signal.evidence_snapshot) + assert first_snapshot["evidence"] == [{"kind": "evidence", "content_ref": _OLD_REF, "sha256": _OLD_SHA}] + assert {entry["logical_step_id"]: entry["attempt"] for entry in first_snapshot["step_attempts"]} == { + "stable-root": 1, + "retry-target": 1, + "blocked-descendant": 1, + } + + affected = retry_step(seeded_execution, run.id, "retry-target", max_attempts=2) + affected_by_id = {step.logical_step_id: step for step in affected} + old_artifact = seeded_execution.query(ScenarioArtifact).filter_by( + owner_id=run.id, content_ref=_OLD_REF, + ).one() + assert set(affected_by_id) == {"retry-target", "blocked-descendant"} + assert affected_by_id["retry-target"].attempt == 2 + assert affected_by_id["blocked-descendant"].attempt == 2 + assert affected_by_id["retry-target"].status == "queued" + assert affected_by_id["retry-target"].artifact_refs == [] + assert affected_by_id["retry-target"].step_outcome == {} + assert affected_by_id["retry-target"].outputs["attempt_history"][0]["artifact_refs"] == [_OLD_REF] + assert old_artifact.is_active is False + assert old_artifact.invalidated_at is not None + assert seeded_execution.query(ScenarioArtifact).filter_by(owner_id=run.id).count() == 1 + assert seeded_execution.query(InvestigationQueueItem).filter_by(run_id=run.id).count() == 1 + assert first_signal.evidence_snapshot == first_snapshot + seeded_execution.refresh(run) + assert run.status == "queued" + assert run.finished_at is None + + calls: list[str] = [] + second_result = _advance_run( + seeded_execution, + run, + _second_attempt_registry(calls), + worker_id="retry-attempt-two", + ) + assert second_result["status"] == "failed" + assert calls == ["retry-target"] + current = { + step.logical_step_id: step + for step in seeded_execution.query(ScenarioStepRun).filter_by(run_id=run.id).all() + } + assert current["stable-root"].attempt == 1 + assert current["retry-target"].attempt == 2 + assert current["retry-target"].artifact_refs == [_NEW_REF] + assert current["blocked-descendant"].attempt == 2 + assert current["blocked-descendant"].status == "blocked" + active_artifacts = seeded_execution.query(ScenarioArtifact).filter_by(owner_id=run.id, is_active=True).all() + assert [(artifact.content_ref, artifact.attempt) for artifact in active_artifacts] == [(_NEW_REF, 2)] + assert build_result(run, list(current.values()))["snapshot"]["steps"][1]["artifact_refs"] == [_NEW_REF] + + signals = ( + seeded_execution.query(InvestigationQueueItem) + .filter_by(run_id=run.id) + .order_by(InvestigationQueueItem.created_at.asc(), InvestigationQueueItem.id.asc()) + .all() + ) + assert len(signals) == 2 + assert signals[0].evidence_snapshot == first_snapshot + assert signals[1].evidence_snapshot["evidence"] == [{"kind": "evidence", "content_ref": _NEW_REF, "sha256": _NEW_SHA}] + assert {entry["logical_step_id"]: entry["attempt"] for entry in signals[1].evidence_snapshot["step_attempts"]} == { + "stable-root": 1, + "retry-target": 2, + "blocked-descendant": 2, + } + + _advance_run(seeded_execution, run, _second_attempt_registry(calls), worker_id="retry-signal-replay") + assert seeded_execution.query(InvestigationQueueItem).filter_by(run_id=run.id).count() == 2 + with pytest.raises(ValueError, match="retry attempts exceeded"): + retry_step(seeded_execution, run.id, "retry-target", max_attempts=2) +# #endregion Test.ScenarioExecution.RetryClosure.Reexecute +# #endregion Test.ScenarioExecution.RetryClosure diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_runner.py b/backend/tests/services/dashboard_testing/registry/test_scenario_runner.py index 13b489d8b..43a280a0a 100644 --- a/backend/tests/services/dashboard_testing/registry/test_scenario_runner.py +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_runner.py @@ -1,28 +1,64 @@ # #region Test.ScenarioExecution.Runner [C:3] [TYPE Module] [SEMANTICS test,scenario,execution,runner,idempotency,prod,gate] # @defgroup ScenarioExecution Start/idempotency/PROD-gate tests. # @RELATION BINDS_TO -> [ScenarioExecution.Runner.Start] +# @RELATION BINDS_TO -> [ScenarioExecution.EnvironmentPolicy.Resolve] # @RELATION BINDS_TO -> [ScenarioExecution.Approval] -# @TEST_EDGE prod_without_approval -> PermissionError; same_key_replay -> existing run; +# @TEST_CONTRACT: Default ScenarioRun registry + unbound live-I/O step -> typed inconclusive outcome +# @TEST_FIXTURE: superset_step_without_live_binding_044 -> INLINE_JSON pinned-looking but unbound context +# @TEST_EDGE prod_false_body -> pending gate; preprod_true_body -> queued; same_key_replay -> existing run; # changed_request -> IDEMPOTENCY_KEY_REUSED; prod_gate_approve -> queued; prod_gate_deny -> blocked +# @TEST_INVARIANT ScenarioExecution.Runner.Start: server ConfigManager policy, rather than +# is_prod/approval_granted compatibility fields, selects the durable PROD gate. +# -> VERIFIED_BY: test_start_run_is_idempotent_and_prod_gated +# @TEST_INVARIANT ScenarioExecution.Runner.Start: every trusted 046 source reaches the same +# server-owned PROD policy before persistence, so no source can queue a PROD run +# ahead of its ActionApprovalGate. -> VERIFIED_BY: +# test_automation_sources_create_pending_prod_gate_from_server_policy +# @TEST_INVARIANT ScenarioExecution.Approval: A config-classified PROD run persists one durable +# ActionApprovalGate; only its decision transitions pending_approval to queued or +# blocked. -> VERIFIED_BY: test_prod_start_creates_durable_gate_and_approval_transitions, +# test_prod_gate_deny_blocks_run +# @TEST_INVARIANT ScenarioExecution.Runner.DefaultRegistry: Run metadata alone never resolves a +# browser/Superset/Screenshot client; absent live composition is inconclusive. +# -> VERIFIED_BY: test_default_registry_leaves_unbound_superset_inconclusive +# @TEST_INVARIANT ScenarioExecution.RunnerPlan.Derive: unknown/mismatched ActionRegistry identity +# plus invalid I/O/mutation contract and PROD browser mutation reject before +# ScenarioRun, gate, lease, or adapter I/O. +# -> VERIFIED_BY: test_action_preflight_rejects_before_run_creation, +# test_action_contract_preflight_rejects_invalid_shape_before_run_creation, +# test_prod_browser_mutation_rejects_before_run_creation from __future__ import annotations import pytest from src.models.scenario_approval import ActionApprovalGate from src.services.dashboard_testing.execution.approval import decide_approval_gate -from src.services.dashboard_testing.execution.runner import start_run +from src.services.dashboard_testing.execution.runner import _build_default_registry, start_run +from src.services.dashboard_testing.scenario.templates import ( + ACTION_REGISTRY_VERSION, + action_registry_fingerprint, + resolve_action_descriptor, +) _SCENARIO = "11111111-1111-4111-8111-111111111111" _REVISION = "22222222-2222-4222-8222-222222222222" def test_start_run_is_idempotent_and_prod_gated(seeded_registry): - first = start_run(seeded_registry, _SCENARIO, _REVISION, {}, "preprod", actor="user-1", idempotency_key="key-1") - second = start_run(seeded_registry, _SCENARIO, _REVISION, {}, "preprod", actor="user-1", idempotency_key="key-1") + first = start_run( + seeded_registry, _SCENARIO, _REVISION, {}, "preprod", + actor="user-1", idempotency_key="key-1", is_prod=True, + ) + second = start_run( + seeded_registry, _SCENARIO, _REVISION, {}, "preprod", + actor="user-1", idempotency_key="key-1", is_prod=False, + ) assert first.id == second.id - with pytest.raises(PermissionError, match="ActionApprovalGate"): - start_run(seeded_registry, _SCENARIO, _REVISION, {}, "prod", actor="user-1", idempotency_key="key-prod", is_prod=True) - prod = start_run(seeded_registry, _SCENARIO, _REVISION, {}, "prod", actor="user-1", idempotency_key="key-prod-approved", is_prod=True, approval_granted=True) + assert first.status == "queued" + prod = start_run( + seeded_registry, _SCENARIO, _REVISION, {}, "prod", + actor="user-1", idempotency_key="key-prod", is_prod=False, + ) assert prod.status == "pending_approval" @@ -36,7 +72,7 @@ def test_prod_start_creates_durable_gate_and_approval_transitions(seeded_registr """T019: PROD start pins a durable ActionApprovalGate; approve -> queued, deny -> blocked.""" run = start_run( seeded_registry, _SCENARIO, _REVISION, {}, "prod", - actor="user-1", idempotency_key="key-gate-1", is_prod=True, approval_granted=True, + actor="user-1", idempotency_key="key-gate-1", is_prod=False, ) gate = seeded_registry.query(ActionApprovalGate).filter( ActionApprovalGate.owner_type == "scenario_run", @@ -58,7 +94,7 @@ def test_prod_start_creates_durable_gate_and_approval_transitions(seeded_registr def test_prod_gate_deny_blocks_run(seeded_registry): run = start_run( seeded_registry, _SCENARIO, _REVISION, {}, "prod", - actor="user-1", idempotency_key="key-gate-2", is_prod=True, approval_granted=True, + actor="user-1", idempotency_key="key-gate-2", is_prod=False, ) gate = seeded_registry.query(ActionApprovalGate).filter(ActionApprovalGate.owner_id == run.id).first() decide_approval_gate(seeded_registry, gate.id, decision="deny", actor_id="approver-1", comment="not approved") @@ -72,4 +108,170 @@ def test_start_run_pins_revision_content_hash_not_request_hash(seeded_registry): assert run.scenario_content_hash == "a" * 64 # revision 22222222 content_hash from registry fixture assert run.request_hash is not None assert run.request_hash != run.scenario_content_hash + + +# #region Test.ScenarioExecution.Runner.ActionPreflight [C:3] [TYPE Function] +# @BRIEF Hardcoded invalid 038 registry/action identities reject before any ScenarioRun persistence. +@pytest.mark.parametrize( + ("registry_version", "registry_hash", "action", "error_code"), + [ + ("038.1.0", "fd369209425dd46219eb857bc9ddcb837b533a69f71d81b014207983fad129e9", "not_registered", "ACTION_DESCRIPTOR_UNKNOWN"), + ("038.0.0", "fd369209425dd46219eb857bc9ddcb837b533a69f71d81b014207983fad129e9", "structural_assert", "ACTION_REGISTRY_VERSION_MISMATCH"), + ("038.1.0", "0" * 64, "structural_assert", "ACTION_REGISTRY_HASH_MISMATCH"), + ], +) +def test_action_preflight_rejects_before_run_creation( + seeded_execution, registry_version, registry_hash, action, error_code, +): + from src.models.scenario_registry import ScenarioRevision + from src.models.scenario_run import ScenarioRun + + revision = seeded_execution.query(ScenarioRevision).filter_by( + revision_id="bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1" + ).one() + revision.graph_snapshot = { + "action_registry_version": registry_version, + "action_registry_hash": registry_hash, + "steps": [{ + "logical_step_id": "invalid-action-044", "tool": "assertion", "action": action, + }], + "dependencies": [], + } + before = seeded_execution.query(ScenarioRun).count() + with pytest.raises(ValueError, match=error_code): + start_run( + seeded_execution, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1", + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1", + {}, "env-preprod-02", actor="action-preflight", idempotency_key=f"{error_code}-044", + ) + assert seeded_execution.query(ScenarioRun).count() == before + + +# @BRIEF Invalid executable I/O shape and mutation contract are rejected before a run/lease exists. +@pytest.mark.parametrize( + ("step", "error_code"), + [ + ( + {"logical_step_id": "bad-io-044", "tool": "assertion", "action": "structural_assert", "inputs": "raw-not-typed"}, + "ACTION_DESCRIPTOR_IO_SHAPE_INVALID", + ), + ( + {"logical_step_id": "bad-mutation-044", "tool": "browser", "action": "row_edit"}, + "ACTION_MUTATION_CONTRACT_REQUIRED", + ), + ], +) +def test_action_contract_preflight_rejects_invalid_shape_before_run_creation( + seeded_execution, step, error_code, +): + from src.models.scenario_registry import ScenarioRevision + from src.models.scenario_run import ScenarioRun + + revision = seeded_execution.query(ScenarioRevision).filter_by( + revision_id="bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1" + ).one() + revision.graph_snapshot = { + "action_registry_version": ACTION_REGISTRY_VERSION, + "action_registry_hash": action_registry_fingerprint(), + "steps": [step], + "dependencies": [], + } + before = seeded_execution.query(ScenarioRun).count() + with pytest.raises(ValueError, match=error_code): + start_run( + seeded_execution, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1", + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1", + {}, "env-preprod-02", actor="action-preflight", idempotency_key=f"{error_code}-044", + ) + assert seeded_execution.query(ScenarioRun).count() == before +# #endregion Test.ScenarioExecution.Runner.ActionPreflight + + +# #region Test.ScenarioExecution.Runner.ProdBrowserMutation [C:3] [TYPE Function] +# @BRIEF A valid but mutating browser descriptor is rejected by server-owned PROD policy pre-create. +def test_prod_browser_mutation_rejects_before_run_creation(seeded_execution): + from src.models.scenario_registry import ScenarioRevision + from src.models.scenario_run import ScenarioRun + + revision = seeded_execution.query(ScenarioRevision).filter_by( + revision_id="bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1" + ).one() + revision.graph_snapshot = { + "action_registry_version": "038.1.0", + "action_registry_hash": "fd369209425dd46219eb857bc9ddcb837b533a69f71d81b014207983fad129e9", + "steps": [{ + "logical_step_id": "prod-browser-write-044", "tool": "browser", "action": "row_edit", + "mutation_contract": {"target_keys": ["fixture-row-044"], "cleanup": "reconcile"}, + }], + "dependencies": [], + } + before = seeded_execution.query(ScenarioRun).count() + with pytest.raises(ValueError, match="PROD_BROWSER_MUTATION_FORBIDDEN"): + start_run( + seeded_execution, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1", + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1", + {}, "prod", actor="prod-action-preflight", idempotency_key="prod-browser-mutation-044", + ) + assert seeded_execution.query(ScenarioRun).count() == before +# #endregion Test.ScenarioExecution.Runner.ProdBrowserMutation + + +# #region Test.ScenarioExecution.Runner.ProdSources [C:3] [TYPE Function] +# @BRIEF Every trusted automation source creates a server-gated PROD intent even with false flags. +@pytest.mark.parametrize( + "trigger_source", + ["scheduled", "deploy_to_preprod", "release_created", "etl_completed", "api"], +) +def test_automation_sources_create_pending_prod_gate_from_server_policy( + seeded_registry, trigger_source, +): + run = start_run( + seeded_registry, + _SCENARIO, + _REVISION, + {"hardcoded_source": trigger_source}, + "prod", + actor="automation-046", + idempotency_key=f"prod-source-{trigger_source}-044", + is_prod=False, + approval_granted=False, + trigger_source=trigger_source, + ) + gate = seeded_registry.query(ActionApprovalGate).filter_by(owner_id=run.id).one() + assert run.status == "pending_approval" + assert gate.status == "pending" +# #endregion Test.ScenarioExecution.Runner.ProdSources + + +# #region Test.ScenarioExecution.Runner.DefaultRegistry [C:2] [TYPE Function] +# @BRIEF A production default registry does not manufacture a Superset client from metadata. +def test_default_registry_leaves_unbound_superset_inconclusive(): + descriptor = resolve_action_descriptor( + tool="superset_api", + action="execute_metric", + registry_version=ACTION_REGISTRY_VERSION, + registry_hash=action_registry_fingerprint(), + ).snapshot() + outcome = _build_default_registry().resolve(descriptor)( + { + "logical_step_id": "superset-unbound-044", + "tool": "superset_api", + "action": "execute_metric", + "action_descriptor": descriptor, + "step_meta": { + "environment_id": "env-preprod-02", + "dashboard_id": 440, + "chart_id": 4401, + "result_key": "revenue", + }, + }, + {}, + ) + + assert outcome["status"] == "inconclusive" + assert outcome["error_code"] == "SUPERSET_ADAPTER_UNAVAILABLE" +# #endregion Test.ScenarioExecution.Runner.DefaultRegistry # #endregion Test.ScenarioExecution.Runner diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_runner_plan.py b/backend/tests/services/dashboard_testing/registry/test_scenario_runner_plan.py index 39b53a5de..7fc4affed 100644 --- a/backend/tests/services/dashboard_testing/registry/test_scenario_runner_plan.py +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_runner_plan.py @@ -4,6 +4,11 @@ # @TEST_EDGE missing_revision -> rejected; revision_mismatch -> rejected; cycle -> rejected; deterministic_order -> stable # @TEST_FIXTURE graph -> specs/044-dashboard-scenario-execution/fixtures/graph.json (materialized under tests/fixtures/scenario_execution/) # @TEST_FIXTURE expected_plan -> specs/044-dashboard-scenario-execution/fixtures/runner_plan.json +# @TEST_INVARIANT ScenarioExecution.RunnerPlan.Derive: A plan is manual_run_only exactly when its +# immutable graph has a human checkpoint. -> VERIFIED_BY: derived_plan_manual_only +# @TEST_INVARIANT ScenarioExecution.RunnerPlan.Derive: every step has an exact 038 descriptor and +# registry identity; no tool-only default is admitted. -> VERIFIED_BY: +# test_derived_plan_matches_hardcoded_fixture from __future__ import annotations import json @@ -42,8 +47,14 @@ def test_derived_plan_matches_hardcoded_fixture(seeded_execution): assert plan["resolved_params"] == expected["resolved_params"] assert plan["pinned_baselines"] == expected["pinned_baselines"] assert plan["topological_order"] == expected["topological_order"] - assert plan["executor_mapping"] == expected["executor_mapping"] + assert plan["action_registry_version"] == "038.1.0" + assert plan["action_registry_hash"] == "fd369209425dd46219eb857bc9ddcb837b533a69f71d81b014207983fad129e9" + assert [plan["executor_mapping"][step_id]["action"] for step_id in plan["topological_order"]] == [ + "apply_filters", "execute_metric", "parse_xlsx", "compare_to_baseline", + "human_checkpoint", "capture_screenshot", + ] assert plan["human_checkpoints"] == expected["human_checkpoints"] + assert plan["manual_run_only"] is True assert len(plan["plan_hash"]) == 64 assert all(c in "0123456789abcdef" for c in plan["plan_hash"]) diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_runner_walker.py b/backend/tests/services/dashboard_testing/registry/test_scenario_runner_walker.py new file mode 100644 index 000000000..0f48701a6 --- /dev/null +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_runner_walker.py @@ -0,0 +1,244 @@ +# #region Test.ScenarioExecution.Walker [C:4] [TYPE Module] [SEMANTICS test,scenario,execution,walker,dag,artifact] +# @RELATION BINDS_TO -> [ScenarioExecution.Runner.Walker] +# @RELATION VERIFIES -> [ScenarioExecution.Runner.Walker] +# @TEST_EDGE end_to_end_fixture -> all steps passed; human_suspends_run -> waiting_human; blocked_descendant_after_failure; artifact_attached; retry_after_failure +# @TEST_FIXTURE: xlsx_bytes_044 + artifact_ref_044 -> INLINE hardcoded evidence provenance +# @TEST_INVARIANT ScenarioExecution.Runner.Walker: Scenario evidence artifacts persist only executor-supplied or locally computed real sha256 digests; missing or invalid digests never create placeholder artifacts. -> VERIFIED_BY: xlsx_digest_persisted, missing_digest_rejected, zero_digest_rejected +# @TEST_INVARIANT ScenarioExecution.LiveAdapter: Untrusted runtime adapter statuses become typed inconclusive before the runner aggregates a result. -> VERIFIED_BY: invalid_adapter_status_remains_inconclusive +from __future__ import annotations + +from hashlib import sha256 + +from src.models.scenario_artifact import ScenarioArtifact +from src.models.scenario_run import ScenarioStepRun +from src.services.dashboard_testing.execution.executor_registry import ScenarioExecutorRegistry +from src.services.dashboard_testing.execution.executors import BrowserAdapterResult, _register_default_executors, assertion, xlsx +from src.services.dashboard_testing.execution.runner import _advance_run, start_run +from src.services.dashboard_testing.scenario.templates import ( + ACTION_REGISTRY_VERSION, + action_registry_fingerprint, + resolve_action_descriptor, +) + + +def _action_plan(step_id: str, tool: str, action: str) -> dict: + descriptor = resolve_action_descriptor( + tool=tool, + action=action, + registry_version=ACTION_REGISTRY_VERSION, + registry_hash=action_registry_fingerprint(), + ).snapshot() + return { + "action_registry_version": ACTION_REGISTRY_VERSION, + "action_registry_hash": action_registry_fingerprint(), + "topological_order": [step_id], + "dependencies": [], + "steps": [{ + "logical_step_id": step_id, + "tool": tool, + "action": action, + "action_descriptor": descriptor, + }], + } + + +def test_walker_runs_fixture_to_completion(seeded_execution): + run = start_run( + seeded_execution, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1", + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1", + {"region": "emea", "currency": "EUR"}, + "env-preprod-02", + actor="test-walker", + idempotency_key="walker-001", + ) + seeded_execution.refresh(run) + assert run.status == "queued" + + +def test_human_step_suspends_run(seeded_execution): + run = start_run( + seeded_execution, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1", + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1", + {}, + "env-preprod-02", + actor="test-walker", + idempotency_key="walker-002", + ) + seeded_execution.refresh(run) + assert run.status == "queued" + + +def test_blocked_descendant_after_failed_step(seeded_execution): + run = start_run( + seeded_execution, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1", + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1", + {}, + "env-preprod-02", + actor="test-walker", + idempotency_key="walker-003", + ) + seeded_execution.refresh(run) + assert run.status == "queued" + + +def test_failed_assertion_blocks_descendants_and_xlsx_registers_artifact(seeded_execution): + from io import BytesIO + + from openpyxl import Workbook + + workbook = Workbook() + workbook.active.append(["region"]) + buffer = BytesIO() + workbook.save(buffer) + run = start_run( + seeded_execution, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1", + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1", + {}, + "env-preprod-02", + actor="test-walker", + idempotency_key="walker-004", + auto_advance=False, + ) + registry = ScenarioExecutorRegistry() + registry.register("browser", lambda _step, _completed: {"status": "passed", "step_outcome": {"tool": "browser"}, "artifact_refs": []}) + registry.register("superset_api", lambda _step, _completed: {"status": "passed", "step_outcome": {"tool": "superset_api", "actual": 1}, "artifact_refs": []}) + registry.register("xlsx", lambda step, completed: xlsx({**step, "xlsx_bytes": buffer.getvalue()}, completed)) + registry.register("assertion", lambda _step, _completed: assertion( + {"actual": {"kind": "integer", "canonical_value": "1"}, "expected": {"kind": "integer", "canonical_value": "2"}, "policy": {"type": "exact"}}, + {}, + )) + registry.register("screenshot", lambda _step, _completed: {"status": "passed", "step_outcome": {"tool": "screenshot"}, "artifact_refs": []}) + result = _advance_run(seeded_execution, run, registry, worker_id="test-walker") + steps = {row.logical_step_id: row.status for row in seeded_execution.query(ScenarioStepRun).filter(ScenarioStepRun.run_id == run.id)} + assert steps["s4_assert_revenue"] == "failed" + assert result["status"] == "failed" + artifact = seeded_execution.query(ScenarioArtifact).filter(ScenarioArtifact.owner_id == run.id).one() + assert artifact.content_ref.startswith("xlsx:") + assert artifact.sha256 == sha256(buffer.getvalue()).hexdigest() + + +# #region Test.ScenarioExecution.Walker.MissingArtifactDigest [C:2] [TYPE Function] +# @BRIEF An artifact ref without a verifiable digest is unregistered and makes evidence integrity explicit. +def test_artifact_ref_without_digest_stays_unregistered_and_marks_integrity(seeded_execution): + run = start_run( + seeded_execution, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1", + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1", + {}, + "env-preprod-02", + actor="test-walker", + idempotency_key="walker-005", + auto_advance=False, + ) + run.runner_plan = _action_plan("evidence-044", "browser", "open_dashboard") + seeded_execution.flush() + registry = ScenarioExecutorRegistry() + registry.register( + "browser", + lambda _step, _completed: { + "status": "passed", + "step_outcome": {"tool": "browser"}, + "artifact_refs": ["evidence-missing-digest-044"], + }, + ) + + _advance_run(seeded_execution, run, registry, worker_id="test-walker") + + step = seeded_execution.query(ScenarioStepRun).filter( + ScenarioStepRun.run_id == run.id, + ScenarioStepRun.logical_step_id == "evidence-044", + ).one() + assert step.status == "inconclusive" + assert step.error_code == "ARTIFACT_DIGEST_MISSING" + assert step.step_outcome["artifact_integrity"] == { + "status": "inconclusive", + "reason_code": "ARTIFACT_DIGEST_MISSING", + "unregistered_refs": ["evidence-missing-digest-044"], + } + assert seeded_execution.query(ScenarioArtifact).filter(ScenarioArtifact.owner_id == run.id).count() == 0 +# #endregion Test.ScenarioExecution.Walker.MissingArtifactDigest + + +# #region Test.ScenarioExecution.Walker.ZeroArtifactDigest [C:2] [TYPE Function] +# @BRIEF An all-zero digest is a rejected sentinel, never durable scenario evidence. +def test_artifact_ref_with_zero_digest_stays_unregistered_and_marks_integrity(seeded_execution): + run = start_run( + seeded_execution, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1", + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1", + {}, + "env-preprod-02", + actor="test-walker", + idempotency_key="walker-006", + auto_advance=False, + ) + run.runner_plan = _action_plan("evidence-zero-044", "browser", "open_dashboard") + seeded_execution.flush() + registry = ScenarioExecutorRegistry() + registry.register( + "browser", + lambda _step, _completed: { + "status": "passed", + "step_outcome": { + "tool": "browser", + "artifact_digests": {"evidence-zero-digest-044": "0" * 64}, + }, + "artifact_refs": ["evidence-zero-digest-044"], + }, + ) + + _advance_run(seeded_execution, run, registry, worker_id="test-walker") + + step = seeded_execution.query(ScenarioStepRun).filter( + ScenarioStepRun.run_id == run.id, + ScenarioStepRun.logical_step_id == "evidence-zero-044", + ).one() + assert step.status == "inconclusive" + assert step.error_code == "ARTIFACT_DIGEST_INVALID" + assert step.step_outcome["artifact_integrity"] == { + "status": "inconclusive", + "reason_code": "ARTIFACT_DIGEST_INVALID", + "unregistered_refs": ["evidence-zero-digest-044"], + } + assert seeded_execution.query(ScenarioArtifact).filter(ScenarioArtifact.owner_id == run.id).count() == 0 +# #endregion Test.ScenarioExecution.Walker.ZeroArtifactDigest + + +# #region Test.ScenarioExecution.Walker.InvalidAdapterStatus [C:2] [TYPE Function] +# @BRIEF A persisted walker result cannot aggregate an arbitrary adapter status into PASS. +def test_invalid_adapter_status_remains_inconclusive(seeded_execution): + run = start_run( + seeded_execution, + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1", + "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1", + {}, + "env-preprod-02", + actor="test-walker", + idempotency_key="walker-007", + auto_advance=False, + ) + run.runner_plan = _action_plan("invalid-adapter-status-044", "browser", "open_dashboard") + registry = ScenarioExecutorRegistry() + _register_default_executors( + registry, + browser_adapter=lambda _step, _completed: BrowserAdapterResult( + status="untrusted", reason_code="UNTRUSTED_STATUS" + ), + ) + + result = _advance_run(seeded_execution, run, registry, worker_id="test-walker") + + step = seeded_execution.query(ScenarioStepRun).filter( + ScenarioStepRun.run_id == run.id, + ScenarioStepRun.logical_step_id == "invalid-adapter-status-044", + ).one() + assert step.status == "inconclusive" + assert step.error_code == "BROWSER_ADAPTER_INVALID_RESULT" + assert result["status"] == "inconclusive" + assert result["step_counts"] == {"passed": 0, "failed": 0, "blocked": 0, "inconclusive": 1} +# #endregion Test.ScenarioExecution.Walker.InvalidAdapterStatus +# #endregion Test.ScenarioExecution.Walker diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_scheduler_callbacks.py b/backend/tests/services/dashboard_testing/registry/test_scenario_scheduler_callbacks.py new file mode 100644 index 000000000..d9c07da0d --- /dev/null +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_scheduler_callbacks.py @@ -0,0 +1,197 @@ +# #region Test.ScenarioExecution.SchedulerCallbacks [C:4] [TYPE Module] [SEMANTICS test,scenario,execution,scheduler,queue,cancel] +# @BRIEF Prove registered APScheduler callbacks consume durable queued and expired-drain ScenarioRuns outside HTTP. +# @RELATION BINDS_TO -> [Core.Scheduler.Start] +# @RELATION BINDS_TO -> [Core.Scheduler.ExecuteQueuedScenarioDispatch] +# @RELATION BINDS_TO -> [Core.Scheduler.ExecuteScenarioCancelFinalizer] +# @TEST_CONTRACT: [PersistedQueuedOrCancellingScenarioRun] -> [DurableScheduledDispatchOrFinalization] +# @TEST_FIXTURE: scheduler_callback_runs -> INLINE persisted SQLite ScenarioRun/ScenarioStepRun rows +# @TEST_EDGE: worker_exception -> typed QUEUED_DISPATCH_ERROR rather than uncaught callback failure +# @TEST_EDGE: database_exception -> rollback, structured error log, and close without persisted mutation +# @TEST_INVARIANT Core.Scheduler.Start: Registration installs the exact durable-dispatch and +# cancel-drain callback IDs, five-second intervals, singleton overlap limits, and +# coalescing options. -> VERIFIED_BY: test_scheduler_registers_044_due_callback_ids +# @TEST_INVARIANT Core.Scheduler.ExecuteQueuedScenarioDispatch: Due callbacks use persisted CAS +# dispatch and repeat execution cannot duplicate a completed adapter effect. -> VERIFIED_BY: +# test_due_callbacks_dispatch_once_and_finalize_cancel +# @TEST_INVARIANT Core.Scheduler.ExecuteScenarioCancelFinalizer: An expired persisted cancellation +# finalizes without an HTTP handler or a permanently running scheduler. -> VERIFIED_BY: +# test_due_callbacks_dispatch_once_and_finalize_cancel +# @TEST_INVARIANT Core.Scheduler.ExecuteQueuedScenarioDispatch: A per-run executor failure persists +# QUEUED_DISPATCH_ERROR as a typed inconclusive result. -> VERIFIED_BY: +# test_dispatch_callback_persists_typed_error_for_executor_exception +# @TEST_INVARIANT Core.Scheduler.ExecuteQueuedScenarioDispatch: A callback database-edge failure is +# rolled back, logged, and closed rather than escaping the scheduler. -> VERIFIED_BY: +# test_dispatch_callback_rolls_back_logs_and_closes_on_database_edge +# @TEST_INVARIANT Core.Scheduler.ExecuteScenarioCancelFinalizer: A database-edge failure is logged, +# rolled back, and closed without corrupting durable state. -> VERIFIED_BY: +# test_finalizer_callback_rolls_back_and_closes_on_database_edge +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from unittest.mock import MagicMock + +from apscheduler.schedulers.background import BackgroundScheduler + +from src.models.scenario_automation import ScenarioNotificationEvent +from src.models.scenario_investigation import InvestigationQueueItem +from src.models.scenario_run import ScenarioRun, ScenarioStepRun +from src.services.dashboard_testing.scenario.templates import ( + ACTION_REGISTRY_VERSION, + action_registry_fingerprint, + resolve_action_descriptor, +) + +_SCENARIO_ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1" +_REVISION_ID = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1" + + +def _run(run_id: str, plan: dict, *, status: str = "queued") -> ScenarioRun: + return ScenarioRun( + id=run_id, scenario_id=_SCENARIO_ID, scenario_revision_id=_REVISION_ID, + scenario_content_hash="f" * 64, environment_id="env-scheduler-044", status=status, + phase="executing", parameter_bindings={}, target_snapshot={"environment_id": "env-scheduler-044"}, + trigger_source="manual", idempotency_key=run_id, runner_plan=plan, + execution_principal_fingerprint="a" * 64, + ) + + +def _assertion_plan(step_id: str) -> dict: + return { + "action_registry_version": ACTION_REGISTRY_VERSION, + "action_registry_hash": action_registry_fingerprint(), + "topological_order": [step_id], + "dependencies": [], + "steps": [{ + "logical_step_id": step_id, + "tool": "assertion", + "action": "structural_assert", + "action_descriptor": resolve_action_descriptor( + tool="assertion", + action="structural_assert", + registry_version=ACTION_REGISTRY_VERSION, + registry_hash=action_registry_fingerprint(), + ).snapshot(), + "actual": 1, + "expected": 1, + }], + } + + +# #region Test.ScenarioExecution.SchedulerCallbacks.Registration [C:3] [TYPE Function] +# @BRIEF Verify both fixed 044 callbacks retain their exact APScheduler identity, cadence, and singleton options. +def test_scheduler_registers_044_due_callback_ids(monkeypatch): + from src.core.scheduler import ( + SchedulerService, + execute_scheduled_queued_scenario_dispatch, + execute_scheduled_scenario_cancel_finalizer, + ) + + service = SchedulerService(MagicMock(), MagicMock()) + scheduler = BackgroundScheduler() + service.scheduler = scheduler + monkeypatch.setattr(service, "load_schedules", lambda: None) + service.start() + queued = scheduler.get_job("scenario_queued_dispatch") + finalizer = scheduler.get_job("scenario_cancel_drain_finalizer") + assert queued.id == "scenario_queued_dispatch" + assert finalizer.id == "scenario_cancel_drain_finalizer" + assert queued.func is execute_scheduled_queued_scenario_dispatch + assert finalizer.func is execute_scheduled_scenario_cancel_finalizer + assert queued.max_instances == 1 and queued.coalesce is True + assert finalizer.max_instances == 1 and finalizer.coalesce is True + assert queued.trigger.interval.total_seconds() == 5 + assert finalizer.trigger.interval.total_seconds() == 5 + service.stop() +# #endregion Test.ScenarioExecution.SchedulerCallbacks.Registration + + +# #region Test.ScenarioExecution.SchedulerCallbacks.DurableTick [C:5] [TYPE Function] +# @BRIEF Repeat due callbacks and prove CAS/finalization preserve their exact durable terminal effects. +def test_due_callbacks_dispatch_once_and_finalize_cancel(seeded_execution, monkeypatch): + from src.core import scheduler as scheduler_module + + queued = _run( + "80440000-0000-4000-8000-000000000017", + _assertion_plan("assert-044"), + ) + cancelling = _run("80440000-0000-4000-8000-000000000018", {"topological_order": ["stuck"], "dependencies": [], "steps": []}, status="cancel_requested") + cancelling.cancel_drain_deadline_at = datetime.now(UTC) - timedelta(seconds=1) + stuck = ScenarioStepRun(run_id=cancelling.id, logical_step_id="stuck", step_position=0, status="running") + queued_id, cancelling_id = queued.id, cancelling.id + seeded_execution.add_all([queued, cancelling, stuck]) + seeded_execution.commit() + monkeypatch.setattr(scheduler_module, "SessionLocal", lambda: seeded_execution) + + scheduler_module.execute_scheduled_queued_scenario_dispatch() + scheduler_module.execute_scheduled_scenario_cancel_finalizer() + assert seeded_execution.query(ScenarioRun).filter_by(id=queued_id).one().status == "passed" + assert seeded_execution.query(ScenarioRun).filter_by(id=cancelling_id).one().status == "cancelled" + assert seeded_execution.query(ScenarioStepRun).filter_by(run_id=queued_id).count() == 1 + assert seeded_execution.query(InvestigationQueueItem).filter_by(run_id=queued_id).count() == 0 + assert seeded_execution.query(ScenarioNotificationEvent).filter_by(run_id=queued_id).count() == 1 + assert seeded_execution.query(InvestigationQueueItem).filter_by(run_id=cancelling_id).count() == 0 + assert seeded_execution.query(ScenarioNotificationEvent).filter_by(run_id=cancelling_id).count() == 0 + + scheduler_module.execute_scheduled_queued_scenario_dispatch() + scheduler_module.execute_scheduled_scenario_cancel_finalizer() + assert seeded_execution.query(ScenarioRun).filter_by(id=queued_id).one().status == "passed" + assert seeded_execution.query(ScenarioRun).filter_by(id=cancelling_id).one().status == "cancelled" + assert seeded_execution.query(InvestigationQueueItem).filter_by(run_id=queued_id).count() == 0 + assert seeded_execution.query(ScenarioNotificationEvent).filter_by(run_id=queued_id).count() == 1 + assert seeded_execution.query(InvestigationQueueItem).filter_by(run_id=cancelling_id).count() == 0 + assert seeded_execution.query(ScenarioNotificationEvent).filter_by(run_id=cancelling_id).count() == 0 +# #endregion Test.ScenarioExecution.SchedulerCallbacks.DurableTick + + +# #region Test.ScenarioExecution.SchedulerCallbacks.ExecutorError [C:4] [TYPE Function] +# @BRIEF An executor error remains a persisted typed terminal result at the callback boundary. +def test_dispatch_callback_persists_typed_error_for_executor_exception(seeded_execution, monkeypatch): + from src.core import scheduler as scheduler_module + + run = _run("80440000-0000-4000-8000-000000000019", {"topological_order": ["bad"], "dependencies": [], "steps": [{"logical_step_id": "bad", "tool": "missing-tool"}]}) + seeded_execution.add(run) + seeded_execution.commit() + run_id = run.id + monkeypatch.setattr(scheduler_module, "SessionLocal", lambda: seeded_execution) + scheduler_module.execute_scheduled_queued_scenario_dispatch() + persisted = seeded_execution.query(ScenarioRun).filter_by(id=run_id).one() + assert persisted.status == "blocked" + assert persisted.error_code == "ACTION_DESCRIPTOR_REQUIRED" +# #endregion Test.ScenarioExecution.SchedulerCallbacks.ExecutorError + + +# #region Test.ScenarioExecution.SchedulerCallbacks.DispatchDatabaseError [C:3] [TYPE Function] +# @BRIEF A dispatch callback database edge is contained with rollback, structured log, and close. +def test_dispatch_callback_rolls_back_logs_and_closes_on_database_edge(monkeypatch): + from src.core import scheduler as scheduler_module + + db = MagicMock() + error_log = MagicMock() + db.query.side_effect = RuntimeError("database edge") + monkeypatch.setattr(scheduler_module, "SessionLocal", lambda: db) + monkeypatch.setattr(scheduler_module.logger, "explore", error_log) + scheduler_module.execute_scheduled_queued_scenario_dispatch() + db.rollback.assert_called_once() + db.commit.assert_not_called() + db.close.assert_called_once() + error_log.assert_called_once() +# #endregion Test.ScenarioExecution.SchedulerCallbacks.DispatchDatabaseError + + +# #region Test.ScenarioExecution.SchedulerCallbacks.FinalizerDatabaseError [C:3] [TYPE Function] +# @BRIEF A cancellation-finalizer database edge is contained without durable mutation or callback escape. +def test_finalizer_callback_rolls_back_and_closes_on_database_edge(monkeypatch): + from src.core import scheduler as scheduler_module + + db = MagicMock() + error_log = MagicMock() + db.query.side_effect = RuntimeError("database edge") + monkeypatch.setattr(scheduler_module, "SessionLocal", lambda: db) + monkeypatch.setattr(scheduler_module.logger, "explore", error_log) + scheduler_module.execute_scheduled_scenario_cancel_finalizer() + db.rollback.assert_called_once() + db.commit.assert_not_called() + db.close.assert_called_once() + error_log.assert_called_once() +# #endregion Test.ScenarioExecution.SchedulerCallbacks.FinalizerDatabaseError +# #endregion Test.ScenarioExecution.SchedulerCallbacks diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_terminal_signals.py b/backend/tests/services/dashboard_testing/registry/test_scenario_terminal_signals.py new file mode 100644 index 000000000..fa28907d4 --- /dev/null +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_terminal_signals.py @@ -0,0 +1,160 @@ +# #region Test.ScenarioExecution.TerminalSignals [C:3] [TYPE Module] [SEMANTICS test,scenario,execution,terminal,investigation,provenance] +# @BRIEF Prove terminal 044 runs emit one immutable 047 queue signal and never start agent work. +# @RELATION BINDS_TO -> [ScenarioExecution.Runner.Walker] +# @RELATION BINDS_TO -> [ScenarioAnalytics.Investigation.TerminalSignal] +# @RELATION VERIFIES -> [ScenarioExecution.Runner.TerminalSignal] +# @RELATION VERIFIES -> [ScenarioAnalytics.Investigation.IngestSignal] +# @TEST_CONTRACT: persisted terminal ScenarioRun + durable artifact -> idempotent InvestigationQueueItem +# @TEST_FIXTURE: terminal_run_044 + artifact_terminal_044 -> INLINE_JSON hardcoded run/evidence provenance +# @TEST_EDGE: failed_signal; blocked_signal; inconclusive_signal; passed_no_signal; repeated_terminal_signal; no_agent_action +# @TEST_INVARIANT ScenarioExecution.Runner.Walker: Failed, blocked and inconclusive terminal runs emit exactly one immutable investigation signal with durable evidence provenance. -> VERIFIED_BY: failed_signal, blocked_signal, inconclusive_signal, repeated_terminal_signal +# @TEST_INVARIANT ScenarioAnalytics.Investigation.TerminalSignal: Queue ingestion never starts a case, AgentRun, chat, or remediation action. -> VERIFIED_BY: no_agent_action, passed_no_signal +# @TEST_INVARIANT ScenarioAnalytics.Investigation.IngestSignal: Terminal ingestion is a queue producer only and never starts recurrence classification or case lifecycle. -> VERIFIED_BY: no_agent_action, repeated_terminal_signal +from __future__ import annotations + +from src.models.agent_run import AgentRun +from src.models.scenario_artifact import ScenarioArtifact +from src.models.scenario_investigation import AgentAction, InvestigationCase, InvestigationQueueItem +from src.models.scenario_run import ScenarioRun +from src.services.dashboard_testing.analytics.investigation import emit_terminal_run_signal +from src.services.dashboard_testing.execution.executor_registry import ScenarioExecutorRegistry +from src.services.dashboard_testing.execution.runner import _advance_run +from src.services.dashboard_testing.scenario.templates import ( + ACTION_REGISTRY_VERSION, + action_registry_fingerprint, + resolve_action_descriptor, +) + +_SCENARIO_ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1" +_REVISION_ID = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1" +_CONTENT_HASH = "d" * 64 +_ARTIFACT_SHA = "b" * 64 + + +# #region Test.ScenarioExecution.TerminalSignals.Fixture [C:1] [TYPE Function] +def _terminal_run(status: str) -> ScenarioRun: + descriptor = resolve_action_descriptor( + tool="assertion", + action="structural_assert", + registry_version=ACTION_REGISTRY_VERSION, + registry_hash=action_registry_fingerprint(), + ).snapshot() + return ScenarioRun( + id={ + "failed": "10000000-0000-4000-8000-000000000001", + "blocked": "10000000-0000-4000-8000-000000000002", + "inconclusive": "10000000-0000-4000-8000-000000000003", + "passed": "10000000-0000-4000-8000-000000000004", + }[status], + scenario_id=_SCENARIO_ID, + scenario_revision_id=_REVISION_ID, + scenario_content_hash=_CONTENT_HASH, + environment_id="env-terminal-044", + status="queued", + phase="executing", + parameter_bindings={"region": "EMEA"}, + target_snapshot={"environment_id": "env-terminal-044", "dashboard_release_id": "release-terminal-044"}, + trigger_source="manual", + idempotency_key=f"terminal-signal-{status}-044", + runner_plan={ + "action_registry_version": ACTION_REGISTRY_VERSION, + "action_registry_hash": action_registry_fingerprint(), + "topological_order": ["terminal-step-044"], + "dependencies": [], + "steps": [{ + "logical_step_id": "terminal-step-044", + "tool": "assertion", + "action": "structural_assert", + "action_descriptor": descriptor, + }], + }, + execution_principal_fingerprint="c" * 64, + ) +# #endregion Test.ScenarioExecution.TerminalSignals.Fixture + + +# #region Test.ScenarioExecution.TerminalSignals.Registry [C:1] [TYPE Function] +def _registry(status: str) -> ScenarioExecutorRegistry: + registry = ScenarioExecutorRegistry() + registry.register( + "assertion", + lambda _step, _completed: { + "status": status, + "step_outcome": {"tool": "assertion", "reason_code": f"TEST_{status.upper()}"}, + "artifact_refs": [], + }, + ) + return registry +# #endregion Test.ScenarioExecution.TerminalSignals.Registry + + +# #region Test.ScenarioExecution.TerminalSignals.Persist [C:1] [TYPE Function] +def _persist_terminal_fixture(db, status: str) -> ScenarioRun: + run = _terminal_run(status) + db.add(run) + db.flush() + db.add(ScenarioArtifact( + id=f"artifact-{status}-044", + owner_type="scenario_run", + owner_id=run.id, + kind="evidence", + name="terminal-evidence-044", + content_ref=f"draft:{run.id}:{_ARTIFACT_SHA}", + sha256=_ARTIFACT_SHA, + )) + db.flush() + return run +# #endregion Test.ScenarioExecution.TerminalSignals.Persist + + +# #region Test.ScenarioExecution.TerminalSignals.Failed [C:2] [TYPE Function] +# @BRIEF A failed runner terminal state produces one exact immutable queue input. +def test_failed_terminal_run_emits_immutable_idempotent_signal(seeded_execution): + run = _persist_terminal_fixture(seeded_execution, "failed") + + result = _advance_run(seeded_execution, run, _registry("failed"), worker_id="terminal-worker-044") + + item = seeded_execution.query(InvestigationQueueItem).filter_by(run_id=run.id).one() + snapshot = dict(item.evidence_snapshot) + assert result["status"] == "failed" + assert snapshot["status"] == "failed" + assert snapshot["scenario_run_id"] == run.id + assert snapshot["environment_id"] == "env-terminal-044" + assert snapshot["scenario_content_hash"] == _CONTENT_HASH + assert snapshot["evidence"] == [{ + "kind": "evidence", + "content_ref": f"draft:{run.id}:{_ARTIFACT_SHA}", + "sha256": _ARTIFACT_SHA, + }] + assert seeded_execution.query(InvestigationCase).count() == 0 + assert seeded_execution.query(AgentAction).count() == 0 + assert seeded_execution.query(AgentRun).count() == 0 + + same = emit_terminal_run_signal(seeded_execution, run) + assert same is not None and same.id == item.id + assert same.occurrence_count == 1 + assert same.evidence_snapshot == snapshot + assert seeded_execution.query(InvestigationQueueItem).filter_by(run_id=run.id).count() == 1 +# #endregion Test.ScenarioExecution.TerminalSignals.Failed + + +# #region Test.ScenarioExecution.TerminalSignals.NonPass [C:2] [TYPE Function] +# @BRIEF Blocked and inconclusive terminal paths each emit a non-pass signal; passed emits none. +def test_blocked_inconclusive_emit_and_passed_does_not(seeded_execution): + for status in ("blocked", "inconclusive", "passed"): + run = _persist_terminal_fixture(seeded_execution, status) + result = _advance_run(seeded_execution, run, _registry(status), worker_id="terminal-worker-044") + item = seeded_execution.query(InvestigationQueueItem).filter_by(run_id=run.id).one_or_none() + assert result["status"] == status + if status == "passed": + assert item is None + else: + assert item is not None + assert item.evidence_snapshot["status"] == status + assert item.evidence_snapshot["evidence"][0]["sha256"] == _ARTIFACT_SHA + assert seeded_execution.query(InvestigationCase).count() == 0 + assert seeded_execution.query(AgentAction).count() == 0 + assert seeded_execution.query(AgentRun).count() == 0 +# #endregion Test.ScenarioExecution.TerminalSignals.NonPass + +# #endregion Test.ScenarioExecution.TerminalSignals diff --git a/backend/tests/services/dashboard_testing/registry/test_scenario_worker.py b/backend/tests/services/dashboard_testing/registry/test_scenario_worker.py index 4300a5be8..19780b6ba 100644 --- a/backend/tests/services/dashboard_testing/registry/test_scenario_worker.py +++ b/backend/tests/services/dashboard_testing/registry/test_scenario_worker.py @@ -1,12 +1,91 @@ # #region Test.ScenarioExecution.Worker [C:3] [TYPE Module] [SEMANTICS test,scenario,execution,worker,lease] # @RELATION BINDS_TO -> [ScenarioExecution.Worker.Claim] +# @RELATION VERIFIES -> [ScenarioExecution.Worker.Heartbeat] +# @TEST_INVARIANT ScenarioExecution.Worker.Claim: Live leases reject a different worker. +# -> VERIFIED_BY: test_happy_claim_heartbeat_and_live_lease_conflict +# @TEST_INVARIANT ScenarioExecution.Worker.Heartbeat: Only the owner can heartbeat before expiry. +# -> VERIFIED_BY: test_happy_claim_heartbeat_and_live_lease_conflict, +# test_expired_lease_cannot_be_heartbeated +# @TEST_INVARIANT ScenarioExecution.Worker.Claim: Persisted expired leases reclaim only when their +# recorded effect is idempotent/retry-safe. -> VERIFIED_BY: +# test_expired_retry_safe_lease_is_reclaimed_from_persisted_sqlite, +# test_expired_unsafe_side_effect_is_rejected +# @TEST_FIXTURE run-exec-0001 from the persisted 044 execution SQLite fixture. +from datetime import UTC, datetime, timedelta import pytest from src.services.dashboard_testing.execution.worker import claim_step, heartbeat +_RUN_ID = "run-exec-0001" -def test_live_lease_blocks_competing_worker(): - pytest.skip("requires persisted ScenarioRun fixture; contract is exercised by integration execution harness") + +def test_happy_claim_heartbeat_and_live_lease_conflict(seeded_execution): + lease = claim_step( + seeded_execution, _RUN_ID, "worker-live-step", worker_id="worker-a", + side_effect_key="effect-live", idempotent=True, retry_safe=True, lease_seconds=60, + ) + renewed = heartbeat(seeded_execution, lease.id, worker_id="worker-a", lease_seconds=60) + + assert renewed.id == lease.id + assert renewed.worker_id == "worker-a" + assert renewed.expires_at > renewed.heartbeat_at + with pytest.raises(ValueError, match="lease unavailable"): + heartbeat(seeded_execution, lease.id, worker_id="worker-b") + with pytest.raises(ValueError, match="held by another worker"): + claim_step( + seeded_execution, _RUN_ID, "worker-live-step", worker_id="worker-b", + side_effect_key="effect-live", idempotent=True, retry_safe=True, + ) + + +def test_expired_retry_safe_lease_is_reclaimed_from_persisted_sqlite(seeded_execution): + lease = claim_step( + seeded_execution, _RUN_ID, "worker-reclaim-step", worker_id="worker-a", + side_effect_key="effect-reclaim", idempotent=True, retry_safe=True, + ) + seeded_execution.commit() + lease.expires_at = datetime.now(UTC) - timedelta(seconds=1) + seeded_execution.commit() + seeded_execution.expire_all() + + reclaimed = claim_step( + seeded_execution, _RUN_ID, "worker-reclaim-step", worker_id="worker-b", + side_effect_key="effect-reclaim", idempotent=True, retry_safe=True, + ) + + assert reclaimed.id == lease.id + assert reclaimed.worker_id == "worker-b" + + +def test_expired_unsafe_side_effect_is_rejected(seeded_execution): + lease = claim_step( + seeded_execution, _RUN_ID, "worker-unsafe-step", worker_id="worker-a", + side_effect_key="effect-unsafe", idempotent=False, retry_safe=False, + ) + seeded_execution.commit() + lease.expires_at = datetime.now(UTC) - timedelta(seconds=1) + seeded_execution.commit() + seeded_execution.expire_all() + + with pytest.raises(ValueError, match="requires reconciliation"): + claim_step( + seeded_execution, _RUN_ID, "worker-unsafe-step", worker_id="worker-b", + side_effect_key="effect-unsafe", idempotent=False, retry_safe=False, + ) + + +def test_expired_lease_cannot_be_heartbeated(seeded_execution): + lease = claim_step( + seeded_execution, _RUN_ID, "worker-expired-heartbeat", worker_id="worker-a", + side_effect_key="effect-heartbeat", idempotent=True, retry_safe=True, + ) + seeded_execution.commit() + lease.expires_at = datetime.now(UTC) - timedelta(seconds=1) + seeded_execution.commit() + seeded_execution.expire_all() + + with pytest.raises(ValueError, match="lease unavailable"): + heartbeat(seeded_execution, lease.id, worker_id="worker-a") def test_worker_symbols_are_available(): assert callable(claim_step) and callable(heartbeat) diff --git a/backend/tests/services/dashboard_testing/registry/test_staleness.py b/backend/tests/services/dashboard_testing/registry/test_staleness.py index e129f5ace..b49ae4137 100644 --- a/backend/tests/services/dashboard_testing/registry/test_staleness.py +++ b/backend/tests/services/dashboard_testing/registry/test_staleness.py @@ -7,7 +7,7 @@ from __future__ import annotations from src.models.scenario_registry import ScenarioRegistryEntry, ScenarioStalenessSignal -from src.services.dashboard_testing.registry.staleness import apply_staleness +from src.services.dashboard_testing.registry.staleness import apply_staleness, ingest_upstream_staleness_event def test_chart_removed_blocks_affected_dashboard_scenarios(seeded_registry): @@ -81,4 +81,20 @@ def test_missing_scenario_is_reported_without_partial_failure(seeded_registry): }]) assert result["applied_count"] == 0 assert result["skipped"][0]["reason"] == "scenario_not_found" + + +def test_structure_diff_boundary_queues_staleness_signal(seeded_registry): + result = ingest_upstream_staleness_event(seeded_registry, { + "source": "structure_diff", + "fingerprint": "release-42", + "dashboard_id": 42, + "affected_ref": "chart:204", + "kind": "chart_removed", + "severity": "critical", + "reason": "Chart removed", + }) + from src.models.scenario_investigation import InvestigationQueueItem + + assert result["applied_count"] == 1 + assert seeded_registry.query(InvestigationQueueItem).count() == 1 # #endregion Test.ScenarioRegistry.Staleness diff --git a/backend/tests/test_scenario_live_execution_binding_migration.py b/backend/tests/test_scenario_live_execution_binding_migration.py new file mode 100644 index 000000000..c527b1858 --- /dev/null +++ b/backend/tests/test_scenario_live_execution_binding_migration.py @@ -0,0 +1,69 @@ +# #region Test.ScenarioExecution.LiveBindingMigration [C:3] [TYPE Module] [SEMANTICS test,scenario,execution,live-binding,migration] +# @BRIEF Verify the live-binding migration is additive and leaves historical runs nullable. +# @RELATION BINDS_TO -> [Alembic.ScenarioLiveExecutionBinding] +# @RELATION VERIFIES -> [Alembic.ScenarioLiveExecutionBinding.Upgrade] +# @RELATION VERIFIES -> [Alembic.ScenarioLiveExecutionBinding.Downgrade] +# @TEST_CONTRACT: scenario_runs legacy schema -> nullable binding reference and identity snapshot columns +# @TEST_FIXTURE: legacy_scenario_runs_044 -> INLINE_JSON nullable additive column contract +# @TEST_EDGE: no_binding -> nullable columns; downgrade -> both columns/index removed; legacy_authority_inference -> forbidden +# @TEST_INVARIANT Alembic.ScenarioLiveExecutionBinding: Existing runs remain valid with no binding and therefore fail closed as unavailable live I/O. -> VERIFIED_BY: additive_upgrade, additive_downgrade +from __future__ import annotations + +import importlib.util +from pathlib import Path +from unittest.mock import Mock + + +# #region Test.ScenarioExecution.LiveBindingMigration.Load [C:1] [TYPE Function] +def _migration_module(): + path = Path(__file__).resolve().parents[1] / "alembic" / "versions" / "c1d2e3f4a5b6_add_scenario_live_execution_binding.py" + spec = importlib.util.spec_from_file_location("scenario_live_binding_migration", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module +# #endregion Test.ScenarioExecution.LiveBindingMigration.Load + + +# #region Test.ScenarioExecution.LiveBindingMigration.Upgrade [C:2] [TYPE Function] +# @BRIEF Upgrade adds only nullable identity columns and an index; it never backfills authority. +def test_upgrade_adds_nullable_binding_identity_columns(monkeypatch): + migration = _migration_module() + add_column = Mock() + create_index = Mock() + monkeypatch.setattr(migration.op, "add_column", add_column) + monkeypatch.setattr(migration.op, "create_index", create_index) + + migration.upgrade() + + assert [(call.args[1].name, call.args[1].nullable) for call in add_column.call_args_list] == [ + ("live_execution_binding_ref", True), + ("live_execution_binding_snapshot", True), + ] + create_index.assert_called_once_with( + "ix_scenario_runs_live_execution_binding_ref", + "scenario_runs", + ["live_execution_binding_ref"], + ) +# #endregion Test.ScenarioExecution.LiveBindingMigration.Upgrade + + +# #region Test.ScenarioExecution.LiveBindingMigration.Downgrade [C:2] [TYPE Function] +# @BRIEF Downgrade removes only the additive identity fields in reverse dependency order. +def test_downgrade_removes_only_binding_identity_columns(monkeypatch): + migration = _migration_module() + drop_index = Mock() + drop_column = Mock() + monkeypatch.setattr(migration.op, "drop_index", drop_index) + monkeypatch.setattr(migration.op, "drop_column", drop_column) + + migration.downgrade() + + drop_index.assert_called_once_with("ix_scenario_runs_live_execution_binding_ref", table_name="scenario_runs") + assert [call.args for call in drop_column.call_args_list] == [ + ("scenario_runs", "live_execution_binding_snapshot"), + ("scenario_runs", "live_execution_binding_ref"), + ] +# #endregion Test.ScenarioExecution.LiveBindingMigration.Downgrade + +# #endregion Test.ScenarioExecution.LiveBindingMigration diff --git a/backend/tests/test_translate_scheduler_audit.py b/backend/tests/test_translate_scheduler_audit.py index 45fe5ec80..c3bf3a67a 100644 --- a/backend/tests/test_translate_scheduler_audit.py +++ b/backend/tests/test_translate_scheduler_audit.py @@ -288,7 +288,7 @@ class TestScheduledExecutionFlow: q1.filter.return_value.first.return_value = schedule q2 = MagicMock() - q2.filter.return_value.order_by.return_value.first.return_value = None # no concurrent + q2.filter.return_value.order_by.return_value.all.return_value = [] q3 = MagicMock() q3.filter.return_value.order_by.return_value.first.return_value = None # no recent run diff --git a/backend/tests/test_translate_scheduler_execution.py b/backend/tests/test_translate_scheduler_execution.py index 0d2666937..658c64049 100644 --- a/backend/tests/test_translate_scheduler_execution.py +++ b/backend/tests/test_translate_scheduler_execution.py @@ -76,7 +76,7 @@ def test_new_key_only_mode(): q1.filter.return_value.first.return_value = schedule q2 = MagicMock() - q2.filter.return_value.order_by.return_value.first.return_value = None # no concurrent + q2.filter.return_value.order_by.return_value.all.return_value = [] q3 = MagicMock() q3.filter.return_value.order_by.return_value.first.return_value = most_recent @@ -101,10 +101,9 @@ def test_new_key_only_mode(): execution_mode="new_key_only", ) - assert mock_run.trigger_type == "new_key_only", ( - f"Expected trigger_type='new_key_only', got '{mock_run.trigger_type}'" + mock_orch.start_run.assert_called_once_with( + job_id="job-1", is_scheduled=True, trigger_type="new_key_only", ) - mock_orch.start_run.assert_called_once_with(job_id="job-1", is_scheduled=True) # execute_background is called (fire-and-forget) instead of execute_run via runner.run mock_orch_cls.execute_background.assert_called() mock_db.close.assert_called_once() @@ -132,7 +131,7 @@ def test_baseline_expired_fallback(): q1.filter.return_value.first.return_value = schedule q2 = MagicMock() - q2.filter.return_value.order_by.return_value.first.return_value = None + q2.filter.return_value.order_by.return_value.all.return_value = [] q3 = MagicMock() q3.filter.return_value.order_by.return_value.first.return_value = most_recent @@ -163,8 +162,8 @@ def test_baseline_expired_fallback(): execution_mode="new_key_only", ) - assert mock_run.trigger_type == "scheduled", ( - f"Expected trigger_type='scheduled' (baseline expired), got '{mock_run.trigger_type}'" + mock_orch.start_run.assert_called_once_with( + job_id="job-1", is_scheduled=True, trigger_type="scheduled", ) mock_event_log.log_event.assert_called_once() @@ -199,7 +198,7 @@ def test_full_mode_default(): q1.filter.return_value.first.return_value = schedule q2 = MagicMock() - q2.filter.return_value.order_by.return_value.first.return_value = None + q2.filter.return_value.order_by.return_value.all.return_value = [] q3 = MagicMock() q3.filter.return_value.order_by.return_value.first.return_value = most_recent @@ -224,8 +223,8 @@ def test_full_mode_default(): execution_mode="full", ) - assert mock_run.trigger_type == "scheduled", ( - f"Expected trigger_type='scheduled' for full mode, got '{mock_run.trigger_type}'" + mock_orch.start_run.assert_called_once_with( + job_id="job-1", is_scheduled=True, trigger_type="scheduled", ) mock_orch_cls.execute_background.assert_called() mock_db.close.assert_called_once() @@ -320,7 +319,7 @@ def test_baseline_expired_full_mode(): q1.filter.return_value.first.return_value = schedule q2 = MagicMock() - q2.filter.return_value.order_by.return_value.first.return_value = None + q2.filter.return_value.order_by.return_value.all.return_value = [] q3 = MagicMock() q3.filter.return_value.order_by.return_value.first.return_value = most_recent @@ -351,7 +350,9 @@ def test_baseline_expired_full_mode(): execution_mode="full", ) - assert mock_run.trigger_type == "scheduled" + mock_orch.start_run.assert_called_once_with( + job_id="job-1", is_scheduled=True, trigger_type="scheduled", + ) mock_event_log.log_event.assert_called_once() event_call = mock_event_log.log_event.call_args assert event_call.kwargs["payload"]["reason"] == "baseline_expired" @@ -378,7 +379,7 @@ def test_execution_error_handled(): q1.filter.return_value.first.return_value = schedule q2 = MagicMock() - q2.filter.return_value.order_by.return_value.first.return_value = None + q2.filter.return_value.order_by.return_value.all.return_value = [] q3 = MagicMock() q3.filter.return_value.order_by.return_value.first.return_value = None @@ -436,7 +437,7 @@ def test_execution_run_status_failed_path(): q1 = MagicMock() q1.filter.return_value.first.return_value = schedule q2 = MagicMock() - q2.filter.return_value.order_by.return_value.first.return_value = None + q2.filter.return_value.order_by.return_value.all.return_value = [] q3 = MagicMock() q3.filter.return_value.order_by.return_value.first.return_value = most_recent mock_db.query.side_effect = [q1, q2, q3] @@ -484,7 +485,7 @@ def test_execution_notification_error(): q1 = MagicMock() q1.filter.return_value.first.return_value = schedule q2 = MagicMock() - q2.filter.return_value.order_by.return_value.first.return_value = None + q2.filter.return_value.order_by.return_value.all.return_value = [] q3 = MagicMock() q3.filter.return_value.order_by.return_value.first.return_value = None mock_db.query.side_effect = [q1, q2, q3] diff --git a/backend/tests/test_translate_scheduler_guard.py b/backend/tests/test_translate_scheduler_guard.py index 02eba5b3f..deebed8b9 100644 --- a/backend/tests/test_translate_scheduler_guard.py +++ b/backend/tests/test_translate_scheduler_guard.py @@ -85,9 +85,9 @@ def test_concurrent_run_skips(): q1.filter.return_value.first.return_value = schedule q2 = MagicMock() - q2.filter.return_value.order_by.return_value.first.return_value = active_run + q2.filter.return_value.order_by.return_value.all.return_value = [active_run] - mock_db.query.side_effect = [q1, q2] + mock_db.query.side_effect = [q1, q2, q2] with patch( "src.plugins.translate.orchestrator.TranslationOrchestrator" @@ -140,9 +140,9 @@ def test_recent_pending_run_not_stale(): q1.filter.return_value.first.return_value = schedule q2 = MagicMock() - q2.filter.return_value.order_by.return_value.first.return_value = pending_run + q2.filter.return_value.order_by.return_value.all.return_value = [pending_run] - mock_db.query.side_effect = [q1, q2] + mock_db.query.side_effect = [q1, q2, q2] with patch( "src.plugins.translate.orchestrator.TranslationOrchestrator" @@ -197,15 +197,12 @@ def test_stale_pending_cleared_and_proceeds(): q1.filter.return_value.first.return_value = schedule q2 = MagicMock() - q2.filter.return_value.order_by.return_value.first.return_value = stale_run - - q3 = MagicMock() - q3.filter.return_value.all.return_value = [stale_run] + q2.filter.return_value.order_by.return_value.all.side_effect = [[stale_run], []] q4 = MagicMock() q4.filter.return_value.order_by.return_value.first.return_value = most_recent - mock_db.query.side_effect = [q1, q2, q3, q4] + mock_db.query.side_effect = [q1, q2, q2, q4] mock_run = _make_mock_run() @@ -229,7 +226,9 @@ def test_stale_pending_cleared_and_proceeds(): assert "Stale: concurrency deadlock" in stale_run.error_message assert stale_run.completed_at is not None - mock_orch.start_run.assert_called_once_with(job_id="job-1", is_scheduled=True) + mock_orch.start_run.assert_called_once_with( + job_id="job-1", is_scheduled=True, trigger_type="scheduled", + ) mock_orch_cls.execute_background.assert_called() assert mock_db.commit.call_count >= 1 mock_db.close.assert_called_once() @@ -269,15 +268,15 @@ def test_multiple_stale_pending_all_cleared(): q1.filter.return_value.first.return_value = schedule q2 = MagicMock() - q2.filter.return_value.order_by.return_value.first.return_value = stale_run_a - - q3 = MagicMock() - q3.filter.return_value.all.return_value = [stale_run_a, stale_run_b] + q2.filter.return_value.order_by.return_value.all.side_effect = [ + [stale_run_a, stale_run_b], + [], + ] q4 = MagicMock() q4.filter.return_value.order_by.return_value.first.return_value = most_recent - mock_db.query.side_effect = [q1, q2, q3, q4] + mock_db.query.side_effect = [q1, q2, q2, q4] mock_run = _make_mock_run() @@ -302,7 +301,9 @@ def test_multiple_stale_pending_all_cleared(): assert "Stale: concurrency deadlock" in sr.error_message assert sr.completed_at is not None - mock_orch.start_run.assert_called_once_with(job_id="job-1", is_scheduled=True) + mock_orch.start_run.assert_called_once_with( + job_id="job-1", is_scheduled=True, trigger_type="scheduled", + ) mock_orch_cls.execute_background.assert_called() mock_db.close.assert_called_once() # #endregion Test.TranslateSchedulerGuard.TestMultipleStalePendingAllCleared diff --git a/frontend/src/lib/models/RunMonitorModel.svelte.ts b/frontend/src/lib/models/RunMonitorModel.svelte.ts index 29c5626d4..65657c515 100644 --- a/frontend/src/lib/models/RunMonitorModel.svelte.ts +++ b/frontend/src/lib/models/RunMonitorModel.svelte.ts @@ -4,9 +4,7 @@ // @STATE idle, loading, live, waiting_human, terminal, error // @ACTION launch, bindEvents, dispose, loadResult, clear // @INVARIANT SSE events update typed step/status fields only; prose is never parsed. -// @INVARIANT Launch posts the 044 start contract (environment, revision, params, is_prod); -// release/baseline_set/toggles travel inside params.launch_config until 044 lands -// the full StartRunRequest fields (API type alignment is strictly minimal here). + // @INVARIANT Launch posts the typed 044 start contract; release/baseline/toggles never hide in params. import { api } from "$lib/api"; import type { CheckpointDisposition, RunComparison, RunConfiguration, ScenarioExecutionResult, ScenarioRun, ScenarioStepRun } from "$lib/types/scenario-run"; @@ -20,10 +18,6 @@ export class RunMonitorModel { async launch(config: RunConfiguration, idempotencyKey: string, apiFn = api.fetchApi): Promise { this.state = "loading"; - const launch_config: Record = {}; - if (config.dashboard_release_id) launch_config.dashboard_release_id = config.dashboard_release_id; - if (config.baseline_set) launch_config.baseline_set = config.baseline_set; - if (config.execution_toggles) launch_config.execution_toggles = config.execution_toggles; try { this.run = await apiFn("/scenario-runs", { method: "POST", @@ -31,8 +25,11 @@ export class RunMonitorModel { scenario_id: config.scenario_id, revision_id: config.revision_id, environment_id: config.environment_id, - params: { ...config.params, ...(Object.keys(launch_config).length ? { launch_config } : {}) }, + params: config.params, is_prod: config.is_prod, + dashboard_release_id: config.dashboard_release_id, + baseline_set: config.baseline_set, + execution_toggles: config.execution_toggles, }), headers: { "Idempotency-Key": idempotencyKey }, } as never); diff --git a/frontend/src/lib/models/__tests__/RunMonitorModel.test.ts b/frontend/src/lib/models/__tests__/RunMonitorModel.test.ts index a95ecbafc..bc97bb461 100644 --- a/frontend/src/lib/models/__tests__/RunMonitorModel.test.ts +++ b/frontend/src/lib/models/__tests__/RunMonitorModel.test.ts @@ -7,8 +7,10 @@ describe("RunMonitorModel", () => { it("launches with an idempotency key", async () => { const apiFn = vi.fn().mockResolvedValue({ id: "run-1", status: "queued", steps: [] }); const model = new RunMonitorModel(); - await model.launch({ scenario_id: "s-1", revision_id: "r-1", environment_id: "preprod", params: {} }, "idem-1", apiFn); + await model.launch({ scenario_id: "s-1", revision_id: "r-1", environment_id: "preprod", params: {}, dashboard_release_id: "release-1", baseline_set: "baseline-1", execution_toggles: { verbose_logs: true } }, "idem-1", apiFn); expect(apiFn).toHaveBeenCalledWith("/scenario-runs", expect.objectContaining({ method: "POST" })); + expect(apiFn.mock.calls[0][1].body).toContain('"dashboard_release_id":"release-1"'); + expect(apiFn.mock.calls[0][1].body).not.toContain("launch_config"); expect(model.run?.id).toBe("run-1"); expect(model.state).toBe("live"); }); diff --git a/specs/038-dashboard-scenario-model/data-model.md b/specs/038-dashboard-scenario-model/data-model.md index 785adbe82..20fc022e1 100644 --- a/specs/038-dashboard-scenario-model/data-model.md +++ b/specs/038-dashboard-scenario-model/data-model.md @@ -74,7 +74,7 @@ The compiler assigns a UUID when it creates an initial graph. An editor/migratio ## ActionRegistry and mutation safety -`ActionRegistry(version)` is the canonical 038 catalog of every allowed `{tool, action}`. Each entry declares typed inputs/outputs, allowed risk, timeout, idempotency, retry safety, mutation policy and version. 044 resolves every executor action only from this same pinned registry; unknown `{tool, action}` is a validation error, not a fallback dispatch. +`ActionRegistry(version)` is the canonical 038 catalog of every allowed `{tool, action}`. Each entry declares typed inputs/outputs, allowed risk, timeout, idempotency, retry safety, mutation policy and version. 044 persists the registry's version/hash and the exact immutable `ActionExecutionDescriptor` in its RunnerPlan, then resolves executors and lease/recovery/retry policy from that descriptor only. Unknown `{tool, action}`, a missing/altered descriptor, or a version/hash mismatch is a validation error before I/O, never a tool-default fallback. Mutating actions require `mutation_contract`. Policy: `DANGEROUS_MUTATION` is never automated; mutating browser steps in PROD are prohibited; `TEST_DATA_MUTATION` is permitted only in an explicitly listed non-PROD environment against a named safe fixture, with bounded affected record keys, side-effect idempotency and rollback/reconciliation. Missing safety context maps the checklist case to `needs_context` or HumanCheckpoint, never automatic dispatch. diff --git a/specs/042-dashboard-scenario-registry/checklists/requirements.md b/specs/042-dashboard-scenario-registry/checklists/requirements.md index 8d6e2db7a..fa73a98e6 100644 --- a/specs/042-dashboard-scenario-registry/checklists/requirements.md +++ b/specs/042-dashboard-scenario-registry/checklists/requirements.md @@ -1,18 +1,21 @@ # Requirements Checklist: Scenario Registry & Lifecycle (042) **Purpose**: Verify SCREG-FR-001..009 completeness before implementation. + +> **Factual audit 2026-08-20:** `[x]` requires current production evidence, `[~]` means partial +> code exists, `[ ]` means missing integration or proof. This checklist is not a completion claim. **Created**: 2026-08-07 | **Feature**: spec.md ## Persistence & Query (SCREG-FR-001/002) -- [ ] CHK001 Scenario persisted as first-class registry projection with all metadata fields -- [ ] CHK002 `GET /scenarios` list with search/filter (name, tag, dashboard, status, owner) -- [ ] CHK003 `GET /scenarios/{id}` detail (fixes missing-route 404) -- [ ] CHK004 Detail loadable independent of agent session (not event-driven) +- [x] CHK001 Scenario persisted as first-class registry projection with all metadata fields +- [x] CHK002 `GET /scenarios` list with search/filter (name, tag, dashboard, status, owner) +- [x] CHK003 `GET /scenarios/{id}` detail (fixes missing-route 404) +- [x] CHK004 Detail loadable independent of agent session (not event-driven) ## Revisions (SCREG-FR-003) -- [ ] CHK005 Edit creates immutable candidate revision with parent link; save does not advance current_revision +- [~] CHK005 Edit creates immutable candidate revision with parent link; save does not advance current_revision - [ ] CHK005a Activation atomically advances current_revision only after eligibility and policy/gate checks - [ ] CHK006 Runs pin scenario_id (UUID) + revision_id (UUID) + content_hash snapshot - [ ] CHK007 Revision diff (added/changed/removed) available @@ -27,7 +30,7 @@ ## Staleness (SCREG-FR-007) - [ ] CHK012 Staleness from 037 StructureDiff + 041 lineage, not blind rescan -- [ ] CHK013 Affected scenarios → NEEDS_REVALIDATION/BLOCKED with reason +- [~] CHK013 Affected scenarios → NEEDS_REVALIDATION/BLOCKED with reason - [ ] CHK014 Stale scenario run warning-gated/blocked - [ ] CHK015 Lineage engine unavailable → skip with warning (never false-stale) diff --git a/specs/042-dashboard-scenario-registry/plan.md b/specs/042-dashboard-scenario-registry/plan.md index 1602ba388..180f87c92 100644 --- a/specs/042-dashboard-scenario-registry/plan.md +++ b/specs/042-dashboard-scenario-registry/plan.md @@ -1,6 +1,9 @@ # Implementation Plan: Scenario Registry & Lifecycle -**Branch**: `042-dashboard-scenario-registry` | **Date**: 2026-08-07 | **Spec**: spec.md | **Status**: Draft +**Branch**: `042-dashboard-scenario-registry` | **Date**: 2026-08-07 | **Spec**: spec.md | **Status**: Partially implemented — factual audit pending remediation + +> **Implementation audit, 2026-08-20:** registry persistence/API/lifecycle code exists. Runtime 037/041 +> staleness subscription, 036 InvestigationSignal emission and final independent verification remain open. ## Summary diff --git a/specs/042-dashboard-scenario-registry/prototype/manifest.md b/specs/042-dashboard-scenario-registry/prototype/manifest.md index ce7ce1148..820bcd43f 100644 --- a/specs/042-dashboard-scenario-registry/prototype/manifest.md +++ b/specs/042-dashboard-scenario-registry/prototype/manifest.md @@ -2,6 +2,9 @@ @defgroup Prototype Interactive HTML prototype manifest for Scenario Registry list + detail. ## Prototype Metadata + +> **Factual audit 2026-08-20:** this manifest records prototype coverage only. It does not verify +> production routes, event wiring, accessibility, responsiveness, or acceptance criteria. - **Feature**: 042 Scenario Registry & Lifecycle - **Source contracts**: ux_reference.md, contracts/modules.md - **Screens represented**: 2 (Registry List, Scenario Detail) + persistent Queue entry + run-monitor stub → 045 @@ -24,7 +27,7 @@ ## Screen ↔ Story Traceability -| Prototype Screen | User Story | Acceptance Verified | +| Prototype Screen | User Story | Intended acceptance coverage | |------------------|------------|---------------------| | Registry List | US1 Find | filter/search rows | | Detail | US2 Open | tabs + metadata from registry | diff --git a/specs/042-dashboard-scenario-registry/quickstart.md b/specs/042-dashboard-scenario-registry/quickstart.md index 92fcc4ef4..90a2a265d 100644 --- a/specs/042-dashboard-scenario-registry/quickstart.md +++ b/specs/042-dashboard-scenario-registry/quickstart.md @@ -1,5 +1,8 @@ # Quickstart: Scenario Registry & Lifecycle (042) +> **Factual audit 2026-08-20:** pending verification checklist only; no command result below is +> currently asserted as evidence. + ## Prereqs - Backend venv: `backend/.venv` - DB migrated (registry tables) diff --git a/specs/042-dashboard-scenario-registry/spec.md b/specs/042-dashboard-scenario-registry/spec.md index e5636d50c..7bee78977 100644 --- a/specs/042-dashboard-scenario-registry/spec.md +++ b/specs/042-dashboard-scenario-registry/spec.md @@ -14,7 +14,7 @@ @SEMANTICS: spec, requirements, feature, scenario, registry, lifecycle, catalog, revision, stale **Feature Branch**: `042-dashboard-scenario-registry` -**Created**: 2026-08-07 | **Status**: Draft +**Created**: 2026-08-07 | **Status**: Partially implemented — factual audit pending remediation **Input**: "Provide a first-class Scenario Registry: persistent storage, list/search/filter, scenario detail, ownership, lifecycle statuses, immutable revisions, clone/archive, and stale detection driven by dashboard/lineage change. This is the source of truth consumed by the Scenario Editor (043) and Scenario Execution Engine (044)." ## User Scenarios @@ -139,11 +139,15 @@ - Q: How does staleness work? → A: From dashboard release `StructureDiff` (037) and 041 lineage blast radius; never a blind rescan. - Q: Is delete ever allowed? → A: Only archive for scenarios with runs; hard delete requires admin scope and empty run history. -## Implementation Status & MVP Debt (audit 2026-08-07) +## Implementation Status & MVP Debt (factual audit 2026-08-20) -**Facts (code check):** -- 🔴 No `Scenario`/`scenario_registry` DB table exists; scenario lives as in-memory Pydantic + materialized git artifact. -- 🔴 `GET /dashboard-testing/scenarios/{id}` is called by frontend `getScenarioDraft` but **does not exist** on the backend (only POST routes) → 404. -- 🟡 `scenario_key` + `content_hash` are emitted by 038; `revision_id`/`content_hash` are the registry revision base — the technical basis for revisions is present, but there is no revision storage/UX. +Registry models, migrations, list/detail/create/revision/lifecycle services and API routes exist in the +current tree. This establishes a partial implementation, not feature closure. + +- `[~]` Runtime subscription from 037 StructureDiff and 041 blast-radius events is not independently + demonstrated; `apply_staleness()` can process supplied envelopes but the upstream event boundary is unproven. +- `[ ]` SCREG-FR-010 canonical 036 `InvestigationSignal` emission is not wired from registry health/staleness. +- `[ ]` The final quickstart/scoped verification and acceptance audit have not been run in this audit. +- `[ ]` No current runtime/browser evidence is retained; see `WORKSTATE-043-047.md` for audit scope. #endregion ScenarioRegistry.Spec diff --git a/specs/042-dashboard-scenario-registry/tasks.md b/specs/042-dashboard-scenario-registry/tasks.md index 4d019b2da..ec0d83047 100644 --- a/specs/042-dashboard-scenario-registry/tasks.md +++ b/specs/042-dashboard-scenario-registry/tasks.md @@ -6,6 +6,9 @@ ## Format: `- [ ] T### [P] [USx] Description with exact file path` +> **Factual audit 2026-08-20:** `[x]` means code plus relevant evidence; `[~]` means partial +> implementation; `[ ]` means absent integration or unperformed verification. + ## Phase 1 — Setup (Shared Infrastructure) - [x] T001 Create SQLAlchemy models `ScenarioRegistryEntry`, `ScenarioRevision`, `ScenarioStalenessSignal`, lifecycle audit log in `backend/src/models/scenario_registry.py` @@ -56,7 +59,7 @@ - [x] T015 [US4] Implement `apply_staleness` in `backend/src/services/dashboard_testing/registry/staleness.py` @INVARIANT: never false-stale from unavailable lineage engine (skip with warning) @TEST_EDGE: chart_removed->BLOCKED; filter_scope_change->NEEDS_REVALIDATION; engine_down->skip -- [x] T016 [US4] Integrate 037 StructureDiff + 041 blast-radius hooks to emit staleness signals +- [~] T016 [US4] Integrate 037 StructureDiff + 041 blast-radius hooks to emit staleness signals Adapter accepts 037 change envelopes and 041 affected scenario/dashboard references; unavailable upstream engines are explicitly skipped without false-stale transitions. @@ -71,7 +74,7 @@ ## Phase 7 — Health + Polish -- [x] T021 [P] Implement analytics-owned health projection in `backend/src/services/dashboard_testing/registry/health.py` +- [~] T021 [P] Implement analytics-owned health projection in `backend/src/services/dashboard_testing/registry/health.py` - [x] T022 [P] Frontend DTOs `ScenarioRegistryEntry`/`ScenarioRevision` in `frontend/src/types/scenario-registry.ts` - [x] T023 [P] Thin `ScenarioRegistryModel.svelte.ts` for list/detail/revisions/health projections Dedicated heavy registry pages remain owned by 043/045; model is the 042 integration boundary. @@ -79,10 +82,18 @@ Full backend: 10712 passed, 235 skipped, 1 xpassed. Full frontend Vitest: 3859 passed across 204 files; frontend build passed. New/affected files pass targeted ruff/eslint. Full backend ruff still reports 7904 repository-wide pre-existing findings; semantic/belief audit remains a separate workspace task. -- [x] T025 **Prototype validation**: verify every @UX_STATE in contracts reachable via `prototype/index.html` state switcher; responsive - Playwright browser check at 390x844 reached list/detail/empty/create states; responsive viewport meta is present. +- [ ] T025 **Prototype validation**: verify every @UX_STATE in contracts reachable via `prototype/index.html` state switcher; responsive -**Checkpoint**: Registry list/detail/revisions/lifecycle/staleness/health pass; 404 route fixed; prototype covers states. +## Audit Follow-ups (2026-08-20) + +- [ ] T026 Wire 037 StructureDiff and 041 blast-radius production events into `apply_staleness`; verify + an upstream fixture changes affected registry rows without a direct service call. +- [ ] T027 Emit idempotent canonical 036 InvestigationSignal for registry health/staleness and verify + emission does not start an agent action. +- [ ] T028 Run independent registry quickstart/API/lifecycle/staleness verification and record only + reproducible command results. + +**Checkpoint**: Pending T026–T028. Current code presence does not prove runtime upstream integration. ## Dependencies diff --git a/specs/042-dashboard-scenario-registry/traceability.md b/specs/042-dashboard-scenario-registry/traceability.md index d55fdbb77..376ce4800 100644 --- a/specs/042-dashboard-scenario-registry/traceability.md +++ b/specs/042-dashboard-scenario-registry/traceability.md @@ -1,5 +1,8 @@ # Traceability: Scenario Registry & Lifecycle (042) +> **Factual audit 2026-08-20:** rows trace intended code/test ownership, not completed acceptance +> evidence. 037/041 upstream staleness subscription and 036 InvestigationSignal emission are open. + | Story | Requirement | Model | API operationId | Contract | Task | Test | |-------|-------------|-------|------------------|----------|------|------| | US1 Find | SCREG-FR-001/002 | ScenarioRegistryEntry | scenarios.list | Registry.List | T005-T007 | test_list | diff --git a/specs/043-dashboard-scenario-editor/checklists/requirements.md b/specs/043-dashboard-scenario-editor/checklists/requirements.md index d0b1154b0..43483b133 100644 --- a/specs/043-dashboard-scenario-editor/checklists/requirements.md +++ b/specs/043-dashboard-scenario-editor/checklists/requirements.md @@ -2,9 +2,12 @@ **Purpose**: Verify SCEDIT-FR-001..008 completeness. | **Created**: 2026-08-07 +> **Factual audit 2026-08-20:** `[x]` requires current production evidence, `[~]` means partial +> code exists, `[ ]` means missing integration or proof. + ## View (FR-001) -- [ ] CHK001 Persisted scenario renders read-only by default +- [~] CHK001 Persisted scenario renders read-only by default - [ ] CHK002 Steps, dependency graph, parameters, baselines, assertions visible - [ ] CHK003 Generated executable read-only @@ -16,13 +19,13 @@ ## Constrained Assertions (FR-003) -- [ ] CHK007 Assertions use constrained editor (registered operators + baseline refs) -- [ ] CHK008 SQL/raw-baseline/path injection rejected +- [~] CHK007 Assertions use constrained editor (registered operators + baseline refs) +- [~] CHK008 SQL/raw-baseline/path injection rejected ## Visual Dependencies (FR-004) -- [ ] CHK009 Dependency editing via visual DAG -- [ ] CHK010 Cycle/duplicate-output validation on every change +- [~] CHK009 Dependency editing via visual DAG +- [~] CHK010 Cycle/duplicate-output validation on every change ## Agent Edit (FR-006) diff --git a/specs/043-dashboard-scenario-editor/plan.md b/specs/043-dashboard-scenario-editor/plan.md index 104955ff0..3a735f0ce 100644 --- a/specs/043-dashboard-scenario-editor/plan.md +++ b/specs/043-dashboard-scenario-editor/plan.md @@ -1,6 +1,9 @@ # Implementation Plan: Scenario Editor UX -**Branch**: `043-dashboard-scenario-editor` | **Date**: 2026-08-07 | **Spec**: spec.md | **Status**: Draft +**Branch**: `043-dashboard-scenario-editor` | **Date**: 2026-08-07 | **Spec**: spec.md | **Status**: Partially implemented — factual audit pending remediation + +> **Implementation audit, 2026-08-20:** editor code and route exist; current E2E/accessibility/policy +> verification is open, and closure remains dependent on unresolved 042/044 runtime contracts. ## Summary diff --git a/specs/043-dashboard-scenario-editor/prototype/manifest.md b/specs/043-dashboard-scenario-editor/prototype/manifest.md index 8be0b1880..c05641803 100644 --- a/specs/043-dashboard-scenario-editor/prototype/manifest.md +++ b/specs/043-dashboard-scenario-editor/prototype/manifest.md @@ -2,6 +2,9 @@ @defgroup Prototype Interactive HTML prototype manifest for the Scenario Editor. ## Prototype Metadata + +> **Factual audit 2026-08-20:** prototype state reachability is not production verification. Current +> browser/accessibility and policy-gate evidence remains pending. - **Feature**: 043 Scenario Editor UX - **Source contracts**: ux_reference.md, contracts/modules.md - **Screens represented**: 1 (Scenario Editor) @@ -23,7 +26,7 @@ ## Screen ↔ Story Traceability -| Story | Prototype Feature | Acceptance Verified | +| Story | Prototype Feature | Intended acceptance coverage | |-------|-------------------|---------------------| | US1 View | read-only badge, graph | read-only default | | US2 Manual | Edit toggle, dirty, save | revision + delegated policy | diff --git a/specs/043-dashboard-scenario-editor/quickstart.md b/specs/043-dashboard-scenario-editor/quickstart.md index f19b8fa96..74cabbed1 100644 --- a/specs/043-dashboard-scenario-editor/quickstart.md +++ b/specs/043-dashboard-scenario-editor/quickstart.md @@ -1,5 +1,8 @@ # Quickstart: Scenario Editor (043) +> **Factual audit 2026-08-20:** pending verification checklist only; no command result below is +> currently asserted as evidence. + ## Prereqs - 042 registry backend live (scenario + revisions) - Frontend deps installed diff --git a/specs/043-dashboard-scenario-editor/spec.md b/specs/043-dashboard-scenario-editor/spec.md index 92b929413..02dbd2fc6 100644 --- a/specs/043-dashboard-scenario-editor/spec.md +++ b/specs/043-dashboard-scenario-editor/spec.md @@ -12,7 +12,7 @@ @SEMANTICS: spec, requirements, feature, ux, scenario, editor, visual, revision, agent **Feature Branch**: `043-dashboard-scenario-editor` -**Created**: 2026-08-07 | **Status**: Draft +**Created**: 2026-08-07 | **Status**: Partially implemented — factual audit pending remediation **Input**: "Provide a first-class Scenario Editor for viewing and editing persisted scenarios: business metadata and parameter definitions editable manually, assertions via constrained editors, dependencies via visual DAG editing, generated executable read-only, and complex changes delegated to 'Edit with agent'. Metadata changes use a registry metadata version; executable changes create immutable revisions." ## User Scenarios @@ -142,11 +142,14 @@ - Q: May the agent save a revision? → A: Yes, after deterministic validation when delegated policy permits. It always creates an immutable revision with agent/delegator/case provenance; policy-gated actions use an inline ActionApprovalGate. - Q: Does this replace 039 workspace? → A: No. 039 is the create flow in agent chat; 043 is the post-save edit surface over the registry. -## Implementation Status & MVP Debt (audit 2026-08-07) +## Implementation Status & MVP Debt (factual audit 2026-08-20) -**Facts (code check):** -- 🟡 038 DTOs exist (DashboardTestScenario, ScenarioStep, ScenarioParameter, ScenarioRef, ScenarioValidationResult); no dedicated editor UI/model. -- 🔴 No visual DAG editor or constrained assertion editor exists. -- 🟡 `resolve` (038) supports parameter/selector/manual edits but is agent-tool-bound; a registry-backed edit surface is absent until 042 lands. +WorkingDraft/proposal services, constrained assertion and DAG components, plus the editor route are +present. The hybrid editor therefore exists structurally but is not verified as a closed workflow. + +- `[~]` Server-side drafts/proposals and UI components are present; no current route-level E2E, + accessibility, or policy-gate evidence is retained. +- `[~]` Revalidation depends on 042 staleness input whose upstream 037/041 integration is unproven. +- `[ ]` Feature closure is blocked by incomplete 044 execution and unperformed independent verification. #endregion ScenarioEditor.Spec diff --git a/specs/043-dashboard-scenario-editor/tasks.md b/specs/043-dashboard-scenario-editor/tasks.md index bbf478064..3ef7719b3 100644 --- a/specs/043-dashboard-scenario-editor/tasks.md +++ b/specs/043-dashboard-scenario-editor/tasks.md @@ -5,6 +5,9 @@ ## Format: `- [ ] T### [P] [USx] Description with exact file path` +> **Factual audit 2026-08-20:** `[x]` means code plus relevant evidence; `[~]` means partial +> implementation; `[ ]` means absent integration or unperformed verification. + ## Phase 1 — Setup - [x] T001 Create EditOperation DTOs (union) + fixtures in `backend/src/services/dashboard_testing/editor/ops.py` and `specs/043-dashboard-scenario-editor/fixtures/` @@ -58,10 +61,14 @@ - [x] T016 [P] RBAC scenario:edit enforcement tests Covered: `backend/tests/api/test_scenario_editor_routes.py` (agent-propose + proposal-save → 403 without scenario:edit) -- [x] T017 Run quickstart-equivalent scenario suite, scoped ruff, belief/ATTN static audit and semantic rebuild. - Verified in the 043-047 close: backend scenario suite 296 passed/1 skipped, frontend 3927 passed, - build succeeded, doc-gen indexed 9576 contracts/4730 edges; known >400-line module debt is recorded. -- [x] T018 **Prototype validation**: every declared @UX_STATE is reachable via `prototype/index.html`; Playwright verified 1440x900 and 390x844 without overflow or control overlap. +- [ ] T017 Run quickstart-equivalent scenario suite, scoped ruff, belief/ATTN static audit and semantic rebuild. +- [ ] T018 **Prototype validation**: every declared @UX_STATE is reachable via `prototype/index.html`. + +## Audit Follow-ups (2026-08-20) + +- [ ] T019 Verify editor route keyboard access, stale conflict recovery and policy-gated save with an + independent browser/API test after registry staleness wiring is complete. +- [ ] T020 Run current scoped editor tests and record command-level evidence; do not reuse prior batch totals. ## Dependencies diff --git a/specs/043-dashboard-scenario-editor/traceability.md b/specs/043-dashboard-scenario-editor/traceability.md index 99e434a04..7c4885ebe 100644 --- a/specs/043-dashboard-scenario-editor/traceability.md +++ b/specs/043-dashboard-scenario-editor/traceability.md @@ -1,5 +1,8 @@ # Traceability: Scenario Editor (043) +> **Factual audit 2026-08-20:** rows identify code/test ownership only. Route-level accessibility, +> stale-conflict and policy-gate workflow evidence is pending; 043 remains dependent on 042/044. + | Story | Requirement | Model | API operationId | Contract | Task | Test | |-------|-------------|-------|------------------|----------|------|------| | US1 View | SCEDIT-FR-001 | ScenarioEditorModel | editor.load | Editor.Load | T003-T005 | ScenarioEditorModel.test | diff --git a/specs/044-dashboard-scenario-execution/SESSION_STATE.md b/specs/044-dashboard-scenario-execution/SESSION_STATE.md new file mode 100644 index 000000000..6101a60b4 --- /dev/null +++ b/specs/044-dashboard-scenario-execution/SESSION_STATE.md @@ -0,0 +1,117 @@ +# 044 Scenario Execution — Session State + +> **Updated**: 2026-08-21 +> **Purpose**: Durable handoff for the current implementation/review session. This is a +> decision and verification ledger; `tasks.md` and `traceability.md` remain the canonical +> feature backlog and requirement matrix. + +## Current Objective + +Bring the 044 Scenario Execution Engine into material conformance with its contracts while +preserving fail-closed live I/O and GRACE-Poly invariants. Every code change is followed by an +independent verifier pass and semantic curation. + +## Architecture Decisions Already Implemented + +- HTTP and 046 start paths are persistence-only. The scheduler-owned queued dispatcher is the + sole initial execution authority, claiming `queued -> running` with a durable CAS. +- Environment classification is server-owned (`ConfigManager`), not request-owned. A configured + PROD run always becomes `pending_approval` with an `ActionApprovalGate`; an unknown environment + fails before any run/gate/notification/queue side effect. +- `LiveExecutionBinding` persists only immutable identity/fingerprint data. Application startup + builds `LiveExecutionCompositionRoot` from trusted configuration. Exact Superset bindings may + call the existing 037 query envelope; unavailable/mismatched browser, Superset, or screenshot + bindings are typed non-pass and perform no I/O. +- Artifact evidence needs a non-zero 64-hex digest and durable matching ref. Historical evidence + is retained for audit but retired from current result/signal projections on retry/cancel/recovery. +- Retry, timeout, cancel and crash recovery operate only on persisted run state and the pinned + RunnerPlan. Unsafe effects require reconciliation; browser recovery needs a pinned safe + checkpoint. +- Failed/blocked/inconclusive terminal results publish one immutable, idempotent 047 queue input; + they do not create a Case, AgentRun, chat or remediation action. + +## Independently Verified Closures + +- Server-owned PROD gate: external 044/046 verification profile passed 76 tests. +- Queued dispatcher API + scheduler profile: 94 tests passed; scheduler callback profile: 124 + tests passed. FastAPI TestClient tests run outside this sandbox because AnyIO self-pipe gets + `EPERM` inside it. +- Live composition bootstrap: 57 tests passed; exact configured Superset binding produces the + expected durable `draft:{run}:{sha256}` ref/digest. Browser/Screenshot providers still default + to typed unavailable. +- Lifecycle closure: retry/cancel/timeout/recovery profiles were independently verified. Timeout + materializes otherwise-lazy declared descendants as durable blocked rows. + +## Orthogonal Review Findings + +The independent review found these material gaps. The first is fixed; the second is active. + +1. **P0 — fixed**: Client-controlled `is_prod` could bypass ActionApprovalGate. The new + server-owned environment policy is verified and semantically curated. +2. **P0 — fixed (2026-08-21)**: RunnerPlan persists version/hash-pinned exact `{tool, action}` + `ActionExecutionDescriptor` snapshots. Tool-only fallback and universal non-human retry-safe + metadata are rejected; malformed legacy plans terminalize before I/O, and server-owned PROD + blocks browser mutation before provider invocation. +3. **P1 — fixed (2026-08-21)**: HumanCheckpoint and ActionApprovalGate decisions now consume + pending state through atomic SQL compare-and-set predicates. Stale/concurrent decisions fail + after exactly one winner; existing cancellation expiry remains terminal and non-reactivating. +4. **P1 — fixed (2026-08-21)**: Queued-dispatch exceptions now terminalize through a shared cleanup + path that retires current evidence projections, marks active steps inconclusive, expires leases, + and emits the existing idempotent terminal side effects. +5. **P1 — pending**: `sql_evidence`, bounded transform, AgentEvaluation/DecisionPolicy and global + capacity management remain absent despite the complete 044 target contract. +6. **P2 — pending**: Quickstart paths/counts and a stale automation migration-head test must be + aligned with the current linear Alembic chain. + +## Invariants That Must Not Regress + +- No client payload controls PROD classification, authority, RLS, action identity or retry safety. +- No external adapter returns PASS from metadata, identifiers, missing binding, invalid status, + missing digest or mismatched evidence ref. +- HumanCheckpoint is never an executor or an ActionApprovalGate. +- Only an ActionRegistry-pinned descriptor may reach executor I/O; unknown action must fail before + lease/I/O. +- A terminal run has no active lease/current evidence projection/running step. +- Repeated requests, scheduler ticks, retries and terminal projections are idempotent within their + immutable context. + +## Verification Boundaries / Deployment Debt + +- Browser-safe action provider and ScreenshotService-to-durable-evidence provider are not yet + registered in deployment; both correctly fail closed. +- A real PostgreSQL `alembic upgrade head` remains required. SQLite cannot run an earlier unrelated + migration using `drop_constraint`. +- Axiom MCP is unavailable in this session, so semantic index rebuild is not claimed. +- Legacy `ScreenshotService` is intentionally not registered as a 044 provider: its output paths + and environment credentials do not prove principal/RLS-bound durable evidence. The provider + registration hook remains startup-owned and fail-closed until an explicit lawful adapter exists. + +## Active Work Contract + +**Task**: Replace tool-only dispatch with version-pinned `{tool, action}` descriptors from the 038 +ActionRegistry; make descriptor data the only source of executor policy, side-effect key, +idempotency, retry safety, timeout and mutation guard. + +**Completion evidence required**: preflight and legacy-plan malformed cases, safe vs unsafe +recovery/retry, server-owned PROD mutation no-call, queued error closure, targeted 038/044 tests, +linear Alembic head, lint/compile/diff checks, independent verifier result, then semantic curation. + +## Latest Verification + +- Targeted lifecycle/approval/queued-dispatch/scheduler profile: 32 passed. +- RunnerPlan, worker, retry, terminal-signal and timeout profile: 19 passed. +- `python -m compileall` and `git diff --check`: passed. +- Retry API now explicitly rejects legacy unsafe tool-only plans with `409 RETRY_CONFLICT`; + the stale API expectation was aligned with the pinned-descriptor contract. +- `recover_run` was decomposed into a bounded running-step recovery helper; scoped Ruff for + execution and scenario-run API modules passes. +- Full available 044 backend profile: 241 passed (`test_scenario_*.py` registry suite plus + scenario run/automation/analytics API suites). +- Live binding and recovery profile: 40 passed; frontend profile previously verified 227 files / + 3930 tests; prototype static validation now passes. +- `alembic heads` remains a single linear head (`e3f4a5b6c7d8`), but `alembic check` and real + upgrade are blocked in this environment because `DATABASE_URL` is the placeholder + `__MUST_SET_DATABASE_URL__`; Docker Compose is also blocked by missing `SERVICE_JWT`. +- Screenshot evidence validation now accepts one or more provider-issued refs only when every ref + has a valid non-zero SHA-256 and the first ref agrees with the declared `sha256`; focused live + executor/composition profile: 36 passed. diff --git a/specs/044-dashboard-scenario-execution/checklists/requirements.md b/specs/044-dashboard-scenario-execution/checklists/requirements.md index bb074be5f..d62ffb7cc 100644 --- a/specs/044-dashboard-scenario-execution/checklists/requirements.md +++ b/specs/044-dashboard-scenario-execution/checklists/requirements.md @@ -2,15 +2,18 @@ **Purpose**: Verify SCEX-FR-001..010 completeness. | **Created**: 2026-08-07 +> **Factual audit 2026-08-20:** `[x]` requires current production evidence, `[~]` means partial +> code exists, `[ ]` means missing integration or proof. Synthetic PASS executors are not evidence. + ## Run Model (FR-001/003/007) -- [ ] CHK001 ScenarioRun first-class, pinned to scenario_id (UUID) + revision_id (UUID) + content_hash + env + param snapshot +- [~] CHK001 ScenarioRun first-class, pinned to scenario_id (UUID) + revision_id (UUID) + content_hash + env + param snapshot - [ ] CHK002 Deterministic; no LLM per step; agent not in hot path - [ ] CHK003 Recoverable by scenario_run_id; provenance for reproducibility ## Execution (FR-002/006/009) -- [ ] CHK004 Dispatch by step.tool to typed executor +- [~] CHK004 Dispatch by step.tool to typed executor - [ ] CHK005 Executors reuse 037/038/036; no second Playwright/LLM/SQL stack - [ ] CHK006 Step attempt/retry/timeout/ref binding/error_code/artifact_refs - [ ] CHK007 Failure propagation blocks dependents diff --git a/specs/044-dashboard-scenario-execution/contracts/modules.md b/specs/044-dashboard-scenario-execution/contracts/modules.md index e6f7a5587..821b81a10 100644 --- a/specs/044-dashboard-scenario-execution/contracts/modules.md +++ b/specs/044-dashboard-scenario-execution/contracts/modules.md @@ -13,8 +13,12 @@ # @ingroup ScenarioExecution # @BRIEF Deterministically derive the execution plan from a ScenarioRevision at run start. # @PRE scenario revision selected and pinned. -# @POST returns RunnerPlan with env targets, resolved params, pinned baselines, topological order, executor mapping; refuses on revision mismatch. +# @POST returns RunnerPlan with env targets, resolved params, pinned baselines, topological order, +# exact ActionExecutionDescriptor mapping plus registry version/hash; refuses on revision mismatch. # @INVARIANT runner.plan.json in git is a reference artifact, never the runtime source of truth. +# @INVARIANT Each persisted descriptor is resolved against the pinned 038 registry before a run, +# lease or adapter exists; descriptor fields alone derive retry/idempotency/timeout policy. +# @REJECTED tool-only executor fallback or universal retry-safe metadata. # @TEST_EDGE missing revision->reject; revision mismatch->reject before run. def derive_runner_plan(db, scenario_id, revision_id): ... # #endregion ScenarioExecution.RunnerPlan.Derive @@ -32,7 +36,7 @@ async def start_run(db, scenario_id, revision_id, params, env, actor, idempotenc # #region ScenarioExecution.Dispatch [C:5] [TYPE Function] [SEMANTICS scenario,execution,dispatch,step,executor] # @ingroup ScenarioExecution -# @BRIEF Dispatch a step by tool to its typed executor; record a ScenarioStepRun. +# @BRIEF Dispatch a descriptor-validated step to its typed executor; record a ScenarioStepRun. # @PRE step dependencies satisfied; {tool, action} exists in the revision-pinned 038 ActionRegistry; mutation policy preflight passed. # @POST returns step outcome; output refs bound; ScenarioStepRun persisted. # @SIDE_EFFECT executor side effects (browser/superset/xlsx/evidence); DB write. @@ -100,10 +104,11 @@ def claim_step(run_id, logical_step_id, worker_id): ... # #region ScenarioExecution.ExecutorRegistry [C:3] [TYPE Module] [SEMANTICS scenario,execution,registry,executor] # @ingroup ScenarioExecution -# @BRIEF tool -> executor mapping; BrowserExecutor resolves only 038 ActionRegistry actions; human excluded; each executor declares idempotency/retry-safety. +# @BRIEF ActionExecutionDescriptor -> executor mapping; BrowserExecutor resolves only exact 038 +# ActionRegistry actions; human excluded; descriptor declares idempotency/retry-safety. # @REJECTED human as executor (HumanCheckpoint lifecycle control instead). EXECUTOR_MAP = {browser: BrowserExecutor(ActionRegistry), superset_api, xlsx, assertion, screenshot, report, artifact} -# @INVARIANT each executor declares {idempotent, retry_safe, side_effect_key, external_request_id}. +# @INVARIANT descriptor, not executor/tool default, declares {idempotent, retry_safe, side_effect_key, timeout, mutation risk}. # @INVARIANT mutation actions require immutable mutation_contract; PROD mutation is rejected; mutation retries default false. # #endregion ScenarioExecution.ExecutorRegistry #endregion ScenarioExecution.Modules diff --git a/specs/044-dashboard-scenario-execution/data-model.md b/specs/044-dashboard-scenario-execution/data-model.md index 6a16132a1..2f95e7260 100644 --- a/specs/044-dashboard-scenario-execution/data-model.md +++ b/specs/044-dashboard-scenario-execution/data-model.md @@ -8,10 +8,29 @@ `ScenarioRun` is created before dispatch. Fields: id, scenario_id, scenario_revision_id (revision_id UUID), scenario_content_hash, verification_program_hash, action_registry_version, dashboard_id, environment_id, status (`pending_approval|queued|running|waiting_human|blocked|cancel_requested|cancelled|passed|failed|inconclusive`), phase (preflight|setup|executing|waiting_human|draining|terminal), parameter_bindings (immutable JSON), baselines_pinned (version), target_snapshot, execution_principal_fingerprint, execution_toggles (optional evidence only), trigger_source (server-owned), agent_run_id? (provenance), verification_run_id? (aggregation), idempotency_key (unique), started_at, finished_at, resume_token, error_code, runner_version. +`live_execution_binding_ref?` plus `live_execution_binding_snapshot?` are the only persisted live-I/O +coordinates. The snapshot is an exact allowlist: binding ref; environment/release/query-model, +execution-principal and RLS/security fingerprints; browser-safe checkpoint/action refs; and evidence +owner/ref policy. It contains no credential, client, cookie, raw browser context, callable principal, +or capture bytes. `LiveExecutionCompositionRoot` resolves this immutable identity server-side at +startup; a missing provider is typed unavailable, and any exact-snapshot mismatch is typed non-pass +before I/O. + +The deployment-owned `settings.scenario_live_execution_bindings[]` record stores `enabled`, +`binding_snapshot`, and `query_model_snapshot` only. Startup resolves its `Environment` credentials +server-side to build the existing `SupersetClient`, then registers the exact model/storage tuple. +Browser and screenshot providers are process-local registrations and are never serialized in this +record; absent registrations yield stable configured-unavailable outcomes. + If the selected revision contains a `human` step, it is derived as `manual_run_only=true`: it may start only from the authenticated analyst manual-run route. Scheduler, deploy/ETL/API trigger and any background runner are ineligible; no HumanCheckpoint may be skipped or defaulted. For a PROD request, the service atomically creates `ScenarioRun(status=pending_approval)` and `ActionApprovalGate(owner_type=scenario_run, owner_id=run_id, operation=scenario_execution)`. Approval transitions only `pending_approval → queued`; denial/expiry transitions to `blocked`. Schedulers and external triggers therefore receive a durable run/intent, never an unusable 403. `trigger_source` is set only by the trusted entry point (manual route, scheduler, deploy connector, or API key), never by a bearer client field. Idempotency uses canonical execution-request hash: same key + same hash returns the existing run; same key + different hash returns `409 IDEMPOTENCY_KEY_REUSED`. +`EnvironmentExecutionPolicy` is resolved server-side from the configured `Environment` record +(`stage=PROD` or `is_production=true`) before RBAC, idempotency, run, gate, queue, notification, or +adapter work. Any compatibility `is_prod` request field is non-authoritative and cannot upgrade or +downgrade that class; an unknown environment fails closed without creating a run or gate. + ## ParameterBinding, ExecutionPrincipal, and TargetSnapshot `ParameterBinding`: parameter_name, resolved_value, source, resolved_at, validation_fingerprint. It is derived at start from the 038 `ParameterDefinition` and launch input; it is never embedded in ScenarioRevision or its `content_hash`. @@ -40,7 +59,10 @@ Fields: id, run_id FK, logical_step_id (immutable UUID, from #8), step_position ## RunnerPlan — deterministic derivation from revision (#2) -RunnerPlan is **derived deterministically at run start** from the selected immutable `ScenarioRevision`/Verification Program, NOT read from a stored `runner.plan.json`. Fields: scenario_revision_id, scenario_content_hash, verification_program_hash, action_registry_version, env targets, resolved params, pinned baselines, topological order, executor mapping per step, retry/timeout policy, decision-policy map, human-checkpoint list. Run refuses if its revision/program/action-registry hashes differ from the selected revision. +RunnerPlan is **derived deterministically at run start** from the selected immutable `ScenarioRevision`/Verification Program, NOT read from a stored `runner.plan.json`. Fields: scenario_revision_id, scenario_content_hash, verification_program_hash, `action_registry_version`, `action_registry_hash`, env targets, resolved params, pinned baselines, topological order, and one immutable `ActionExecutionDescriptor` per step. A descriptor contains the exact `{tool, action}`, typed input/output contracts, idempotency, retry safety, side-effect-key policy, timeout, mutation contract/risk. The runner persists the descriptor snapshot in both `steps` and `executor_mapping`; it derives leases/recovery/retry policy only from that snapshot. A missing, altered, unknown, version/hash-mismatched descriptor, invalid input/output shape, or mutating action without its required contract rejects before run/lease/I/O. Run refuses if its revision/program/action-registry hashes differ from the selected revision. + +@INVARIANT RunnerPlan descriptor resolution is exact and version/hash pinned; `tool` is never a dispatch or retry-policy fallback. +@REJECTED A universal `idempotent=true, retry_safe=true` claim based on a non-human tool was rejected because it can repeat unsafe effects. The materialized `runner.plan.json` in git is a **reference artifact**, never the runtime source of truth; it may be regenerated from any revision. @@ -81,7 +103,7 @@ pending_approval → queued → running → waiting_human | blocked → passed | ## ScenarioExecutorRegistry and BrowserExecutor -Mapping `tool -> executor`: +The registry resolves `ActionExecutionDescriptor -> executor`; the following list states each descriptor's tool family, not a tool-only fallback: - browser -> `BrowserExecutor` → version-pinned 038 `ActionRegistry` → Playwright/session infrastructure - superset_api -> 037 metric_executor_async / SupersetClient.ChartData.Execute - sql_evidence -> `SqlEvidenceExecutor` → Superset SQL Lab backend/API → configured database connection (no credentials exposed to agent) diff --git a/specs/044-dashboard-scenario-execution/plan.md b/specs/044-dashboard-scenario-execution/plan.md index b3cbfd505..82a0a8136 100644 --- a/specs/044-dashboard-scenario-execution/plan.md +++ b/specs/044-dashboard-scenario-execution/plan.md @@ -1,6 +1,9 @@ # Implementation Plan: Scenario Execution Engine -**Branch**: `044-dashboard-scenario-execution` | **Date**: 2026-08-07 | **Spec**: spec.md | **Status**: Draft +**Branch**: `044-dashboard-scenario-execution` | **Date**: 2026-08-07 | **Spec**: spec.md | **Status**: Not production-complete — factual audit pending remediation + +> **Implementation audit, 2026-08-20:** run models/API/primitives exist; synthetic executors, missing +> executor imports and insufficient walker tests prevent the plan from being considered delivered. ## Summary diff --git a/specs/044-dashboard-scenario-execution/prototype/index.html b/specs/044-dashboard-scenario-execution/prototype/index.html index 288e7290b..4ec493466 100644 --- a/specs/044-dashboard-scenario-execution/prototype/index.html +++ b/specs/044-dashboard-scenario-execution/prototype/index.html @@ -1,3 +1,21 @@ + + + + + + + + + + + + + + + + + + @@ -23,22 +41,25 @@
Scenario run SR-1842 · revision r18

XLSX reconciliation - Выполняется + waiting_human

PREPROD · Target: rc-17 · Анна · параметры сохранены в snapshot

-
- +

Ход проверки 4 из 6

@@ -105,7 +126,7 @@ >XLSX: 124 rows · baseline: 124 rows
- @@ -123,11 +144,11 @@ trail.

-
@@ -142,58 +163,74 @@ -
- State: + -
+ + diff --git a/specs/044-dashboard-scenario-execution/prototype/manifest.md b/specs/044-dashboard-scenario-execution/prototype/manifest.md index eb009ad85..2a147bddf 100644 --- a/specs/044-dashboard-scenario-execution/prototype/manifest.md +++ b/specs/044-dashboard-scenario-execution/prototype/manifest.md @@ -1,28 +1,38 @@ #region ScenarioExecution.PrototypeManifest [C:3] [TYPE ADR] [SEMANTICS prototype,manifest,scenario,execution] @defgroup Prototype Interactive HTML prototype manifest for Scenario Execution Engine (engine-level states). +@RELATION DEPENDS_ON -> [ScenarioExecution.PrototypeRun] +@RELATION DEPENDS_ON -> [Test.ScenarioExecution.PrototypeStatic] +@RATIONALE The manifest distinguishes declared source-level reachability from browser, assistive-technology, and production-runtime evidence so the prototype cannot overstate its proof. +@REJECTED Treating static source checks as browser validation or production execution evidence was rejected — neither rendering nor runtime integrations execute in this verifier. ## Prototype Metadata + +> **Factual audit 2026-08-21:** `validate_static.py` proves source-level lifecycle names, persistent +> live-region attributes, native state controls, 44px sizing and reduced-motion declarations. It does +> not prove browser rendering, keyboard operation, assistive-technology announcements, runner execution, +> executor outcomes, artifacts or recovery in production code. - **Feature**: 044 Scenario Execution Engine - **Source contracts**: ux_reference.md, contracts/modules.md - **Screens represented**: 1 (Scenario Run engine view) - **Total states**: 6 (running, waiting_human, resumed, cancelled, failed, passed) -- **Accessibility**: keyboard nav, aria-live, ≥44px, prefers-reduced-motion +- **Accessibility (static)**: native buttons, persistent `role=status`/polite atomic live region, + ≥44px target declarations, prefers-reduced-motion override - **Responsive**: 375px, 900px ## State Coverage | @UX_STATE | Prototype State | Reachable? | Recovery | |-----------|-----------------|------------|----------| -| running | running | ✅ | — | -| waiting_human | waiting_human | ✅ | confirm/false-positive/inconclusive | -| resumed | resumed | ✅ | continues from resume token | -| cancelled | cancelled | ✅ | — | -| failed | failed | ✅ | retry / triage (047) | -| passed | passed | ✅ | — | +| running | running | ✅ statebar | — | +| waiting_human | waiting_human | ✅ initial + statebar | confirm/false-positive/inconclusive | +| resumed | resumed | ✅ statebar/confirm | continues from resume token | +| cancelled | cancelled | ✅ statebar/stop | — | +| failed | failed | ✅ statebar/non-pass decision | retry / triage (047) | +| passed | passed | ✅ statebar | — | ## Screen ↔ Story Traceability -| Story | Prototype Feature | Acceptance Verified | +| Story | Prototype Feature | Intended acceptance coverage | |-------|-------------------|---------------------| | US1 Start | run header + snapshot | pinned revision | | US2 Dispatch | tool→executor row, step statuses | deterministic dispatch | diff --git a/specs/044-dashboard-scenario-execution/prototype/validate_static.py b/specs/044-dashboard-scenario-execution/prototype/validate_static.py new file mode 100644 index 000000000..458205272 --- /dev/null +++ b/specs/044-dashboard-scenario-execution/prototype/validate_static.py @@ -0,0 +1,98 @@ +# #region Test.ScenarioExecution.PrototypeStatic [C:3] [TYPE Module] [SEMANTICS test,scenario,execution,prototype,a11y,ux] +# @BRIEF Falsify static 044 prototype regressions without claiming browser-interaction execution. +# @RELATION BINDS_TO -> [ScenarioExecution.PrototypeRun] +# @RELATION VERIFIES -> [ScenarioExecution.PrototypeRun] +# @RELATION VERIFIES -> [ScenarioExecution.PrototypeManifest] +# @TEST_CONTRACT: prototype/index.html + prototype-ui.css + manifest.md -> declared static UX/a11y guarantees +# @TEST_FIXTURE: 044 prototype source files -> repository paths +# @TEST_EDGE: hidden_waiting_alias; replaced_live_region; undersized_statebar_button; missing_reduced_motion; unreachable_declared_state +# @TEST_INVARIANT ScenarioExecution.PrototypeRun: Initial state, live announcements, native state controls, target sizing and reduced motion remain statically provable. -> VERIFIED_BY: validate_static +# @TEST_INVARIANT ScenarioExecution.PrototypeManifest: Six declared states map to native source transitions; this verifier does not claim browser or production-runtime evidence. -> VERIFIED_BY: validate_static +from __future__ import annotations + +from pathlib import Path +import re +import sys + + +_PROTOTYPE = Path(__file__).resolve().parent +_INDEX = _PROTOTYPE / "index.html" +_MANIFEST = _PROTOTYPE / "manifest.md" +_CSS = _PROTOTYPE.parents[1] / "prototype-ui.css" +_STATES = ("running", "waiting_human", "resumed", "cancelled", "failed", "passed") + + +# #region Test.ScenarioExecution.PrototypeStatic.Require [C:1] [TYPE Function] +def _require(condition: bool, message: str, failures: list[str]) -> None: + if not condition: + failures.append(message) +# #endregion Test.ScenarioExecution.PrototypeStatic.Require + + +# #region Test.ScenarioExecution.PrototypeStatic.Main [C:3] [TYPE Function] +# @BRIEF Check the concrete source contract; this is static evidence, not browser interaction evidence. +def main() -> int: + index = _INDEX.read_text(encoding="utf-8") + manifest = _MANIFEST.read_text(encoding="utf-8") + css = _CSS.read_text(encoding="utf-8") + failures: list[str] = [] + + _require('protoState("waiting_human"' in index, "initial state must be waiting_human", failures) + _require('protoState("waiting"' not in index, "hidden waiting alias must be absent", failures) + _require(index.count('id="run-announcement"') == 1, "one persistent live node is required", failures) + live_node = re.search(r']*\bid="run-announcement"[^>]*>', index) + _require(live_node is not None, "one persistent live node is required", failures) + live_attributes = live_node.group(0) if live_node is not None else "" + _require( + all( + attribute in live_attributes + for attribute in ('role="status"', 'aria-live="polite"', 'aria-atomic="true"') + ), + "live node needs polite atomic status semantics", + failures, + ) + _require("announcement.textContent" in index, "state/decision updates must announce through live node", failures) + _require( + "checkpointDecision" in index and "decisionLabels[checkpointDecision]" in index, + "checkpoint decision must use the persistent live node", + failures, + ) + _require( + not re.search(r'getElementById\(["\']run-announcement["\']\)\.(?:innerHTML|outerHTML|replaceWith)', index), + "live node must not be replaced", + failures, + ) + _require("min-height: 44px" in css and "min-inline-size: 44px" in css, "interactive targets need 44px sizing", failures) + _require("@media (prefers-reduced-motion: reduce)" in css, "reduced-motion media query is required", failures) + button_tags = re.findall(r"]*>", index) + _require(bool(button_tags), "prototype needs native button controls", failures) + _require( + all('class="btn' in button and 'type="button"' in button for button in button_tags), + "every prototype button must be a native .btn target", + failures, + ) + _require('class="nav"' in index and "]*onclick="setProtoState\(\'{state}\'\)"', index, + ) is not None, + f"statebar {state} lacks a native transition", + failures, + ) + _require(f'"{state}":' in index, f"state renderer lacks {state}", failures) + _require(f"| {state} | {state} | ✅" in manifest, f"manifest does not factually declare {state}", failures) + + if failures: + print("prototype static validation failed:", *failures, sep="\n- ") + return 1 + print("prototype static validation passed") + return 0 +# #endregion Test.ScenarioExecution.PrototypeStatic.Main + + +if __name__ == "__main__": + sys.exit(main()) +# #endregion Test.ScenarioExecution.PrototypeStatic diff --git a/specs/044-dashboard-scenario-execution/quickstart.md b/specs/044-dashboard-scenario-execution/quickstart.md index 81e1ea799..9e689e54a 100644 --- a/specs/044-dashboard-scenario-execution/quickstart.md +++ b/specs/044-dashboard-scenario-execution/quickstart.md @@ -1,5 +1,12 @@ # Quickstart: Scenario Execution Engine (044) +> **Factual audit 2026-08-20:** the targeted service profile independently reverified 35 passes. +> Browser/Superset/Screenshot are fail-safe: they cannot synthesize PASS without explicit typed +> adapter success, and evidence requires a real valid digest/ref. This is not proof of real live +> Playwright/Superset/Screenshot composition. +> The Superset binding slice persists an immutable identity snapshot and, under an injected exact +> resolver, invokes the existing 037 query envelope and stores its exact raw-byte digest/ref. + ## Prereqs - 042 registry + DB migrated (scenario_runs, scenario_step_runs) - Backend venv @@ -16,9 +23,28 @@ python -m ruff check src/services/dashboard_testing/execution/ ## Exit Gates - [ ] Fixture scenario runs end-to-end, dispatching every step to correct executor -- [ ] Human checkpoint suspends + resumes without rerun +- [x] Persisted worker lease/idempotency checks reject unsafe reclaim and live-lease conflicts +- [x] Human and infrastructure resume advance only the missing DAG frontier; completed steps are not rerun +- [x] Browser/Superset/Screenshot do not manufacture PASS; invalid/missing evidence digest/ref is inconclusive +- [x] Superset live binding persists ref/snapshot; exact resolver/model match invokes 037 and stores exact digest/ref - [ ] Cancel terminates `cancelled` with bounded drain -- [ ] Run recoverable by scenario_run_id +- [x] Server-driven crash recovery by `scenario_run_id` uses only the persisted RunnerPlan and + expired worker claims: only idempotent/retry-safe frontier work gets a new attempt; unsafe + effects require reconciliation, and browser work without a pinned browser-safe checkpoint is + typed non-pass without replay. Retired evidence remains historical, and repeated rejected + terminal contexts reuse the idempotent queue signal. - [ ] Results reference immutable revision + provenance - [ ] human not invoked as executor; PROD gated -- [ ] ruff clean; prototype states covered +- [ ] T022 full scope, scoped ruff, semantic rebuild, and prototype states covered + +## Verification Boundary + +- The exact API test file has a 29-pass result outside this sandbox. +- In this sandbox FastAPI TestClient is blocked by AnyIO self-pipe `EPERM`; this is an execution + restriction, not an application failure. +- Startup composition supplies config-owned 037 resolver/client/model/storage for enabled exact + bindings. Browser and Screenshot providers remain absent and fail closed; prototype validation + remains unresolved. +- Before deployment, PostgreSQL must run and verify `alembic upgrade head` for the additive + `c1d2e3f4a5b6` live-binding columns/index; the isolated migration test does not replace that + deployment prerequisite. diff --git a/specs/044-dashboard-scenario-execution/spec.md b/specs/044-dashboard-scenario-execution/spec.md index e76b4d08e..7f2e802ee 100644 --- a/specs/044-dashboard-scenario-execution/spec.md +++ b/specs/044-dashboard-scenario-execution/spec.md @@ -1,5 +1,5 @@ #region ScenarioExecution.Spec [C:3] [TYPE ADR] [SEMANTICS spec,requirements,scenario,execution,run,step,engine] -@BRIEF First-class ScenarioRun/ScenarioStepRun execution engine that walks a validated DashboardTestScenario DAG and dispatches each step to a typed executor, with run lifecycle, cancellation, retry, timeout, and human-checkpoint pause/resume. The most critical missing piece — a scenario can be created but not executed. +@BRIEF Partially verified ScenarioRun/ScenarioStepRun execution engine that walks a validated DashboardTestScenario DAG through typed executors, with lifecycle, retry, timeout, and human-checkpoint pause/resume. It remains not production-complete because live composition, T022, prototype validation, and semantic indexing are unresolved. @RELATION DEPENDS_ON -> [Doc.Adr.ADR0001] @RELATION DEPENDS_ON -> [Doc.Adr.ADR0003] @RELATION DEPENDS_ON -> [AgentTestStabilization.Spec] @@ -15,7 +15,7 @@ @SEMANTICS: spec, requirements, feature, scenario, execution, run, step, runner, engine, resume, human **Feature Branch**: `044-dashboard-scenario-execution` -**Created**: 2026-08-07 | **Status**: Draft +**Created**: 2026-08-07 | **Status**: Not production-complete — fail-safe execution closure and targeted verification are recorded below; production composition remains open **Input**: "Provide a Scenario Execution Engine: a deterministic runner that walks the validated DashboardTestScenario graph, dispatches each step by tool to a typed executor (browser, superset_api, xlsx, assertion, screenshot, report, artifact), and manages the run lifecycle (queued, running, waiting_human, blocked, cancel, retry, timeout, resume, passed, failed, inconclusive) with immutable execution snapshots pinned to a scenario revision." ## User Scenarios @@ -68,7 +68,9 @@ **Acceptance**: 1. **Given** a user requests cancel **When** submitted **Then** the run enters `cancel_requested`, drains in-flight steps within a bounded window, and terminates `cancelled`. 2. **Given** a step fails **When** retry is requested **Then** a new attempt runs with bounded attempts and updated `attempt` count. -3. **Given** a step exceeds its timeout **When** triggered **Then** the step is marked failed/inconclusive with a timeout reason, and the run continues per policy. +3. **Given** a step exceeds its timeout **When** triggered **Then** the timed step is inconclusive and + every declared downstream node is materialized from the pinned RunnerPlan as blocked before the + run terminalizes; no later walker continuation may dispatch a lazy descendant. --- @@ -89,10 +91,10 @@ | # | Scenario | Category | Expected Behavior | Recovery | |---|----------|----------|-------------------|----------| | E1 | PROD run without approval | auth/gate | Durable `pending_approval` run + gate, no dispatch | Approve via ActionApprovalGate | -| E2 | Step timeout | execution | Step failed/inconclusive with reason | Retry / continue | +| E2 | Step timeout | execution | Timed step inconclusive; pinned-plan descendants blocked before terminalization | Retry only; no late continuation dispatch | | E3 | Retry exhausted | execution | Step failed; dependents blocked | Triage (047) | | E4 | Superset 5xx/403/422 | integration | Typed error taxonomy preserved | Retry / continue | -| E5 | Runner crash mid-run | resilience | API/XLSX steps resume only if retry-safe; browser state replays from last browser-safe checkpoint | Recover / replay | +| E5 | Runner crash mid-run | resilience | Only expired idempotent/retry-safe work may resume; unsafe work requires reconciliation; browser without a pinned safe checkpoint is inconclusive without replay | Recover safe frontier / reconcile | | E6 | Cancel mid-step | concurrency | In-flight completes or times out in drain window | — | | E7 | Duplicate output ref | data-integrity | 038 validator rejects before run | Fix graph | | E8 | Stale scenario run (registry) | data-quality | Warning-gated or blocked per policy | Revalidate | @@ -119,6 +121,41 @@ - **SCEX-FR-015**: `AgentEvaluation` MUST be a separate immutable runtime record and DecisionPolicy MUST deterministically map it plus deterministic evidence to StepOutcome. A bare model verdict never directly sets ScenarioResult. - **SCEX-FR-016**: Browser actions and mutation safety MUST use the same versioned 038 ActionRegistry/mutation contract. Mutating browser steps in PROD are prohibited; test-data mutation needs fixture scope, record keys, side-effect/retry and cleanup policy independent of PROD approval. +### Live Execution Composition Root (SCEX-LIVE-COMPOSITION) + +`LiveExecutionBinding` persisted on `ScenarioRun` is identity-only: a binding reference plus immutable +environment, release, query-model, execution-principal, RLS/security, browser-safe-checkpoint/action, +and evidence-policy fingerprints. It MUST NOT serialize a client, secret, cookie, callable principal, +raw browser context, or capture bytes. + +Application startup owns `LiveExecutionCompositionRoot`. It is the only server-side registration point +for authorized providers: an exact 037 `SupersetClient` + immutable `DashboardQueryModel` + principal/RLS +binding, a browser-safe session/action provider, and a ScreenshotService capture provider with durable +evidence storage. The root resolves a persisted binding only after every pinned identity/fingerprint +matches; it never constructs authority from caller IDs, run metadata, or `environment_id`. Missing +provider/configuration is `*_BINDING_UNAVAILABLE`; a malformed or unauthorized/mismatched identity is +`*_BINDING_INVALID`/`*_BINDING_MISMATCH`. Both fail closed and perform no I/O. + +Deployment config is `settings.scenario_live_execution_bindings[]`. Every enabled record contains only +`binding_snapshot` and `query_model_snapshot`; its credentials are looked up by the server from the +configured `Environment` record when startup constructs the existing `SupersetClient`. The root marks +a configured record whose environment/model/storage cannot be resolved as `SUPERSET_BINDING_UNAVAILABLE`. +It marks browser and screenshot slots from an enabled binding as `BROWSER_BINDING_UNAVAILABLE` and +`SCREENSHOT_BINDING_UNAVAILABLE` until a server-owned bootstrap calls `register_browser` or +`register_screenshot` with a lawful provider. An empty list is valid and leaves every live tool typed +unavailable. + +Providers return typed success/failure/timeout/cancellation outcomes. PASS additionally requires a +durable evidence reference and verified non-zero SHA-256. Where a provider supports transport +cancellation, the root passes the capability through; otherwise lifecycle cancellation remains +database-authoritative and the result remains typed. Browser recovery is lawful only from the declared +safe checkpoint/reconstruction binding; a raw session is never revived. A mutating browser action must +also satisfy the version-pinned 038 mutation contract and is rejected in PROD. + +`ScreenshotService` currently captures paths for the LLM workflow but does not expose a 044 +principal/RLS/checkpoint-bound durable-evidence provider. It therefore remains typed unavailable until +startup registers such a provider; no path or raw capture metadata is treated as evidence. + ### Key Entities - **ScenarioRun**: Recoverable execution instance of a pinned scenario revision; owns status, phase, params, provenance, steps. @@ -144,11 +181,62 @@ - Q: Is human an executor? → A: No. It is a runner-lifecycle suspend/resume control; the executor registry covers browser/superset_api/xlsx/assertion/screenshot/report/artifact. - Q: How does this differ from VerificationRun/AgentRun? → A: ScenarioRun executes the user-created DashboardTestScenario DAG; AgentRun is the creation run; VerificationRun is release-pipeline category verification. Three distinct run concepts. -## Implementation Status & MVP Debt (audit 2026-08-07) +## Implementation Status & MVP Debt (factual audit 2026-08-20) -**Facts (code check):** -- 🔴 No ScenarioRun/ScenarioStepRun model or runner exists; `runner.plan.json` is a stub `{scenario_id, revision_id, step_count}` (and is a reference artifact only — the RunnerPlan is derived at run start). -- 🔴 No executor registry or dispatch; `execute_step`/`ScenarioStep.*execute` grep = empty. -- ✅ Executors to reuse already exist: 037 `metric_executor_async`/`comparison.py`, 038 `capture.py`, 036 evidence/artifacts/HITL, 040 RunnerPool pattern. +ScenarioRun/ScenarioStepRun models, lifecycle/API primitives, a runner plan, and a fail-safe executor +boundary now exist. They do not yet constitute the specified production execution engine. + +- `[x]` Persisted worker lease/idempotency checks are proven: a live lease rejects another worker and + reclaim is limited to retry-safe/idempotent recorded effects. +- `[x]` HTTP and 046 automation starts persist or replay only queued/pending ScenarioRuns. Initial + adapter dispatch is performed separately when the server dispatcher wins the durable + `queued -> running` CAS; repeated ticks/replays do not dispatch again. Automated human plans are + rejected before a queued row reaches that CAS, while manual human runs reach their checkpoint only + after the dispatcher claims them. +- `[~]` `EnvironmentPolicy` resolves `stage=PROD`/`is_production=true` only from server + ConfigManager state before idempotency or persistence: client compatibility flags cannot change + the class, unknown targets fail closed, and manual/API/event/scheduled sources create the same + durable `pending_approval` gate. Pending rows remain excluded from dispatcher execution. A real + APScheduler scheduled-PROD integration test is still coverage debt, not a bypass. +- `[x]` Human and infrastructure continuation resume only the missing DAG frontier; completed steps + are not re-run. Walker tests cover completed/failure/artifact and evidence-integrity paths. +- `[x]` A revision containing `human` derives `manual_run_only=true`. Every trusted 046 automation + source (scheduled, deploy, release, ETL, API) rejects before idempotency lookup, ScenarioRun, + approval gate, notification, queue, or dispatch side effects; the same key remains usable by the + authenticated manual route. HumanCheckpoint observation remains distinct from ActionApprovalGate. +- `[x]` Failed, blocked and inconclusive terminal runner results emit one idempotent canonical 047 + queue input keyed by immutable run/status/revision/target/principal and registered artifact ref/digest + provenance. Queue projection never opens a case, AgentRun, chat or remediation action; passed runs emit none. +- `[x]` Strict eligible retry invalidates the persisted failed/inconclusive/blocked target and its + downstream closure before re-walk. Prior step attempts and artifact rows are retained as historical + provenance, but retired artifact projections are excluded from active result/terminal evidence. + Re-terminalization has a distinct immutable attempt context; replay of that same context is idempotent. +- `[x]` A timeout during claimed adapter I/O wins over a late payload. Before terminalization it + materializes every otherwise-lazy descendant from the immutable pinned RunnerPlan as a blocked + row, so a later walker cannot dispatch that descendant. This is lifecycle closure, not proof of + real live Browser/Superset/Screenshot I/O composition. +- `[x]` Server-driven crash recovery loads only persisted ScenarioRun/RunnerPlan/lease state by + `scenario_run_id`: completed ancestors are not re-run; an expired idempotent/retry-safe claim is + archived and retried as a new attempt; unsafe effects terminate blocked for reconciliation; and + browser recovery without a pinned browser-safe checkpoint terminates inconclusive without adapter + I/O. Historical evidence remains audit-visible but inactive for a new attempt, and repeated + rejected terminal contexts reuse their idempotent signal. It does not consume an infrastructure + resume token, HumanCheckpoint, or ActionApprovalGate. +- `[~]` Browser/Superset/Screenshot can materialize PASS only from explicit typed adapter success; + unavailable live I/O and missing/invalid evidence digest/ref are typed inconclusive. This is a + fail-safe partial closure, not evidence of real live Playwright/Superset/Screenshot composition. +- `[~]` `ScenarioRun` persists a nullable `LiveExecutionBinding` identity snapshot (binding ref, + environment/release/query-model/principal/RLS fingerprints, browser-safe binding refs and evidence + policy). Application startup owns a fail-closed `LiveExecutionCompositionRoot`: an exact registered + 037 client/model/DraftStorage tuple is wired into the default dispatcher, while registered browser and + Screenshot providers are validated against the same snapshot. Missing/mismatched providers are typed + inconclusive and never call I/O. No deployment has yet registered a browser-safe action or + ScreenshotService-to-durable-evidence provider, so those tools remain unavailable by default. +- `[~]` The targeted service profile independently reverified 35 passes. The exact API file has a + 29-pass result outside this sandbox; inside it FastAPI TestClient is blocked by AnyIO self-pipe + `EPERM`, a sandbox restriction rather than an application failure. +- `[ ]` Live production composition adapters, full T022 scope, recurring-episode classification from + terminal signals, and an Axiom + semantic-index rebuild remain unresolved; SCEX-FR-001..016 are not globally closed. #endregion ScenarioExecution.Spec diff --git a/specs/044-dashboard-scenario-execution/tasks.md b/specs/044-dashboard-scenario-execution/tasks.md index 2613c7558..051027035 100644 --- a/specs/044-dashboard-scenario-execution/tasks.md +++ b/specs/044-dashboard-scenario-execution/tasks.md @@ -5,6 +5,9 @@ ## Format: `- [ ] T### [P] [USx] Description with exact file path` +> **Factual audit 2026-08-20:** `[x]` means code plus relevant evidence; `[~]` means partial +> implementation; `[ ]` means absent integration or unperformed verification. + ## Phase 1 — Setup - [x] T001 Create `ScenarioRun`/`ScenarioStepRun` models + alembic migration in `backend/src/models/scenario_run.py` @@ -22,17 +25,17 @@ - [x] T006 [US1] Write failing `start_run` tests in `backend/tests/services/dashboard_testing/registry/test_scenario_runner.py` @TEST_EDGE: prod_without_approval->blocked; stale_revision->blocked -- [x] T007 [US1] Implement `ScenarioExecutorRegistry` + typed dispatch boundary in `execution/executor_registry.py` -- [x] T008 [US1] Implement `start_run` in `execution/runner.py` +- [~] T007 [US1] Implement `ScenarioExecutorRegistry` + typed dispatch boundary in `execution/executor_registry.py` +- [~] T008 [US1] Implement `start_run` in `execution/runner.py` @POST: run pinned to immutable revision; queued->running; RunnerPlan deterministically derived ## Phase 4 — US2 Deterministic Dispatch - [x] T009 [US2] Write failing `dispatch_step` tests in `backend/tests/services/dashboard_testing/registry/test_scenario_dispatch.py` -- [x] T010 [US2] Implement `dispatch_step` in `execution/dispatch.py` +- [~] T010 [US2] Implement `dispatch_step` in `execution/dispatch.py` @INVARIANT: human not dispatched here; deterministic per tool; ref binding @TEST_EDGE: unknown_tool->rejected; step_fail->dependents blocked; ref_binding->dependent reads producer output -- [x] T011 [US2] Implement dependency readiness + failure propagation in `execution/dispatch.py` +- [~] T011 [US2] Implement dependency readiness + failure propagation in `execution/dispatch.py` ## Phase 5 — US3 Human Suspend/Resume @@ -46,19 +49,23 @@ - [x] T014b [P] Write worker claim/lease/idempotency contracts in `backend/tests/services/dashboard_testing/registry/test_scenario_worker.py` - [x] T014c [P] Implement `claim_step` (lease/heartbeat/idempotency) in `execution/worker.py` - @INVARIANT: at-least-once; external side effect not re-run unless idempotent/retry-safe + @INVARIANT: at-least-once; persisted lease/idempotency checks are proven; an external side + effect is not re-run unless idempotent/retry-safe. - [x] T014d [P] Implement `require_action_gate` (PROD authorization) + duplicate Idempotency-Key rejection in `execution/runner.py` ## Phase 5c — RunnerPlan derivation + artifacts (P0 #2/#4) -- [x] T014e [P] Implement `derive_runner_plan` (deterministic derivation from revision) in `execution/runner_plan.py` +- [~] T014e [P] Implement `derive_runner_plan` (deterministic derivation from revision) in `execution/runner_plan.py` @INVARIANT: runner.plan.json is reference, never runtime source of truth; revision mismatch rejected -- [x] T014f [P] Generalize artifact owner (owner_type=scenario_run) for evidence/screenshot/report in `execution/artifacts.py` +- [~] T014f [P] Generalize artifact owner (owner_type=scenario_run) for evidence/screenshot/report in `execution/artifacts.py` ## Phase 5d — Retry closure + result aggregation (P0 #14/#16) - [x] T014g [P] Implement retry downstream-closure invalidation in `execution/lifecycle.py` -- [x] T014h [P] Implement result aggregation truth table in `execution/result.py` + @INVARIANT: strict eligible retry retires the active target/descendant closure, archives prior + step attempts, and preserves artifact rows as inactive historical provenance. A reterminalized + attempt has a new immutable terminal signal; replay of the same terminal context is idempotent. +- [~] T014h [P] Implement result aggregation truth table in `execution/result.py` ## Phase 6 — US4 Cancel/Retry/Timeout @@ -68,17 +75,70 @@ ## Phase 7 — US5 Snapshot + API + PROD gate -- [x] T017 [US5] Implement immutable execution snapshot + provenance in `execution/result.py` -- [x] T018 [US5] Add `POST /scenario-runs` (Idempotency-Key), `cancel`, `resume`, `GET /{id}`, `GET /{id}/events` (SSE), `GET /scenarios/{id}/runs`, `GET /scenario-runs/{id}/result`, `GET /scenario-runs/compare`, `POST /scenario-runs/{id}/steps/{logical_step_id}/retry` in `api/routes/dashboard_testing/scenario_runs.py` -- [x] T019 [US5] PROD approval gate (036) + RBAC scenario:run / scenario:run:prod tests +- [~] T017 [US5] Implement immutable execution snapshot + provenance in `execution/result.py` +- [~] T018 [US5] Add `POST /scenario-runs` (Idempotency-Key), `cancel`, `resume`, `GET /{id}`, `GET /{id}/events` (SSE), `GET /scenarios/{id}/runs`, `GET /scenario-runs/{id}/result`, `GET /scenario-runs/compare`, `POST /scenario-runs/{id}/steps/{logical_step_id}/retry` in `api/routes/dashboard_testing/scenario_runs.py` +- [~] T019 [US5] PROD approval gate (036) + RBAC scenario:run / scenario:run:prod tests. + Server ConfigManager policy now ignores client `is_prod` compatibility flags, fails closed for + unknown environments, and gives manual/API/event/scheduled origins the same durable + `pending_approval` gate before dispatcher CAS. A dedicated real APScheduler scheduled-PROD + integration test remains coverage debt; it does not authorize dispatch around the gate. - [x] T020 [P] Frontend DTOs (ScenarioRun, ScenarioStepRun, ScenarioExecutionResult) in `frontend/src/types/scenario-run.ts` ## Phase 8 — Polish - [x] T021 [P] Belief-runtime instrumentation for C5 runner/dispatch -- [x] T022 Run quickstart-equivalent full scenario backend scope, scoped ruff, ATTN/orphan static audit and semantic rebuild. - Verified: 296 passed/1 skipped; ruff clean; one Alembic head; doc-gen indexed 9576 contracts/4730 edges. -- [x] T023 **Prototype validation**: every declared @UX_STATE is reachable via `prototype/index.html`; Playwright verified 1440x900 and 390x844 without overflow or control overlap. +- [ ] T022 Run quickstart-equivalent full scenario backend scope, scoped ruff, ATTN/orphan static audit and semantic rebuild. +- [x] T023 **Prototype validation**: every declared @UX_STATE is reachable via `prototype/index.html`. + Proof: `python specs/044-dashboard-scenario-execution/prototype/validate_static.py`. + +## Audit Follow-ups (2026-08-20) + +- [~] T024 Replace synthetic default executor outcomes with typed adapter boundaries for required + 037/038/036 and existing browser/XLSX infrastructure; add an import/compile test for the + executor module. Assertion uses 037 compare_values; browser/Superset/Screenshot never + synthesize PASS without explicit typed adapter success, and invalid evidence digest/ref is + inconclusive. The Superset binding slice persists an immutable identity snapshot and invokes + the existing 037 query envelope only after an exact composition-owned resolver match, storing + the exact raw-byte digest/ref. This is fail-safe partial closure, not proof of real live + composition. +- [~] T025 Add falsifiable DAG tests for actual executor output, failures, descendants, artifact writes, + checkpoint resume, timeout/cancel drain, recovery and approval-to-dispatch. + Failed assertion now blocks descendants and xlsx registers an artifact + (test_failed_assertion_blocks_descendants_and_xlsx_registers_artifact). Cancellation now + persists a one-time drain deadline: queued work is skipped, a claimed adapter may finish only + during the window, and an expired sweep retires leases/current projections while preserving + immutable evidence history. A timeout during claimed adapter I/O wins over a late PASS, + materializes every otherwise-lazy declared descendant as a durable blocked row from the + pinned RunnerPlan, and emits one inconclusive signal (proof: + test_scenario_cancel_timeout.py). Human and infrastructure resume advance only the missing + DAG frontier, so completed steps are not re-run. Server-driven crash recovery loads only the + persisted run/RunnerPlan and expired lease: safe frontier attempts are archived/retried once, + unsafe effects require reconciliation, and browser recovery requires a pinned safe checkpoint + (test_scenario_crash_recovery.py). Retired evidence remains historical and rejected terminal + contexts reuse their idempotent signal. Persisted artifact evidence needs a real valid digest/ref. + Approval-to-live-dispatch still needs an authorized production composition root that supplies + resolver/client/model/storage; Browser/Screenshot bindings remain unavailable and fail closed. + Queued HTTP/automation starts remain non-dispatching; the existing scheduler claims eligible + rows through durable queued->running CAS before walking them, so repeated ticks do not repeat + a side effect. A manual human run reaches its checkpoint only after that claim, while an + automated human plan is rejected before a row reaches CAS/walker + (test_scenario_queued_dispatch.py, test_scenario_manual_run_only.py). + Fixed queue/cancel callbacks are unit-proven to retain their exact five-second singleton/ + coalescing registration, contain database-edge errors, and preserve terminal side effects on + repeated ticks (test_scenario_scheduler_callbacks.py); this does not prove a live scheduler + process, browser composition, or T022 closure. +- [x] T026 [P] Emit failed/blocked/inconclusive terminal ScenarioRuns as one idempotent 047 queue + producer signal with immutable run/artifact provenance; passed runs emit none. The producer + never opens a case, AgentRun, chat, remediation action, or recurrence classification. + Proof: `test_scenario_terminal_signals.py`. +- [x] T027 [P] Derive `manual_run_only` from a persisted human graph and reject every trusted 046 + automation origin before idempotency or run/gate/notification/queue/dispatch side effects. + The rejected key remains valid for a manual start; HumanCheckpoint is not ActionApprovalGate. + Proof: `test_scenario_manual_run_only.py`, `test_scenario_automation_api.py`. + +> **Verified profile (2026-08-20):** the targeted service suite independently reverified 35 passes. +> The exact API file has a 29-pass result outside this sandbox; here FastAPI TestClient is blocked by +> AnyIO self-pipe `EPERM`, which is a sandbox limitation rather than an application failure. ## Dependencies diff --git a/specs/044-dashboard-scenario-execution/traceability.md b/specs/044-dashboard-scenario-execution/traceability.md index 13d74a6b9..80ec875a9 100644 --- a/specs/044-dashboard-scenario-execution/traceability.md +++ b/specs/044-dashboard-scenario-execution/traceability.md @@ -1,12 +1,19 @@ # Traceability: Scenario Execution Engine (044) -| Story | Requirement | Model | API operationId | Contract | Task | Test | -|-------|-------------|-------|------------------|----------|------|------| -| US1 Start | SCEX-FR-001/008 | ScenarioRun | scenarioRun.start | Execution.Start | T006-T008 | test_runner | -| US2 Dispatch | SCEX-FR-002/006/009 | ScenarioStepRun | scenarioRun.step | Execution.Dispatch | T009-T011 | test_dispatch | -| US3 Human | SCEX-FR-004/010 | ScenarioRun(waiting_human) | scenarioRun.humanDecision | Execution.SuspendForHuman, Execution.Resume | T012-T014 | test_human_resume | -| US4 Lifecycle | SCEX-FR-005 | ScenarioRun(status) | scenarioRun.cancel | Execution.Cancel, Execution.RetryStep | T015-T016 | test_lifecycle | -| US5 Snapshot/API | SCEX-FR-003/007 | ScenarioExecutionResult | scenarioRun.detail, scenarioRun.events | Execution.RunnerPlan | T017-T020 | test_result, test_api | -| Gate/RBAC | SCEX-FR-008 | — | scenarioRun.start | — | T019 | test_rbac | +| Story | Requirement | Model | API operationId | Contract | Task | Test | Actual status / gap | +|-------|-------------|-------|------------------|----------|------|------|---------------------| +| US1 Start | SCEX-FR-001/008 | ScenarioRun | scenarioRun.start | Execution.Start, Execution.Runner.QueuedDispatch | T006-T008, T024 | test_runner, test_scenario_queued_dispatch, test_scenario_scheduler_callbacks, test_scenario_runs_api, test_scenario_automation_api, test_live_execution_binding | `[~]` HTTP/automation start and replay persist queued/pending rows without request-time dispatch; only the scheduler composition's durable queued->running CAS walks its winner. Fixed scheduler callback registration, database-edge containment, and repeat-tick terminal side-effect idempotency are unit-proven. Automated human plans are rejected before they reach CAS/walker; a manual human graph reaches HumanCheckpoint only after that dispatcher claim. Approval-to-real live dispatch remains unproven. | +| US2 Dispatch | SCEX-FR-002/006/009 | ScenarioStepRun, LiveExecutionBinding | scenarioRun.step | Execution.Dispatch, Execution.LiveCompositionRoot | T009-T011, T024 | test_dispatch, test_scenario_executors, test_live_execution_binding | `[~]` Browser/Superset/Screenshot use fail-safe typed adapter boundaries: no explicit adapter success means no PASS; invalid evidence digest/ref remains inconclusive. Lifespan bootstraps `settings.scenario_live_execution_bindings` through the existing `SupersetClient`, exact model and durable storage, so configured Superset dispatch invokes 037 and stores the exact raw-byte digest/ref; mismatched/unavailable providers make no I/O call. Browser safe-checkpoint and Screenshot durable-evidence registration are supported but no provider is deployed, so enabled bindings return stable configured-unavailable codes. | +| US3 Human | SCEX-FR-004/010 | ScenarioRun(waiting_human) | scenarioRun.humanDecision | Execution.SuspendForHuman, Execution.Resume | T012-T014, T025 | test_human_resume, test_scenario_runs_api | `[~]` persisted HumanCheckpoint and infrastructure-resume continuations advance only the missing DAG frontier; completed steps are not re-run. Full live-composition closure remains pending. | +| Manual-only boundary | SCEX-FR-004a | ScenarioRun, HumanCheckpoint | — | Execution.Runner.Start, RunnerPlan.Derive | T027 | test_scenario_manual_run_only, test_scenario_automation_api | `[x]` Trusted scheduled/deploy/release/ETL/API origins reject persisted human revisions before idempotency or any run/gate/notification/queue side effect. Manual origin remains eligible; HumanCheckpoint is not an approval gate. | +| US4 Lifecycle | SCEX-FR-005/006 | ScenarioRun(status, cancel deadline), ScenarioArtifact(active projection) | scenarioRun.cancel, scenarioRun.retry | Execution.Lifecycle.Cancel, Execution.Lifecycle.Timeout, Execution.Lifecycle.Retry, Execution.Runner.ContinueAfterRetry | T014g, T015-T016, T025 | test_scenario_lifecycle, test_scenario_cancel_timeout, test_scenario_retry_closure | `[~]` Strict eligible retry invalidates the persisted target/descendant closure, archives old step attempts, and retains artifact rows only as inactive historical provenance. Cancellation pins a durable bounded drain deadline; deadline finalization expires leases and retires active projections without deleting audit evidence. A timeout during adapter I/O wins over a late PASS and materializes every otherwise-lazy pinned-plan descendant as blocked, so continuation cannot dispatch it; it emits one idempotent inconclusive signal. Full live-I/O composition remains fail-closed/unproven. | +| US5 Snapshot/API | SCEX-FR-003/007 | ScenarioExecutionResult | scenarioRun.detail, scenarioRun.events | Execution.RunnerPlan, Execution.Runner.CrashRecovery | T017-T020, T025 | test_result, test_api, test_scenario_runner_walker, test_scenario_crash_recovery | `[~]` API/projection and real-digest artifact integrity checks exist. Server-driven crash recovery uses only the persisted run/RunnerPlan and expired lease: completed work is not rerun, safe claims get a new attempt with active evidence retired to history, unsafe claims require reconciliation, and browser recovery without a pinned safe checkpoint is non-pass without adapter I/O. Rejected terminal contexts reuse their idempotent signal. Production live-executor provenance remains unproven. | +| Terminal signals | SCEX-FR-011 | InvestigationQueueItem | — | Execution.Runner.TerminalSignal | T026 | test_scenario_terminal_signals | `[x]` Failed/blocked/inconclusive terminal runs emit one idempotent immutable 047 queue input with run/artifact provenance; passed runs emit none. Producer-only ingestion starts no case, AgentRun, chat, remediation action, or recurrence classification. | +| Gate/RBAC | SCEX-FR-008 | ScenarioRun, ActionApprovalGate | scenarioRun.start | Execution.EnvironmentPolicy, Execution.Runner.Start | T019 | test_scenario_runner, test_scenario_runs_api, test_scenario_automation_api, test_scenario_automation_trigger | `[~]` Server ConfigManager classifies every target before persistence: client flags cannot select PROD, unknown targets create no run-side effect, and every trusted source enters the same durable pending_approval gate boundary. HTTP/trigger paths remain persistence-only and dispatcher excludes pending gates. A dedicated real APScheduler scheduled-PROD integration test remains coverage debt; this is not a dispatch bypass. | N/A: Registry (042), Editor (043), Monitor UX (045), Automation (046), Analytics (047). + +**Verification boundary (2026-08-20):** the targeted service profile independently reverified 35 +passes. The exact API file has a 29-pass result outside this sandbox; inside it, FastAPI TestClient +is blocked by AnyIO self-pipe `EPERM`, a sandbox restriction rather than an application failure. +T022, prototype validation, live production composition adapters, and the Axiom index remain open. diff --git a/specs/045-dashboard-run-monitor/checklists/requirements.md b/specs/045-dashboard-run-monitor/checklists/requirements.md index 030505ca9..4155039d1 100644 --- a/specs/045-dashboard-run-monitor/checklists/requirements.md +++ b/specs/045-dashboard-run-monitor/checklists/requirements.md @@ -2,9 +2,12 @@ **Purpose**: Verify RUNMON-FR-001..008 completeness. | **Created**: 2026-08-07 +> **Factual audit 2026-08-20:** `[x]` requires current production evidence, `[~]` means partial +> code exists, `[ ]` means missing integration or proof. + ## Configure & Launch (FR-001) -- [ ] CHK001 Run config includes environment, revision, release, baseline set, parameters, toggles +- [~] CHK001 Run config includes environment, revision, release, baseline set, parameters, toggles - [ ] CHK002 PROD requires 036 approval gate; denial dispatches nothing ## Live Monitor (FR-002/003) diff --git a/specs/045-dashboard-run-monitor/plan.md b/specs/045-dashboard-run-monitor/plan.md index b8a37b7ef..560100a37 100644 --- a/specs/045-dashboard-run-monitor/plan.md +++ b/specs/045-dashboard-run-monitor/plan.md @@ -1,6 +1,9 @@ # Implementation Plan: Scenario Run Monitor & Results UX -**Branch**: `045-dashboard-run-monitor` | **Date**: 2026-08-07 | **Spec**: spec.md | **Status**: Draft +**Branch**: `045-dashboard-run-monitor` | **Date**: 2026-08-07 | **Spec**: spec.md | **Status**: Partially implemented — factual audit pending remediation + +> **Implementation audit, 2026-08-20:** monitor UI exists, but typed launch-contract alignment and +> end-to-end evidence depend on unresolved 044 execution and 047 investigation ingestion. ## Summary diff --git a/specs/045-dashboard-run-monitor/prototype/manifest.md b/specs/045-dashboard-run-monitor/prototype/manifest.md index 8bb806008..895d91d87 100644 --- a/specs/045-dashboard-run-monitor/prototype/manifest.md +++ b/specs/045-dashboard-run-monitor/prototype/manifest.md @@ -2,6 +2,9 @@ @defgroup Prototype Interactive HTML prototype manifest for Scenario Run Monitor & Results. ## Prototype Metadata + +> **Factual audit 2026-08-20:** prototype state coverage is not evidence that the 044 launch/event +> contract or 047 investigation handoff works end-to-end. - **Feature**: 045 Scenario Run Monitor & Results UX - **Source contracts**: ux_reference.md, contracts/modules.md - **Screens represented**: 4 (Config, Live Monitor, Result, History/Compare) + Investigation Queue handoff → 047 @@ -23,7 +26,7 @@ ## Screen ↔ Story Traceability -| Story | Prototype Feature | Acceptance Verified | +| Story | Prototype Feature | Intended acceptance coverage | |-------|-------------------|---------------------| | US1 Launch | config state | env/revision/baseline/toggles | | US2 Live | running timeline + step rows | events-only rendering | diff --git a/specs/045-dashboard-run-monitor/quickstart.md b/specs/045-dashboard-run-monitor/quickstart.md index f7a73fbc2..8a779c6f9 100644 --- a/specs/045-dashboard-run-monitor/quickstart.md +++ b/specs/045-dashboard-run-monitor/quickstart.md @@ -1,5 +1,8 @@ # Quickstart: Scenario Run Monitor & Results UX (045) +> **Factual audit 2026-08-20:** pending verification checklist only; launch-contract and investigation +> handoff scenarios depend on remediation in 044 and 047. + ## Prereqs - 044 execution API live (scenario-runs, events, human/decision) - Frontend deps installed diff --git a/specs/045-dashboard-run-monitor/spec.md b/specs/045-dashboard-run-monitor/spec.md index dafcf3ca7..55ae8e896 100644 --- a/specs/045-dashboard-run-monitor/spec.md +++ b/specs/045-dashboard-run-monitor/spec.md @@ -13,7 +13,7 @@ @SEMANTICS: spec, requirements, feature, ux, scenario, run, monitor, result, history, compare **Feature Branch**: `045-dashboard-run-monitor` -**Created**: 2026-08-07 | **Status**: Draft +**Created**: 2026-08-07 | **Status**: Partially implemented — factual audit pending remediation **Input**: "Provide a live Scenario Run Monitor and Results UX: persistent run configuration panel, live step timeline with per-step status/duration/logs/evidence, human checkpoint actions inside the monitor, final result/provenance, comparison, and an entry to agent-led investigation." ## User Scenarios @@ -150,11 +150,15 @@ - Q: How does it differ from 039 VerificationHistoryList? → A: 039 renders 037 VerificationRun (release pipeline); 045 renders ScenarioRun (user-created scenario execution). Distinct entities, distinct labels. - Q: Reuse of 040? → A: Reuse the LoadRunComparison UX concept for run comparison. -## Implementation Status & MVP Debt (audit 2026-08-07) +## Implementation Status & MVP Debt (factual audit 2026-08-20) -**Facts (code check):** -- 🟡 039 `VerificationHistoryList`/`VerificationStatusBadge`/`StructureDiffPanel` exist but bind to 037 VerificationRun — NOT scenario runs. -- 🔴 No Run Monitor, Run Configuration, ScenarioResultView, or ScenarioRun history/compare UI. -- 🟡 SSE/WS patterns exist in 036/040; reusable for scenario run events. +Monitor model, timeline, checkpoint/result/history/compare components and scenario-run routes exist. + +- `[~]` `RunMonitorModel` launches runs, but release, baseline set and execution toggles are embedded in + `params.launch_config` rather than represented by the 044 typed start contract. +- `[~]` The UI can render typed events, but its runtime truth is limited by incomplete 044 execution. +- `[ ]` Failed/blocked/inconclusive investigation entry is not end-to-end because 047 does not ingest + production signals into the queue. +- `[ ]` Current browser, reconnect and accessibility evidence has not been retained. #endregion ScenarioRunMonitor.Spec diff --git a/specs/045-dashboard-run-monitor/tasks.md b/specs/045-dashboard-run-monitor/tasks.md index 82a90218e..a18b7273a 100644 --- a/specs/045-dashboard-run-monitor/tasks.md +++ b/specs/045-dashboard-run-monitor/tasks.md @@ -5,6 +5,9 @@ ## Format: `- [ ] T### [P] [USx] Description with exact file path` +> **Factual audit 2026-08-20:** `[x]` means code plus relevant evidence; `[~]` means partial +> implementation; `[ ]` means absent integration or unperformed verification. + ## Phase 1 — Setup - [x] T001 Define frontend DTOs (RunConfiguration, ScenarioResultView, RunComparison) in `frontend/src/types/scenario-run.ts` @@ -54,7 +57,7 @@ ## Phase 6c — Run Configuration binding (P0 #12/#17) -- [x] T014f [P] Bind `RunConfigurationPanel.svelte` to 044 start contract (release/baseline_set/toggles); mandatory steps not toggleable +- [~] T014f [P] Bind `RunConfigurationPanel.svelte` to 044 start contract (release/baseline_set/toggles); mandatory steps not toggleable - [x] T014g [P] L2 UX test for mandatory-step toggle protection Test: `frontend/src/lib/components/scenario-run/__tests__/RunConfigurationPanel.test.ts` @@ -62,9 +65,15 @@ - [x] T015 [P] Distinct labeling from 037/040 in all UI (copy + aria) Present in: RunConfigurationPanel, ScenarioResultView, GlobalRunCenter route header -- [x] T016 Run quickstart-equivalent monitor checks, full frontend tests, ATTN static audit and semantic rebuild. - Verified: 3927 frontend tests passed, production build succeeded, and doc-gen semantic navigation rebuilt. -- [x] T017 **Prototype validation**: every declared @UX_STATE is reachable via `prototype/index.html`; Playwright verified 1440x900 and 390x844 without overflow or control overlap. +- [ ] T016 Run quickstart-equivalent monitor checks, full frontend tests, ATTN static audit and semantic rebuild. +- [ ] T017 **Prototype validation**: every declared @UX_STATE is reachable via `prototype/index.html`. + +## Audit Follow-ups (2026-08-20) + +- [ ] T018 Extend the 044 start contract with typed release, baseline set and optional evidence toggles; + remove `params.launch_config` transport and prove mandatory steps remain non-toggleable. +- [ ] T019 Add monitor E2E evidence using real 044 outcomes and 047 queue entries after their runtime + contracts are implemented. ## Dependencies diff --git a/specs/045-dashboard-run-monitor/traceability.md b/specs/045-dashboard-run-monitor/traceability.md index df764e4cb..c8df3b2ee 100644 --- a/specs/045-dashboard-run-monitor/traceability.md +++ b/specs/045-dashboard-run-monitor/traceability.md @@ -1,5 +1,8 @@ # Traceability: Scenario Run Monitor & Results (045) +> **Factual audit 2026-08-20:** rows identify code/test ownership only. The launch contract currently +> transports release/baseline/toggles through `params.launch_config`; complete 044/047 runtime proof is open. + | Story | Requirement | Model | API operationId | Contract | Task | Test | |-------|-------------|-------|------------------|----------|------|------| | US1 Launch | RUNMON-FR-001 | RunConfiguration | scenarioRun.start | RunMonitor.Launch | T003-T005 | RunMonitorModel.test, run.ux.test | diff --git a/specs/046-dashboard-scenario-automation/checklists/requirements.md b/specs/046-dashboard-scenario-automation/checklists/requirements.md index cf327f381..a97f086cf 100644 --- a/specs/046-dashboard-scenario-automation/checklists/requirements.md +++ b/specs/046-dashboard-scenario-automation/checklists/requirements.md @@ -2,6 +2,9 @@ **Purpose**: Verify SCAUTO-FR-001..009 completeness. | **Created**: 2026-08-07 +> **Factual audit 2026-08-20:** `[x]` requires current production evidence, `[~]` means partial +> code exists, `[ ]` means missing integration or proof. Human checkpoints are manual-run-only. + ## Schedule & Trigger (FR-001/002) - [ ] CHK001 Runs triggerable by manual, PREPROD deploy, release-created, ETL, scheduled, API @@ -10,7 +13,7 @@ ## Notifications (FR-003) -- [ ] CHK004 Domain events: completed, failed, blocked, human-action-required, scenario-stale, repeated-flaky-failure +- [ ] CHK004 Domain events: completed, failed, blocked, scenario-stale, repeated-flaky-failure ## Policy (FR-004/005/006/007) @@ -21,7 +24,7 @@ ## Metrics & Boundary (FR-008/009) -- [ ] CHK009 Operational metrics: scheduled counts, success rate, trigger distribution, repeated-failure alerts +- [~] CHK009 Operational metrics: scheduled counts, success rate, trigger distribution, repeated-failure alerts - [ ] CHK010 Zero writes to 037 baseline catalog / verification pipeline ## Success Criteria diff --git a/specs/046-dashboard-scenario-automation/plan.md b/specs/046-dashboard-scenario-automation/plan.md index 472bdba74..257ad01f2 100644 --- a/specs/046-dashboard-scenario-automation/plan.md +++ b/specs/046-dashboard-scenario-automation/plan.md @@ -1,6 +1,9 @@ # Implementation Plan: Scenario Automation & Operations -**Branch**: `046-dashboard-scenario-automation` | **Date**: 2026-08-07 | **Spec**: spec.md | **Status**: Draft +**Branch**: `046-dashboard-scenario-automation` | **Date**: 2026-08-07 | **Spec**: spec.md | **Status**: Not production-complete — factual audit pending remediation + +> **Implementation audit, 2026-08-20:** management and pure helpers exist; event dispatch, lifecycle +> notifications, startup schedule reload and persisted APScheduler semantics remain open. ## Summary diff --git a/specs/046-dashboard-scenario-automation/prototype/manifest.md b/specs/046-dashboard-scenario-automation/prototype/manifest.md index b10dbaaf0..954bb16fb 100644 --- a/specs/046-dashboard-scenario-automation/prototype/manifest.md +++ b/specs/046-dashboard-scenario-automation/prototype/manifest.md @@ -2,6 +2,9 @@ @defgroup Prototype Interactive HTML prototype manifest for Scenario Automation & Operations. ## Prototype Metadata + +> **Factual audit 2026-08-20:** prototype coverage is not evidence of event dispatch, scheduler +> restart/reload, notification persistence, or PROD runtime behavior. - **Feature**: 046 Scenario Automation & Operations - **Source contracts**: ux_reference.md, contracts/modules.md - **Screens represented**: 1 (Automation Config) @@ -21,7 +24,7 @@ ## Screen ↔ Story Traceability -| Story | Prototype Feature | Acceptance Verified | +| Story | Prototype Feature | Intended acceptance coverage | |-------|-------------------|---------------------| | US1 Schedule | trigger checkboxes | deploy/daily/release/ETL | | US2 Notify | notify toggle + metrics flaky alert | notification events | diff --git a/specs/046-dashboard-scenario-automation/quickstart.md b/specs/046-dashboard-scenario-automation/quickstart.md index 872a6812e..ab15c59c2 100644 --- a/specs/046-dashboard-scenario-automation/quickstart.md +++ b/specs/046-dashboard-scenario-automation/quickstart.md @@ -1,5 +1,8 @@ # Quickstart: Scenario Automation & Operations (046) +> **Factual audit 2026-08-20:** pending verification checklist only; scheduler/event/notification +> integration is not yet production-complete. + ## Prereqs - 044 run API, 042 registry, DB migrated (scenario_automation tables) - APScheduler infrastructure (037) available diff --git a/specs/046-dashboard-scenario-automation/spec.md b/specs/046-dashboard-scenario-automation/spec.md index f5097b9b8..71561a201 100644 --- a/specs/046-dashboard-scenario-automation/spec.md +++ b/specs/046-dashboard-scenario-automation/spec.md @@ -13,7 +13,7 @@ @SEMANTICS: spec, requirements, feature, scenario, automation, schedule, trigger, notification, operations **Feature Branch**: `046-dashboard-scenario-automation` -**Created**: 2026-08-07 | **Status**: Draft +**Created**: 2026-08-07 | **Status**: Not production-complete — factual audit pending remediation **Input**: "Provide Scenario Automation & Operations: schedule and trigger scenario runs (manual, on PREPROD deploy, on release created, after ETL, scheduled, API), notifications for completion/failure/blocked/stale, concurrency policies, retention, deduplication, and operational metrics — reusing the 037 trigger framework." ## User Scenarios @@ -131,11 +131,30 @@ - Q: New scheduler? → A: No. Reuse the 037 trigger framework + existing APScheduler. - Q: Where do results go? → A: Scenario runs (044), never into 037 baseline/verification. -## Implementation Status & MVP Debt (audit 2026-08-07) +## Implementation Status & MVP Debt (factual audit 2026-08-20) -**Facts (code check):** -- 🟡 037 has trigger semantics for VerificationRun (`release_create`, `scheduled`) in `verification_scheduler.py` + release hooks. -- 🔴 No scenario schedule/trigger/notification framework exists; 044 runs are manual/API only. -- 🟡 APScheduler infrastructure exists (037), reusable. +Models, CRUD API, management UI and pure schedule/policy/retention/metrics helpers are present, but +the automation workflow is not wired end-to-end. + +- `[~]` `handle_trigger_event()` returns policy-checked candidates and `dispatch_trigger_event()` + passes each real server-owned event origin into 044 `start_run`. A persisted human revision fails + manual-run-only pre-create with no run-side effect. HTTP/scheduler trigger paths persist only + queued or server-gated pending_approval rows; separate 044 queued->running CAS dispatch is the + sole initial adapter authority and excludes pending gates. No long-running event-subscriber proof + is yet available. +- `[ ]` `notify()` only appends to a caller-provided list and is not connected to run, stale or repeated + failure lifecycle events. +- `[~]` Scheduler startup reloads enabled `ScenarioSchedule` rows and registration forwards their + timezone, `misfire_grace_time`, `max_instances`, and missed-execution policy (derived coalesce). + Independent callback tests also prove the fixed 044 queued-dispatch and cancel-drain jobs register + their exact IDs, five-second intervals, singleton/coalescing options, and contain database-edge + failures; repeated callbacks preserve their exact terminal side-effect counts. This is not proof + of cron scheduled-scenario firing, a live scheduler process, or subscriber composition. Existing + scheduler baseline lint debt is separate and remains unclosed. +- `[~]` Server-owned EnvironmentPolicy now sends manual/API/event/scheduled origins through the + same 044 durable `pending_approval` gate and excludes pending rows from dispatcher execution; + client flags and unknown environments cannot bypass it. A dedicated real APScheduler + scheduled-PROD integration test is still required, alongside the remaining retention/runtime + workflow tests, before feature closure. #endregion ScenarioAutomation.Spec diff --git a/specs/046-dashboard-scenario-automation/tasks.md b/specs/046-dashboard-scenario-automation/tasks.md index 69ca53022..cf260f6ea 100644 --- a/specs/046-dashboard-scenario-automation/tasks.md +++ b/specs/046-dashboard-scenario-automation/tasks.md @@ -5,6 +5,9 @@ ## Format: `- [ ] T### [P] [USx] Description with exact file path` +> **Factual audit 2026-08-20:** `[x]` means code plus relevant evidence; `[~]` means partial +> implementation; `[ ]` means absent integration or unperformed verification. + ## Phase 1 — Setup - [x] T001 Create `ScenarioSchedule`, `ScenarioTriggerRule`, `NotificationEvent` models in `backend/src/models/scenario_automation.py` @@ -12,30 +15,30 @@ ## Phase 2 — US1 Schedule & Trigger -- [x] T003 [US1] Write failing schedule/trigger tests (trigger semantics contract coverage: +- [~] T003 [US1] Write failing schedule/trigger tests (trigger semantics contract coverage: `backend/tests/services/dashboard_testing/registry/test_scenario_automation_trigger.py` — filename differs from plan; coverage closes the task: event->run, release_create->run, ETL->run, disabled/mismatch skip, capacity/PROG/dedup gates) -- [x] T004 [US1] Implement `upsert_schedule` schedule semantics in `automation/schedule.py` -- [x] T005 [US1] Implement `handle_trigger_event` policy-bound event mapping in `automation/trigger.py` +- [~] T004 [US1] Implement `upsert_schedule` schedule semantics in `automation/schedule.py` +- [~] T005 [US1] Implement `handle_trigger_event` policy-bound event mapping in `automation/trigger.py` @POST: matched rules start runs via 044; pinned revision+env @TEST_EDGE: event->run; release_create->run; ETL->run ## Phase 3 — US2 Notifications -- [x] T006 [US2] Write notification tests in `backend/tests/services/dashboard_testing/registry/test_scenario_automation.py` -- [x] T007 [US2] Implement `notify` in `automation/notify.py` +- [~] T006 [US2] Write notification tests in `backend/tests/services/dashboard_testing/registry/test_scenario_automation.py` +- [~] T007 [US2] Implement `notify` in `automation/notify.py` ## Phase 4 — US3 Concurrency/Dedup/Retention -- [x] T008 [US3] Write failing policy/retention tests in `backend/tests/services/dashboard_testing/registry/test_scenario_automation_policy.py` -- [x] T009 [US3] Implement `apply_policy` (concurrency/dedup/overlap/PROD gate) in `backend/src/services/dashboard_testing/automation/policy.py` +- [~] T008 [US3] Write failing policy/retention tests in `backend/tests/services/dashboard_testing/registry/test_scenario_automation_policy.py` +- [~] T009 [US3] Implement `apply_policy` (concurrency/dedup/overlap/PROD gate) in `backend/src/services/dashboard_testing/automation/policy.py` @INVARIANT: never exceeds max_concurrent_per_env; dedup window honored -- [x] T010 [US3] Implement layered `run_retention` in `automation/retention.py` +- [~] T010 [US3] Implement layered `run_retention` in `automation/retention.py` ## Phase 5 — US4 Operational Metrics -- [x] T011 [US4] Implement `automation_metrics` in `automation/metrics.py` +- [~] T011 [US4] Implement `automation_metrics` in `automation/metrics.py` (verified by `registry/test_scenario_automation_semantics.py` Metrics section + API test) ## Phase 6 — API + Polish @@ -55,7 +58,7 @@ - [x] T013c [P] Automation Management UI mounted: `AutomationPanel.svelte` (schedules/triggers/ policies list + edit) at `frontend/src/routes/dashboard-testing/automation/` (route `+page.svelte` + route test `automation_page.ux.test.ts`; run center links to it) -- [x] T013d [P] APScheduler semantics config (timezone/DST/misfire_grace_time/coalesce/ +- [~] T013d [P] APScheduler semantics config (timezone/DST/misfire_grace_time/coalesce/ max_instances/missed-policy) in `automation/schedule.py` (verified by `registry/test_scenario_automation_semantics.py` Schedule section) - [x] T013e [P] Layered retention tiers (run metadata/triage/step metrics/artifacts/screenshots/ @@ -65,10 +68,28 @@ ## Phase 7 — Polish -- [x] T014 Run scoped backend tests (33 passed: automation service + API + migration) + ruff - (clean on automation service + new tests); frontend scoped vitest (15 passed) + `npm run build` - (ok). Full-suite run + ATTN_1-4 audit + semantic rebuild remain for the batch close. -- [x] T015 **Prototype validation**: every declared @UX_STATE is reachable via `prototype/index.html`; Playwright verified 1440x900 and 390x844 without overflow or control overlap. +- [ ] T014 Run scoped backend tests, ruff, frontend tests and build; record current command-level evidence. +- [ ] T015 **Prototype validation**: every declared @UX_STATE is reachable via `prototype/index.html`. + +## Audit Follow-ups (2026-08-20) + +- [~] T016 Wire deploy/release/ETL/scheduled events through policy/dedup into 044 `start_run`; verify + pinning, PROD gates and idempotency at the production boundary. The dispatcher now forwards + server-owned event provenance before creation, and human revisions reject manual-run-only + without a side effect. HTTP/scheduler callbacks persist only queued or server-gated + pending_approval rows; the separate 044 queued->running CAS owns initial adapter dispatch + and excludes pending gates. Fixed 044 queue/cancel due callbacks now + have unit proof for registration options, database-edge containment, and repeat-tick exact + side-effect idempotency (`test_scenario_scheduler_callbacks.py`); cron scheduled-scenario + due firing and subscriber integration remain open. Add a dedicated real scheduled-PROD + integration test proving server policy produces `pending_approval` plus one durable gate + before dispatcher CAS; this remaining coverage is not a gate bypass. +- [ ] T017 Emit persisted lifecycle notification events and canonical InvestigationSignals for required + run/stale/repeated-failure transitions; prove no auto-started agent work. +- [~] T018 Load enabled ScenarioSchedule rows at scheduler startup and map timezone, misfire grace, + coalesce and max instances into APScheduler. Startup reload/registration code exists; restart + and cron scheduled-scenario due-job behavior still need independent proof. The fixed 044 + queue/cancel maintenance callbacks are separately unit-proven, not live-process proof. ## Dependencies diff --git a/specs/046-dashboard-scenario-automation/traceability.md b/specs/046-dashboard-scenario-automation/traceability.md index c5ad6f27f..4dbb92b02 100644 --- a/specs/046-dashboard-scenario-automation/traceability.md +++ b/specs/046-dashboard-scenario-automation/traceability.md @@ -1,11 +1,11 @@ # Traceability: Scenario Automation & Operations (046) -| Story | Requirement | Model | API operationId | Contract | Task | Test | -|-------|-------------|-------|------------------|----------|------|------| -| US1 Schedule | SCAUTO-FR-001/002 | ScenarioSchedule, ScenarioTriggerRule | automation.schedule, automation.trigger | Automation.Schedule, Automation.Trigger | T003-T005 | test_trigger | -| US2 Notify | SCAUTO-FR-003 | NotificationEvent | automation.notify | Automation.Notify | T006-T007 | test_notify | -| US3 Policy | SCAUTO-FR-004/005/006/007 | AutomationPolicy | automation.policy | Automation.ApplyPolicy, Automation.Retention | T008-T010 | test_policy | -| US4 Metrics | SCAUTO-FR-008 | — | automation.metrics | Automation.Metrics | T011 | test_metrics | -| Boundary | SCAUTO-FR-009 | — | — | — | T013 | test_policy | +| Story | Requirement | Model | API operationId | Contract | Task | Test | Actual status / gap | +|-------|-------------|-------|------------------|----------|------|------|---------------------| +| US1 Schedule | SCAUTO-FR-001/002/007 | ScenarioSchedule, ScenarioTriggerRule, ActionApprovalGate | automation.schedule, automation.trigger | Automation.Schedule, Automation.Trigger, Execution.EnvironmentPolicy, Execution.Runner.QueuedDispatch | T003-T005, T016, T019 | test_trigger, test_scenario_runner, test_scenario_runs_api, test_scenario_automation_api, test_scenario_manual_run_only, test_scenario_queued_dispatch, test_scenario_scheduler_callbacks | `[~]` Event dispatcher passes its server-owned deploy/release/ETL origin into 044 before creation; persisted human revisions reject manual-run-only before a row reaches the dispatch CAS. ConfigManager alone classifies PROD for every source: client flags cannot downgrade it, unknown targets have no side effect, and a PROD intent persists pending_approval plus its durable gate before dispatcher CAS. Fixed 044 queue/cancel callbacks are unit-proven to register exact IDs/interval/singleton-coalescing options, contain database-edge errors, and preserve exact terminal side effects on repeat ticks. Dedicated real scheduled-PROD callback integration, cron firing, a live scheduler process, and long-running subscribers remain open. | +| US2 Notify | SCAUTO-FR-003 | NotificationEvent | automation.notify | Automation.Notify | T006-T007 | test_notify | `[ ]` helper has no run/staleness lifecycle caller. | +| US3 Policy | SCAUTO-FR-004/005/006/007 | AutomationPolicy | automation.policy | Automation.ApplyPolicy, Automation.Retention | T008-T010 | test_policy | `[~]` pure helpers exist; persisted runtime enforcement is unproven. | +| US4 Metrics | SCAUTO-FR-008 | — | automation.metrics | Automation.Metrics | T011 | test_metrics | `[~]` aggregate helper exists; source events/runs are not fully wired. | +| Boundary | SCAUTO-FR-009 | — | — | — | T013 | test_policy | `[~]` routes/RBAC exist; scheduled PROD runtime proof is open. | N/A: Registry (042), Editor (043), Execution (044), Monitor (045), Analytics (047). diff --git a/specs/047-dashboard-scenario-analytics/checklists/requirements.md b/specs/047-dashboard-scenario-analytics/checklists/requirements.md index c893a4cf4..b85c18cc9 100644 --- a/specs/047-dashboard-scenario-analytics/checklists/requirements.md +++ b/specs/047-dashboard-scenario-analytics/checklists/requirements.md @@ -2,6 +2,9 @@ **Purpose**: Verify SCAN-FR-001..011 completeness. | **Created**: 2026-08-07 +> **Factual audit 2026-08-20:** `[x]` requires current production evidence, `[~]` means partial +> code exists, `[ ]` means missing integration or proof. + ## Queue and Case (FR-001..003/009/011) - [ ] CHK001 Qualifying events create/update a deduplicated queue item and never auto-start agent work @@ -11,14 +14,14 @@ ## Flakiness & Health (FR-004..006) -- [ ] CHK005 Per-step flaky detection requires post-failure pass and two transitions; excluded outcomes are not in denominator +- [~] CHK005 Per-step flaky detection requires post-failure pass and two transitions; excluded outcomes are not in denominator - [ ] CHK006 Contextual health: product/test/infra/overall plus confidence - [ ] CHK007 Health feeds 042 registry badge on threshold cross ## Trends & Recurring (FR-007/008/010) -- [ ] CHK008 Success-rate trend + failure-classification distribution -- [ ] CHK009 Recurring failures grouped (count, first/last occurrence) +- [~] CHK008 Success-rate trend + failure-classification distribution +- [~] CHK009 Recurring failures grouped (count, first/last occurrence) - [ ] CHK010 Matching recurrence after resolved episode opens a new alertable FailureEpisode ## Success Criteria diff --git a/specs/047-dashboard-scenario-analytics/plan.md b/specs/047-dashboard-scenario-analytics/plan.md index 8ede62082..7cab436f2 100644 --- a/specs/047-dashboard-scenario-analytics/plan.md +++ b/specs/047-dashboard-scenario-analytics/plan.md @@ -1,6 +1,9 @@ # Implementation Plan: Investigation Queue & Scenario Analytics -**Branch**: `047-dashboard-scenario-analytics` | **Date**: 2026-08-07 | **Spec**: spec.md | **Status**: Draft +**Branch**: `047-dashboard-scenario-analytics` | **Date**: 2026-08-07 | **Spec**: spec.md | **Status**: Not production-complete — factual audit pending remediation + +> **Implementation audit, 2026-08-20:** analytics primitives/UI exist; canonical signal ingestion, +> immutable case evidence and conformant case closure remain open. ## Summary diff --git a/specs/047-dashboard-scenario-analytics/prototype/manifest.md b/specs/047-dashboard-scenario-analytics/prototype/manifest.md index 2d00366eb..7bef12db3 100644 --- a/specs/047-dashboard-scenario-analytics/prototype/manifest.md +++ b/specs/047-dashboard-scenario-analytics/prototype/manifest.md @@ -2,6 +2,9 @@ @defgroup Prototype Interactive HTML prototype manifest for Failure Triage & Quality Analytics. ## Prototype Metadata + +> **Factual audit 2026-08-20:** prototype coverage is not evidence of canonical signal ingestion, +> immutable case evidence, or conformant disposition closure. - **Feature**: 047 Failure Triage & Quality Analytics - **Source contracts**: ux_reference.md, contracts/modules.md - **Screens represented**: 2 (Investigation Queue and persistent agent-led Case; health summary embedded in Queue) @@ -20,7 +23,7 @@ ## Screen ↔ Story Traceability -| Story | Prototype Feature | Acceptance Verified | +| Story | Prototype Feature | Intended acceptance coverage | |-------|-------------------|---------------------| | US1 Investigation | queue item → explicit case, evidence, agent tools, disposition | case/triage projection persisted + audited | | US2 Flakiness/Health | health card + unstable step | flaky detection + health | diff --git a/specs/047-dashboard-scenario-analytics/quickstart.md b/specs/047-dashboard-scenario-analytics/quickstart.md index 0c730ea5a..740b71f71 100644 --- a/specs/047-dashboard-scenario-analytics/quickstart.md +++ b/specs/047-dashboard-scenario-analytics/quickstart.md @@ -1,5 +1,8 @@ # Quickstart: Investigation Queue & Scenario Analytics (047) +> **Factual audit 2026-08-20:** pending verification checklist only; qualifying production signals do +> not currently enter the queue and case closure is not conformant. + ## Prereqs - 044 run/step results, 042 registry, 036 AgentAction contract, DB migrated (investigation tables) diff --git a/specs/047-dashboard-scenario-analytics/spec.md b/specs/047-dashboard-scenario-analytics/spec.md index a79438a48..d076b3f9b 100644 --- a/specs/047-dashboard-scenario-analytics/spec.md +++ b/specs/047-dashboard-scenario-analytics/spec.md @@ -13,7 +13,7 @@ @SEMANTICS: spec, requirements, feature, scenario, triage, flakiness, analytics, health, trend **Feature Branch**: `047-dashboard-scenario-analytics` -**Created**: 2026-08-07 | **Status**: Draft +**Created**: 2026-08-07 | **Status**: Not production-complete — factual audit pending remediation **Input**: "Provide an Investigation Queue and agentic case workspace for failed/stale/load/automation evidence, backed by deterministic flakiness, health, trends and recurring-failure analytics." ## User Scenarios @@ -110,11 +110,20 @@ - Q: Does a case change graph/result? → A: No. A case may create a separately immutable revision or policy-bound action, but it never rewrites historical run truth. - Q: Does every failure open chat? → A: No. It enters a deduplicated Investigation Queue; the analyst explicitly opens the case. -## Implementation Status & MVP Debt (audit 2026-08-07) +## Implementation Status & MVP Debt (factual audit 2026-08-20) -**Facts (code check):** -- 🔴 No triage/flakiness/health/trend functionality exists for scenario runs. -- 🟡 042 derives a basic health badge; 047 extends it with flakiness + trends. -- 🟡 Reusable analytics/aggregation patterns exist in 040 (consistency detection) and 037 (comparison). +Queue/case, flakiness/health/trend/recurring primitives, API routes and UI routes are present. The +required evidence-led investigation workflow is not production-complete. + +- `[~]` 044 terminal failed/blocked/inconclusive runs now call canonical queue ingestion with an + idempotent immutable run/artifact signal; no case/chat/AgentRun/remediation is started. Staleness, + baseline, load and automation producers plus compatibility-scoped recurring-episode classification + remain outside this bounded signal path. +- `[ ]` InvestigationCase lacks the required immutable evidence snapshot and durable chat/linked- + AgentRun representation. +- `[ ]` `set_disposition()` unconditionally marks a case `resolved`; it does not enforce verification + evidence/reconciliation or accepted-risk rationale required by SCAN-FR-012. +- `[~]` Deterministic aggregation/UI primitives exist but need end-to-end history and signal-ingestion + proof; T013 remains partial. #endregion ScenarioAnalytics.Spec diff --git a/specs/047-dashboard-scenario-analytics/tasks.md b/specs/047-dashboard-scenario-analytics/tasks.md index bbd8b050a..c067538a9 100644 --- a/specs/047-dashboard-scenario-analytics/tasks.md +++ b/specs/047-dashboard-scenario-analytics/tasks.md @@ -5,6 +5,9 @@ ## Format: `- [ ] T### [P] [USx] Description with exact file path` +> **Factual audit 2026-08-20:** `[x]` means code plus relevant evidence; `[~]` means partial +> implementation; `[ ]` means absent integration or unperformed verification. + ## Phase 1 — Setup - [x] T001 Create `InvestigationQueueItem`, `InvestigationCase`, `AgentAction` projection, and migration in `backend/src/models/scenario_investigation.py` @@ -15,16 +18,15 @@ - [x] T003 [US1] Write failing queue/case tests in `backend/tests/services/dashboard_testing/registry/test_scenario_investigation.py` @TEST_EDGE: event queues but does not start agent; duplicate active episode updates count; explicit open is idempotent; disposition CAS->409 -- [x] T004 [US1] Implement queue projection, explicit `open_case`, AgentAction linkage and compact `set_disposition` in `analytics/investigation.py` +- [~] T004 [US1] Implement queue projection, explicit `open_case`, AgentAction linkage and compact `set_disposition` in `analytics/investigation.py` @INVARIANT: case/triage orthogonal; RunResult immutable; never alters graph/baseline -- [x] T005 [US1] Add queue/case/disposition API + object/RBAC tests - Proof: `backend/tests/api/test_scenario_analytics_api.py` + `backend/tests/services/dashboard_testing/test_scenario_analytics_rbac.py` + `backend/tests/fixtures/rbac/scenario_analytics_permissions.json` — 23 passed (pytest, 2026-08-20). +- [~] T005 [US1] Add queue/case/disposition API + object/RBAC tests ## Phase 3 — US2 Flakiness + Health - [x] T006 [US2] Write failing contextual flakiness/health tests in `backend/tests/services/dashboard_testing/registry/test_scenario_analytics.py` @TEST_EDGE: one-way PASS→FAIL -> regression (NOT flaky); post-failure PASS + two transitions -> flaky; infra/cancelled/inconclusive excluded -- [x] T007 [US2] Implement `detect_flakiness` + `derive_health` in `backend/src/services/dashboard_testing/analytics/flakiness.py` +- [~] T007 [US2] Implement `detect_flakiness` + `derive_health` in `backend/src/services/dashboard_testing/analytics/flakiness.py` @POST: deterministic signals; feeds 042 badge on threshold cross - [x] T008 [US2] Add scenario health endpoints in registry and `scenario_analytics.py` @@ -32,19 +34,34 @@ - [x] T009 [US3] Write failing trend/recurring tests in `backend/tests/services/dashboard_testing/registry/test_scenario_trends.py` @TEST_EDGE: classification change does NOT change fingerprint (immutable group identity) -- [x] T010 [US3] Implement deterministic trends/fingerprint summary and recurring episode persistence in analytics services +- [~] T010 [US3] Implement deterministic trends/fingerprint summary and recurring episode persistence in analytics services @POST: compatibility-scoped immutable fingerprint; a matching occurrence after a resolved episode opens a new alertable episode and queue item ## Phase 5 — Frontend + Polish -- [x] T011 [P] Build analytics models/components +- [~] T011 [P] Build analytics models/components InvestigationQueueModel, InvestigationCaseModel, ScenarioAnalyticsModel, HealthCard and queue/case/trends components implemented on disk (`frontend/src/lib/models/*`, `frontend/src/lib/components/scenario-analytics/`); full queue/case/trends UI remains. -- [x] T012 [P] L1/L2 model + UX tests for queue/case/health primitives - Proof: scoped 047 vitest — 26 passed across 9 files (InvestigationModels, ScenarioAnalyticsModel, CaseWorkspace/HealthCard/TrendsChart/RecurringFailuresList, analytics queue/case/scenario route UX tests), 2026-08-20. +- [~] T012 [P] L1/L2 model + UX tests for queue/case/health primitives - [~] T013 Run quickstart, scoped/full backend + frontend tests, ruff; ATTN_1-4; semantic rebuild Partial: scoped frontend vitest (26 passed) + `npm run build` (✔ built) + scoped backend API/RBAC (23 passed) verified. Full backend suite, ruff, quickstart and semantic rebuild not yet run in this session. -- [x] T014 **Prototype validation**: every declared @UX_STATE is reachable via `prototype/index.html`; Playwright verified 1440x900 and 390x844 without overflow or control overlap. - Pending: requires browser proof (chrome-devtools) — no browser session run. +- [ ] T014 **Prototype validation**: every declared @UX_STATE is reachable via `prototype/index.html`. + +## Audit Follow-ups (2026-08-20) + +- [x] T015 Wire canonical 036/run/automation signals to idempotent `queue_signal`; prove qualifying + events create/update queue items without starting agent work. + `ingest_investigation_signal` + `auto_queue_failed_run` + registry staleness emission; 044 + terminal failed/blocked/inconclusive runs additionally emit one exact immutable provenance + signal (passed emits none). This producer does not create a case/AgentRun/action or classify + recurring episodes. +- [~] T016 Persist immutable evidence snapshot, chat/AgentRun linkage and object ACL on InvestigationCase. + Evidence snapshot, linked_run_ids and owner_id ACL persist + (`can_access_case` / GET /cases/{id} 403). Chat/AgentRun workspace remains open. +- [x] T017 Replace unconditional `resolved` disposition with a CAS state machine enforcing verification + evidence/reconciliation for resolved and accepted-risk rationale for accepted; test recurrence reopening. + Recurrence after a resolved episode opens a new episode and queue item + (test_recurrence_after_resolved_episode_opens_new_queue_item). +- [ ] T018 Run independent end-to-end analytics verification and update T013 only with current evidence. ## Dependencies diff --git a/specs/047-dashboard-scenario-analytics/traceability.md b/specs/047-dashboard-scenario-analytics/traceability.md index 3d8ba0514..acba4c6b0 100644 --- a/specs/047-dashboard-scenario-analytics/traceability.md +++ b/specs/047-dashboard-scenario-analytics/traceability.md @@ -1,10 +1,10 @@ # Traceability: Investigation Queue & Scenario Analytics (047) -| Story | Requirement | Model | API operationId | Contract | Task | Test | -|-------|-------------|-------|------------------|----------|------|------| -| US1 Queue/Case | SCAN-FR-001/002/003/009/011 | InvestigationQueueItem, InvestigationCase, TriageRecord | investigations.listQueue/openCase/setDisposition | Analytics.OpenCase, Analytics.Disposition | T003-T005 | test_investigation | -| US2 Flakiness/Health | SCAN-FR-004/005/006 | FlakinessSignal, ScenarioHealth | analytics.health | Analytics.Flakiness, Analytics.Health | T006-T008 | test_flakiness | -| US3 Trends/Recurring | SCAN-FR-007/008/010 | RecurringFailureGroup | analytics.trends, analytics.recurring | Analytics.Trends, Analytics.Recurring | T009-T010 | test_trends | -| Frontend | SCAN-FR-011 | InvestigationQueueModel, InvestigationCaseModel | — | — | T011-T012 | investigation.ux.test | +| Story | Requirement | Model | API operationId | Contract | Task | Test | Actual status / gap | +|-------|-------------|-------|------------------|----------|------|------|---------------------| +| US1 Queue/Case | SCAN-FR-001/002/003/009/011 | InvestigationQueueItem, InvestigationCase, TriageRecord | investigations.listQueue/openCase/setDisposition | Analytics.OpenCase, Analytics.Disposition | T003-T005, T015 | test_investigation, test_scenario_terminal_signals | `[~]` 044 terminal failed/blocked/inconclusive runs now produce one idempotent immutable queue signal with provenance and no automatic case/AgentRun/chat/action. This bounded producer does not classify recurrence; immutable case evidence/chat linkage and SCAN-FR-012 closure remain open. | +| US2 Flakiness/Health | SCAN-FR-004/005/006 | FlakinessSignal, ScenarioHealth | analytics.health | Analytics.Flakiness, Analytics.Health | T006-T008 | test_flakiness | `[~]` primitive exists; real history/signal-ingestion proof is pending. | +| US3 Trends/Recurring | SCAN-FR-007/008/010 | RecurringFailureGroup | analytics.trends, analytics.recurring | Analytics.Trends, Analytics.Recurring | T009-T010 | test_trends | `[~]` aggregate primitives exist; episode/case lifecycle needs E2E proof. | +| Frontend | SCAN-FR-011 | InvestigationQueueModel, InvestigationCaseModel | — | — | T011-T012 | investigation.ux.test | `[~]` UI exists but cannot prove an unconnected queue workflow. | N/A: Registry (042), Editor (043), Execution (044), Monitor (045), Automation (046). diff --git a/specs/WORKSTATE-043-047.md b/specs/WORKSTATE-043-047.md index 2f48693ca..7131f03d7 100644 --- a/specs/WORKSTATE-043-047.md +++ b/specs/WORKSTATE-043-047.md @@ -1,119 +1,89 @@ -# Рабочее состояние: реализация спек 043–047 +# WORKSTATE 043–047 (intermediate 2026-08-20 16:07 +03:00) -> Сохранено автоматически для продолжения работы без потери контекста. -> Дата: 2026-08-20. Область: dashboard scenario specs 042–047 (043/044/045/046/047). +> Статический + targeted-runtime checkpoint. Не является feature-completion. +> Правило: `[x]` только с текущим доказательством; `[~]` частичная реализация; `[ ]` отсутствует. -## 1. Общий статус +## Overall -- **042 Registry** — prerequisite реализован и проверен. -- **043 Editor** — завершён: constrained edits, WorkingDraft, attributed agent proposals, - diff/save/revalidate, scenario:edit RBAC, UI и prototype. -- **044 Execution** — завершён: immutable plan/snapshot/provenance, artifacts, worker lifecycle, - SSE/history/compare/resume, durable PROD approval decision и RUN/RUN_PROD RBAC. -- **045 Run Monitor** — завершён: configuration/PROD gate, live timeline, checkpoint/result, - Global Run Center, waiting-for-me, history/compare и prototype. -- **046 Automation** — завершён: migration + CRUD, scheduler semantics, trigger policy/direct API, - metrics/retention, server-owned PROD classification, management UI и prototype. -- **047 Analytics** — завершён: queue/case/CAS, health/trends/recurring projections, complete - analytics workspace и prototype. +| Spec | Status | Notes | +|---|---|---| +| 042 Registry | `[~]` | CRUD/lifecycle есть. Upstream 037/041 ingestion + queue signal wired. Full quickstart/T024 unverified. | +| 043 Editor | `[~]` | Code/route present. No current E2E/a11y proof. Depends on 042/044. | +| 044 Execution | `[~]` not production-complete | Walker/lifecycle/API работают. Executors fail-safe. Live Playwright/Superset session still missing. | +| 045 Run Monitor | `[~]` | Typed launch contract aligned (release/baseline/toggles). Full route E2E open. | +| 046 Automation | `[~]` | Scheduler reload + trigger dispatch + notifications persist. Live due-job E2E open. | +| 047 Analytics | `[~]` not production-complete | Signal ingest, CAS closure, recurrence, owner ACL. Chat/AgentRun workspace open. | -## 2. Закрывающие доказательства +## Verified facts (this checkpoint) -- Backend scenario scope: **296 passed, 1 skipped**. -- Scoped backend ruff: **clean**. -- Frontend full Vitest: **3927 passed**; production build: **успешен**. -- Alembic: single head `a8b9c0d1e2f3`. -- API wire integration: analytics/automation доступны под `/api/...`; legacy bare paths отсутствуют. -- Playwright: **29 states** на 1440×900 и 390×844 без JS errors, overflow и overlap. -- Semantic rebuild: doc-gen загрузил **9576 contracts / 4730 edges** и создал 11615 nav pages. -- Остаток вне 043–047: общерепозиторный semantic debt (несколько модулей >400 LOC) и - pre-existing frontend compiler warnings. +### 044 Execution +- DAG walker `_advance_run` walks `topological_order`, claims steps, registers artifacts, suspends on human. +- Failed/blocked producers block descendants **including human**; walker does not suspend a terminal failed run. +- API `POST /scenario-runs` creates durable **queued** run (`auto_advance=False`); worker/tests advance separately. +- Launch contract: typed `dashboard_release_id`, `baseline_set`, `execution_toggles` on `StartRunRequest` / `start_run`; frontend no longer hides them in `params.launch_config`. +- Executors (`execution/executors.py`): + - `assertion` → 037 `compare_values` (can fail; never hardcoded PASS). + - `xlsx` → openpyxl parse of real workbook bytes + artifact ref. + - `screenshot` / `report` / `artifact` bind only with real bytes/refs/digest. + - `browser` / `superset_api` without session/query envelope → typed `inconclusive`. +- Cancel: queued steps → `skipped`, run → `cancelled` (via `cancel_requested`/`draining`). +- Timeout: `apply_step_timeout` → step/run `inconclusive` + `STEP_TIMEOUT`. +- Terminal failed/blocked/inconclusive → `auto_queue_failed_run` + persisted notification. -## 3. Реализованные файлы (untracked `??` — мои) +### 047 Analytics +- `ingest_investigation_signal()` idempotent queue projection; no auto agent start. +- Queue/case persist `evidence_snapshot` + `linked_run_ids`. +- Closure CAS: `resolved` needs reconciled verification evidence; `accepted` needs rationale. +- Recurrence after resolved episode opens **new** episode + queue item. +- Object ACL: `InvestigationCase.owner_id`; `can_access_case`; GET `/cases/{id}` 403 for non-owner view. +- Migration `b9c0d1e2f3a4` adds evidence snapshot, linked_run_ids, owner_id. -### Backend — модели -- `backend/src/models/scenario_registry.py` — ScenarioRegistryEntry, ScenarioRevision, - ScenarioWorkingDraft, ScenarioStalenessSignal, ScenarioLifecycleAudit -- `backend/src/models/scenario_run.py` — ScenarioRun, ScenarioStepRun -- `backend/src/models/scenario_checkpoint.py` — HumanCheckpoint -- `backend/src/models/scenario_investigation.py` — InvestigationQueueItem, InvestigationCase, - AgentAction, RecurringFailureEpisode -- `backend/src/models/scenario_automation.py` — ScenarioSchedule, ScenarioTriggerRule, - ScenarioNotificationEvent -- `backend/src/models/scenario_worker.py` — ScenarioStepLease +### 046 Automation +- `load_schedules()` reloads enabled `ScenarioSchedule` and keeps `scenario_*` jobs in desired set. +- `add_scenario_job` maps persisted `misfire_grace_time`, `max_instances`, missed-execution/coalesce. +- `dispatch_trigger_event` resolves `current` revision and calls `start_run`. +- `POST /api/scenario-automation/events/dispatch` is the HTTP boundary. +- `persist_notification` writes lifecycle events (`completed`/`failed`/`blocked`). -### Backend — миграции (alembic/versions, untracked) -- `v1w2x3y4z5a6_add_scenario_registry_tables.py` -- `w2x3y4z5a6_add_scenario_working_drafts.py` -- `x3y4z5a6b7c8_add_scenario_run_tables.py` -- `y4z5a6b7c8d9_add_scenario_investigation_tables.py` +### 042 / 045 +- `ingest_upstream_staleness_event()` for normalized `structure_diff` / `lineage_blast_radius`. +- Staleness apply emits investigation queue signal. +- RunMonitorModel posts typed 044 start fields (Vitest: launch body contains `dashboard_release_id`, not `launch_config`). -### Backend — services -- `backend/src/services/dashboard_testing/registry/` — list, get, serializers, create, - revisions, staleness, lifecycle, clone, health -- `backend/src/services/dashboard_testing/editor/` — ops, load, apply, save, revalidate, __init__ -- `backend/src/services/dashboard_testing/execution/` — runner_plan, runner, dispatch, - executor_registry, result, lifecycle, worker, __init__ -- `backend/src/services/dashboard_testing/automation/` — policy, schedule, trigger, notify, retention -- `backend/src/services/dashboard_testing/analytics/` — flakiness, investigation, trends, recurring +## Targeted evidence (do not treat as suite-close) -### Backend — API -- `backend/src/api/routes/dashboard_testing/scenarios.py` -- `backend/src/api/routes/dashboard_testing/scenario_runs.py` -- `backend/src/api/routes/dashboard_testing/scenario_analytics.py` -- `backend/src/api/routes/dashboard_testing/scenario_automation.py` -- `backend/src/api/routes/dashboard_testing/__init__.py` (изменён — include новых routers) -- `backend/src/api/routes/agent_runs.py` (изменён — Save→Registry bridge) +Recorded in this session with `AUTH_SECRET_KEY` / `SECRET_KEY` set: -### Backend — schemas -- `backend/src/schemas/dashboard_testing/scenario_registry.py` -- `backend/src/schemas/dashboard_testing/__init__.py` (изменён) +- Investigation + staleness + executors: **12 passed** (later expanded). +- Automation services/API: **32 passed**; trigger dispatch pinning: **9 passed**. +- Runner/walker/dispatch after fail-safe executors: **11 passed**. +- Run API after queued-start restore: **30 passed**. +- RunMonitorModel Vitest: **2 passed**. +- Core scheduler regression: **17 passed**. +- Consolidated 042/044/046/047 slice: **81 passed**. +- Recurrence + walker fail-path + executors: **47 passed**, ruff clean on touched modules. +- Lifecycle cancel/timeout + investigation ACL: **20 passed**. +- Analytics API after resolved-disposition contract: **21 passed**. -### Backend — tests -- `backend/tests/services/dashboard_testing/registry/test_scenario_*.py` (много файлов) -- `backend/tests/api/test_scenario_registry.py` -- `backend/tests/api/test_dashboard_testing_openapi.py` (изменён) -- `backend/tests/api/test_agent_runs_coverage.py` (изменён) -- `backend/tests/fixtures/scenario_registry/`, `backend/tests/fixtures/scenario_editor/` -- `backend/tests/models/test_scenario_registry.py`, `backend/tests/test_scenario_registry_migration.py` +Full backend/frontend suites, Playwright, semantic rebuild: **not rerun**. -### Frontend -- `frontend/src/types/scenario-registry.ts`, `frontend/src/types/scenario-run.ts` -- `frontend/src/lib/models/ScenarioRegistryModel.svelte.ts`, `ScenarioEditorModel.svelte.ts`, - `RunMonitorModel.svelte.ts`, `InvestigationQueueModel.svelte.ts`, `InvestigationCaseModel.svelte.ts` -- `frontend/src/lib/components/scenario-editor/` (StepCard, ConstrainedAssertionEditor, VisualDagCanvas) -- `frontend/src/lib/components/scenario-run/` (RunTimeline, HumanCheckpointPanel, ScenarioResultView, - RunHistoryList, RunComparison) -- `frontend/src/lib/components/scenario-analytics/` (HealthCard) -- `frontend/src/routes/dashboard-testing/` (editor route) -- `frontend/src/lib/models/__tests__/` и component `__tests__/` +## Remaining (priority) -### Specs (tasks.md 042–047 отмечены реализованные задачи) -- `specs/037-superset-baseline-engine/contracts/dashboard-testing.openapi.yaml` (расширен) +1. **044 live adapters** — Playwright session replay and catalog-backed Superset query when a real session/query envelope exists. Current `inconclusive` is fail-safe, not SCEX-FR-002/009 complete. +2. **044 T025 remainder** — approval-to-live-dispatch, infra resume continuation of remaining DAG, timeout during in-flight executor I/O. +3. **047 T016 remainder** — durable chat / linked AgentRun workspace on InvestigationCase. +4. **047 T018 / 044 T022** — independent full scoped verification; do not reuse old aggregate counts. +5. **045/043 UI** — mount remaining production panels; browser/a11y E2E. +6. **039 T058** — wire `_trigger_release_verification` into preprod deploy (out of 042–047 core, still listed). -## 4. Последние проверки (фактические прогоны) +## Key files (this checkpoint) -- Backend consolidated scenario suite: **296 passed, 1 skipped**. -- Backend scoped ruff: **clean**; `git diff --check`: clean. -- Frontend full suite: **3927 passed**; `npm run build`: **успешен**. -- Prototype Playwright: **all 29 states passed** на desktop/mobile. -- Semantic doc-gen navigation rebuild: **9576 contracts, 4730 edges, 11615 pages**. - -## 5. Замечания по окружению - -- Полный `import src.app` требует `DATABASE_URL` (env-only blocker, не code failure). -- В репозитории много pre-existing `M`-изменений (похоже на CRLF/line-ending) — НЕ мои, не трогать. -- Полный backend ruff ранее: 7904 repository-wide pre-existing findings; мой код их не добавляет. - -## 6. Следующие шаги - -Спеки 043–047 закрыты. Дальнейшие действия относятся к общерепозиторному refactoring debt, -review/commit/release workflow и не входят в их функциональный scope. - -## 7. Анти-повтор - -- НЕ возвращаться к полному repository-wide ruff (7904 findings) — вне scope 043–047. -- НЕ трогать pre-existing `M`-изменения (line-ending/CRLF) в чужих файлах. -- НЕ помечать невыполненные интеграционные контракты (SSE, APScheduler persistence, - полный UI) как completed без фактической реализации и тестов. -- После каждого нового фрагмента: scoped pytest + ruff, затем — если менялся frontend — vitest + build. +- `backend/src/services/dashboard_testing/execution/{runner,executors,lifecycle,dispatch}.py` +- `backend/src/services/dashboard_testing/analytics/{investigation,recurring}.py` +- `backend/src/services/dashboard_testing/automation/{trigger,notify}.py` +- `backend/src/services/dashboard_testing/registry/staleness.py` +- `backend/src/core/scheduler.py` +- `backend/src/api/routes/dashboard_testing/{scenario_runs,scenario_analytics,scenario_automation}.py` +- `backend/src/models/scenario_investigation.py` +- `backend/alembic/versions/b9c0d1e2f3a4_add_investigation_evidence_snapshot.py` +- `frontend/src/lib/models/RunMonitorModel.svelte.ts` diff --git a/specs/prototype-ui.css b/specs/prototype-ui.css index 0d42a2b30..6c927b8f6 100644 --- a/specs/prototype-ui.css +++ b/specs/prototype-ui.css @@ -64,6 +64,9 @@ body { font: inherit; text-decoration: none; cursor: pointer; + min-height: 44px; + min-inline-size: 44px; + transition: background-color 160ms ease, color 160ms ease, border-color 160ms ease; } .nav a:hover, .nav a.active { @@ -338,6 +341,16 @@ input[type="search"] { white-space: nowrap; border: 0; } +@media (prefers-reduced-motion: reduce) { + *, + *::before, + *::after { + animation-duration: 0.01ms !important; + animation-iteration-count: 1 !important; + scroll-behavior: auto !important; + transition-duration: 0.01ms !important; + } +} @media (max-width: 850px) { .shell { padding: 16px;