busya
def2bf4038
- poisoned_store.py: durable JSONL identical-failure counter (fcntl.flock + intra-process lock
+ thread-local reentrancy, append+fsync, torn-line tolerant, CAS quarantine epoch); no
Alembic/ORM changes (046 migrations frozen).
- poisoned.py: N=3 identical infra failures -> typed POISONED_RUN_QUARANTINE, matching
schedules disabled, exactly one blocked notification carrying the DLQ payload; success
resets the pair; operator unquarantine (CAS) re-enables schedules and resets counters.
- REST: GET /scenario-automation/quarantine and POST /quarantine/{scenario_id}/release
(scenario:automation MANAGE; 404 NOT_QUARANTINED / 409 STALE_QUARANTINE_VERSION).
- Tests: 13 policy + 2 API quarantine vectors; merged with the master ReadAcl/RetentionReceipts
classes (conflict resolved keeping both). Focused 31 passed; 046 T023 offline slice.
869 lines
43 KiB
Python
869 lines
43 KiB
Python
# #region Test.Api.ScenarioAutomation [C:3] [TYPE Module] [SEMANTICS test,api,scenario,automation,crud,rbac,trigger]
|
|
# @BRIEF Verify scenario-automation API surface: schedule/trigger-rule/policy CRUD, metrics,
|
|
# notifications, RBAC scopes and the direct PROD-gated scenario trigger.
|
|
# @RELATION BINDS_TO -> [Api.ScenarioAutomation.Routes]
|
|
# @RELATION VERIFIES -> [Api.ScenarioAutomation.DirectTrigger]
|
|
# @RELATION BINDS_TO -> [ScenarioExecution.Runner.QueuedDispatch]
|
|
# @RELATION VERIFIES -> [ScenarioExecution.EnvironmentPolicy.Resolve]
|
|
# @RELATION VERIFIES -> [ScenarioExecution.Runner.TriggerSource.RejectAutomatedHuman]
|
|
# @TEST_EDGE missing_manage_scope -> 403 on schedule mutation
|
|
# @TEST_EDGE missing_trigger_scope -> 403 on direct trigger
|
|
# @TEST_EDGE missing_prod_scope -> 403 on PROD direct trigger
|
|
# @TEST_EDGE missing_read_scope -> 403 on all six operational reads (DG-2, T019)
|
|
# @TEST_EDGE foreign_owner_rows -> filtered out of every read collection, no totals leak
|
|
# @TEST_EDGE disabled_human_schedule -> identical 409 AUTOMATION_INELIGIBLE_HUMAN_STEP as enabled (DEF-02)
|
|
# @TEST_EDGE idempotency_reuse -> 409 on changed request hash
|
|
# @TEST_INVARIANT ScenarioExecution.Runner.Start: The 046 API trigger supplies a server-owned
|
|
# automation source; a persisted human graph returns the typed manual-only
|
|
# rejection before it creates a run, gate, or notification.
|
|
# @TEST_INVARIANT Api.ScenarioAutomation.DirectTrigger: A non-PROD HTTP trigger/replay persists
|
|
# only its queued row; its external boundary does not advance until the separate
|
|
# 044 dispatcher claims it through status CAS. -> VERIFIED_BY:
|
|
# test_direct_trigger_starts_run_with_idempotency
|
|
# @TEST_INVARIANT ScenarioExecution.EnvironmentPolicy: A direct target's server-owned PROD class
|
|
# requires automation PROD and scenario RUN_PROD before start; client payload has
|
|
# no override. -> VERIFIED_BY: test_prod_trigger_requires_prod_scope
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Iterator
|
|
from contextlib import contextmanager
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock
|
|
|
|
from fastapi.testclient import TestClient
|
|
|
|
from src.app import app
|
|
from src.core.database import SessionLocal
|
|
from src.dependencies import get_config_manager, get_current_user
|
|
from src.models.auth import Permission, Role, User
|
|
from src.models.scenario_automation import (
|
|
AutomationPolicy,
|
|
ScenarioNotificationEvent,
|
|
ScenarioRetentionDeletion,
|
|
ScenarioSchedule,
|
|
ScenarioTriggerRule,
|
|
)
|
|
from src.models.scenario_registry import ScenarioRegistryEntry, ScenarioRevision
|
|
from src.models.scenario_run import ScenarioRun
|
|
from src.services.dashboard_testing.automation.poisoned_store import PoisonedRunStore
|
|
from src.services.dashboard_testing.scenario.templates import (
|
|
ACTION_REGISTRY_VERSION,
|
|
action_registry_fingerprint,
|
|
resolve_action_descriptor,
|
|
)
|
|
|
|
|
|
def _make_user_with_permissions(permissions: list[tuple[str, str]]) -> User:
|
|
role = Role(id="automation-role-test", name="AutomationRole")
|
|
role.permissions = [
|
|
Permission(resource=resource, action=action)
|
|
for resource, action in permissions
|
|
]
|
|
user = User(id="automation-user-test", username="automation.tester", email="auto@test.com")
|
|
user.roles = [role]
|
|
return user
|
|
|
|
|
|
def _action_step(step_id: str, tool: str, action: str, **extra) -> dict:
|
|
"""Hardcoded exact 038 action fixture; no tool-only dispatch is valid."""
|
|
return {
|
|
"id": step_id,
|
|
"logical_step_id": step_id,
|
|
"tool": tool,
|
|
"action": action,
|
|
"action_descriptor": resolve_action_descriptor(
|
|
tool=tool,
|
|
action=action,
|
|
registry_version=ACTION_REGISTRY_VERSION,
|
|
registry_hash=action_registry_fingerprint(),
|
|
).snapshot(),
|
|
**extra,
|
|
}
|
|
|
|
|
|
# #region Test.Api.ScenarioAutomation.ConfigManager [C:2] [TYPE Block] [SEMANTICS test,api,scenario,automation,env,fixture]
|
|
# @BRIEF Server-owned environment classification stub: PROD classification must come from
|
|
# ConfigManager (stage/is_production), never from the environment_id string.
|
|
# @TEST_FIXTURE env-preprod-01/env-preprod-02 -> PREPROD (non-prod); prod-01 -> PROD stage;
|
|
# prod-mirror -> PREPROD stage but is_production=True (proves the boolean wins).
|
|
def _make_config_manager():
|
|
envs = {
|
|
"env-preprod-01": SimpleNamespace(stage="PREPROD", is_production=False),
|
|
"env-preprod-02": SimpleNamespace(stage="PREPROD", is_production=False),
|
|
"prod-01": SimpleNamespace(stage="PROD", is_production=True),
|
|
"prod-mirror": SimpleNamespace(stage="PREPROD", is_production=True),
|
|
}
|
|
cm = MagicMock()
|
|
cm.get_environment.side_effect = lambda environment_id: envs.get(str(environment_id))
|
|
return cm
|
|
# #endregion Test.Api.ScenarioAutomation.ConfigManager
|
|
|
|
|
|
@contextmanager
|
|
def _client_for(user: User) -> Iterator[TestClient]:
|
|
"""TestClient with get_current_user + get_config_manager overridden for the whole context."""
|
|
app.dependency_overrides[get_current_user] = lambda: user
|
|
app.dependency_overrides[get_config_manager] = lambda: _make_config_manager()
|
|
try:
|
|
yield TestClient(app)
|
|
finally:
|
|
app.dependency_overrides.pop(get_current_user, None)
|
|
app.dependency_overrides.pop(get_config_manager, None)
|
|
|
|
|
|
# #region Test.Api.ScenarioAutomation.Crud [C:2] [TYPE Class] [SEMANTICS test,api,scenario,automation,crud]
|
|
# @BRIEF Persisted CRUD for schedules, trigger rules and policies; metrics and notifications list.
|
|
class TestScenarioAutomationCrud:
|
|
def test_schedule_crud_roundtrip(self, dashboard_testing_client):
|
|
client = dashboard_testing_client
|
|
created = client.post(
|
|
"/api/scenario-automation/schedules",
|
|
json={
|
|
"scenario_id": "11111111-1111-4111-8111-111111111111",
|
|
"environment_id": "env-preprod-01",
|
|
"cron_expr": "0 7 * * 1-5",
|
|
"timezone": "Europe/Moscow",
|
|
"missed_execution_policy": "run_latest",
|
|
},
|
|
)
|
|
assert created.status_code == 201, created.text
|
|
schedule_id = created.json()["id"]
|
|
updated = client.patch(
|
|
f"/api/scenario-automation/schedules/{schedule_id}",
|
|
json={
|
|
"scenario_id": "11111111-1111-4111-8111-111111111111",
|
|
"environment_id": "env-preprod-01",
|
|
"cron_expr": "30 6 * * *",
|
|
"enabled": False,
|
|
},
|
|
)
|
|
assert updated.status_code == 200
|
|
assert updated.json()["enabled"] is False
|
|
listed = client.get("/api/scenario-automation/schedules")
|
|
assert listed.status_code == 200
|
|
assert any(item["id"] == schedule_id for item in listed.json())
|
|
deleted = client.delete(f"/api/scenario-automation/schedules/{schedule_id}")
|
|
assert deleted.status_code == 204
|
|
|
|
def test_trigger_rule_crud_roundtrip(self, dashboard_testing_client):
|
|
client = dashboard_testing_client
|
|
created = client.post(
|
|
"/api/scenario-automation/trigger-rules",
|
|
json={
|
|
"scenario_id": "11111111-1111-4111-8111-111111111111",
|
|
"environment_id": "env-preprod-01",
|
|
"trigger": "etl_completed",
|
|
},
|
|
)
|
|
assert created.status_code == 201, created.text
|
|
rule_id = created.json()["id"]
|
|
updated = client.patch(
|
|
f"/api/scenario-automation/trigger-rules/{rule_id}",
|
|
json={
|
|
"scenario_id": "11111111-1111-4111-8111-111111111111",
|
|
"environment_id": "env-preprod-01",
|
|
"trigger": "release_created",
|
|
"enabled": False,
|
|
},
|
|
)
|
|
assert updated.status_code == 200
|
|
assert updated.json()["trigger"] == "release_created"
|
|
deleted = client.delete(f"/api/scenario-automation/trigger-rules/{rule_id}")
|
|
assert deleted.status_code == 204
|
|
|
|
def test_trigger_rule_rejects_unknown_trigger(self, dashboard_testing_client):
|
|
resp = dashboard_testing_client.post(
|
|
"/api/scenario-automation/trigger-rules",
|
|
json={"scenario_id": "s", "environment_id": "e", "trigger": "moon_phase"},
|
|
)
|
|
assert resp.status_code == 422
|
|
|
|
def test_policy_crud_roundtrip(self, dashboard_testing_client):
|
|
client = dashboard_testing_client
|
|
created = client.post(
|
|
"/api/scenario-automation/policies",
|
|
json={
|
|
"name": "test-policy",
|
|
"workload_class": "scenario_smoke",
|
|
"max_concurrent_per_env": 2,
|
|
"dedup_window_seconds": 600,
|
|
"overlap_rule": "warn",
|
|
"retention_days": 30,
|
|
"prod_gate_required": True,
|
|
"on_repeated_failure": "disable",
|
|
},
|
|
)
|
|
assert created.status_code == 201, created.text
|
|
policy_id = created.json()["id"]
|
|
listed = client.get("/api/scenario-automation/policies")
|
|
assert listed.status_code == 200
|
|
assert any(item["id"] == policy_id for item in listed.json())
|
|
deleted = client.delete(f"/api/scenario-automation/policies/{policy_id}")
|
|
assert deleted.status_code == 204
|
|
|
|
def test_metrics_and_notifications_readable(self, dashboard_testing_client):
|
|
metrics = dashboard_testing_client.get("/api/scenario-automation/metrics")
|
|
assert metrics.status_code == 200
|
|
body = metrics.json()
|
|
assert "schedules_total" in body
|
|
assert "success_rate" in body
|
|
assert "trigger_distribution" in body
|
|
notifications = dashboard_testing_client.get("/api/scenario-automation/notifications")
|
|
assert notifications.status_code == 200
|
|
retention = dashboard_testing_client.get("/api/scenario-automation/retention")
|
|
assert retention.status_code == 200
|
|
assert retention.json()["tiers"]["run_metadata"] == 180
|
|
assert retention.json()["tiers"]["raw_vlm"] == 7
|
|
# #endregion Test.Api.ScenarioAutomation.Crud
|
|
|
|
|
|
# #region Test.Api.ScenarioAutomation.Rbac [C:2] [TYPE Class] [SEMANTICS test,api,scenario,automation,rbac,prod]
|
|
# @BRIEF RBAC automation scopes: MANAGE/TRIGGER/PROD gates and the direct API trigger.
|
|
class TestScenarioAutomationRbac:
|
|
def test_schedule_mutation_requires_manage_scope(self):
|
|
user = _make_user_with_permissions([("scenario:automation", "TRIGGER")])
|
|
with _client_for(user) as client:
|
|
resp = client.post(
|
|
"/api/scenario-automation/schedules",
|
|
json={"scenario_id": "s", "environment_id": "e", "cron_expr": "0 7 * * *"},
|
|
)
|
|
assert resp.status_code == 403
|
|
|
|
def test_direct_trigger_requires_trigger_scope(self):
|
|
user = _make_user_with_permissions([("scenario:automation", "MANAGE")])
|
|
with _client_for(user) as client:
|
|
resp = client.post(
|
|
"/api/scenario-automation/scenarios/sc-1/trigger",
|
|
headers={"Idempotency-Key": "key-1"},
|
|
json={"environment_id": "env-preprod-01", "revision_id": "rev-1"},
|
|
)
|
|
assert resp.status_code == 403
|
|
|
|
def test_prod_trigger_requires_prod_scope(self):
|
|
user = _make_user_with_permissions([("scenario:automation", "TRIGGER")])
|
|
with _client_for(user) as client:
|
|
resp = client.post(
|
|
"/api/scenario-automation/scenarios/sc-1/trigger",
|
|
headers={"Idempotency-Key": "key-2"},
|
|
json={"environment_id": "prod-01", "revision_id": "rev-1"},
|
|
)
|
|
assert resp.status_code == 403
|
|
|
|
def test_direct_trigger_starts_run_with_idempotency(self):
|
|
|
|
from src.models.scenario_registry import ScenarioRevision
|
|
from src.models.scenario_run import ScenarioRun
|
|
|
|
setup = SessionLocal()
|
|
try:
|
|
setup.query(ScenarioRun).filter(ScenarioRun.idempotency_key == "auto-key-001").delete()
|
|
setup.query(ScenarioRegistryEntry).filter(
|
|
ScenarioRegistryEntry.scenario_id == "44444444-4444-4444-8444-444444444444"
|
|
).delete()
|
|
setup.commit()
|
|
entry = ScenarioRegistryEntry(
|
|
scenario_id="44444444-4444-4444-8444-444444444444",
|
|
scenario_key="auto-trigger-fixture",
|
|
name="Auto trigger fixture",
|
|
dashboard_id=42,
|
|
environment_ids=["env-preprod-01"],
|
|
owner_id="user-qa-1",
|
|
owner_username="qa.analyst",
|
|
lifecycle_status="READY",
|
|
validation_status="valid",
|
|
current_revision_id="55555555-5555-4555-8555-555555555555",
|
|
)
|
|
setup.add(entry)
|
|
setup.add(
|
|
ScenarioRevision(
|
|
revision_id="55555555-5555-4555-8555-555555555555",
|
|
scenario_id="44444444-4444-4444-8444-444444444444",
|
|
content_hash="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
|
graph_snapshot={
|
|
"action_registry_version": ACTION_REGISTRY_VERSION,
|
|
"action_registry_hash": action_registry_fingerprint(),
|
|
"steps": [_action_step("step-1", "assertion", "structural_assert")],
|
|
"dependencies": [],
|
|
"environment_ids": ["env-preprod-01"],
|
|
},
|
|
created_by="qa.analyst",
|
|
activation_status="current",
|
|
)
|
|
)
|
|
setup.commit()
|
|
finally:
|
|
setup.close()
|
|
|
|
user = _make_user_with_permissions([("scenario:automation", "TRIGGER")])
|
|
with _client_for(user) as client:
|
|
resp = client.post(
|
|
"/api/scenario-automation/scenarios/44444444-4444-4444-8444-444444444444/trigger",
|
|
headers={"Idempotency-Key": "auto-key-001"},
|
|
json={"environment_id": "env-preprod-01", "revision_id": "55555555-5555-4555-8555-555555555555"},
|
|
)
|
|
assert resp.status_code == 202, resp.text
|
|
body = resp.json()
|
|
# HTTP trigger persists only; no executor runs before the server dispatcher claim.
|
|
assert body["status"] == "queued"
|
|
assert body["run_id"]
|
|
# Same idempotency key returns the same run; a changed request is rejected.
|
|
same = client.post(
|
|
"/api/scenario-automation/scenarios/44444444-4444-4444-8444-444444444444/trigger",
|
|
headers={"Idempotency-Key": "auto-key-001"},
|
|
json={"environment_id": "env-preprod-01", "revision_id": "55555555-5555-4555-8555-555555555555"},
|
|
)
|
|
assert same.status_code == 202
|
|
assert same.json()["run_id"] == body["run_id"]
|
|
changed = client.post(
|
|
"/api/scenario-automation/scenarios/44444444-4444-4444-8444-444444444444/trigger",
|
|
headers={"Idempotency-Key": "auto-key-001"},
|
|
json={"environment_id": "env-preprod-02", "revision_id": "55555555-5555-4555-8555-555555555555"},
|
|
)
|
|
assert changed.status_code == 409
|
|
assert changed.json()["detail"]["code"] == "IDEMPOTENCY_KEY_REUSED"
|
|
|
|
from src.services.dashboard_testing.execution.runner import dispatch_queued_runs
|
|
|
|
dispatcher = SessionLocal()
|
|
try:
|
|
outcomes = dispatch_queued_runs(dispatcher, worker_id="automation-api-dispatch")
|
|
dispatcher.commit()
|
|
finally:
|
|
dispatcher.close()
|
|
assert [outcome["status"] for outcome in outcomes] == ["inconclusive"]
|
|
|
|
# #region Test.Api.ScenarioAutomation.Rbac.ManualOnly [C:3] [TYPE Function]
|
|
# @BRIEF The external automation endpoint refuses a persisted human graph before any run-side effect.
|
|
# @TEST_INVARIANT ScenarioExecution.Runner.Start: An API automation request for a human graph
|
|
# cannot materialize a manual ScenarioRun or skip its HumanCheckpoint.
|
|
def test_direct_trigger_rejects_human_revision_before_run_creation(self):
|
|
from src.models.scenario_approval import ActionApprovalGate
|
|
from src.models.scenario_automation import ScenarioNotificationEvent
|
|
from src.models.scenario_registry import ScenarioRevision
|
|
from src.models.scenario_run import ScenarioRun
|
|
|
|
scenario_id = "60460000-0000-4000-8000-000000000004"
|
|
revision_id = "60460000-0000-4000-8000-000000000014"
|
|
setup = SessionLocal()
|
|
try:
|
|
setup.query(ScenarioRun).filter(ScenarioRun.scenario_id == scenario_id).delete()
|
|
setup.query(ScenarioRegistryEntry).filter(ScenarioRegistryEntry.scenario_id == scenario_id).delete()
|
|
setup.commit()
|
|
setup.add(ScenarioRegistryEntry(
|
|
scenario_id=scenario_id,
|
|
scenario_key="api-manual-only-044",
|
|
name="API manual-only fixture",
|
|
dashboard_id=46,
|
|
environment_ids=["env-preprod-01"],
|
|
owner_id="analyst-046",
|
|
owner_username="analyst.046",
|
|
lifecycle_status="READY",
|
|
validation_status="valid",
|
|
current_revision_id=revision_id,
|
|
))
|
|
setup.add(ScenarioRevision(
|
|
revision_id=revision_id,
|
|
scenario_id=scenario_id,
|
|
content_hash="6" * 64,
|
|
graph_snapshot={
|
|
"action_registry_version": ACTION_REGISTRY_VERSION,
|
|
"action_registry_hash": action_registry_fingerprint(),
|
|
"steps": [_action_step("human-api-044", "human", "human_checkpoint")],
|
|
"dependencies": [],
|
|
},
|
|
created_by="analyst-046",
|
|
activation_status="current",
|
|
))
|
|
setup.commit()
|
|
before = (
|
|
setup.query(ActionApprovalGate).count(),
|
|
setup.query(ScenarioNotificationEvent).filter(ScenarioNotificationEvent.scenario_id == scenario_id).count(),
|
|
)
|
|
finally:
|
|
setup.close()
|
|
|
|
user = _make_user_with_permissions([("scenario:automation", "TRIGGER")])
|
|
with _client_for(user) as client:
|
|
response = client.post(
|
|
f"/api/scenario-automation/scenarios/{scenario_id}/trigger",
|
|
headers={"Idempotency-Key": "api-human-manual-only-044"},
|
|
json={"environment_id": "env-preprod-01", "revision_id": revision_id},
|
|
)
|
|
|
|
assert response.status_code == 409, response.text
|
|
assert response.json()["detail"]["code"] == "AUTOMATION_INELIGIBLE_HUMAN_STEP"
|
|
verify = SessionLocal()
|
|
try:
|
|
assert verify.query(ScenarioRun).filter(ScenarioRun.scenario_id == scenario_id).count() == 0
|
|
assert (
|
|
verify.query(ActionApprovalGate).count(),
|
|
verify.query(ScenarioNotificationEvent).filter(ScenarioNotificationEvent.scenario_id == scenario_id).count(),
|
|
) == before
|
|
finally:
|
|
verify.close()
|
|
# #endregion Test.Api.ScenarioAutomation.Rbac.ManualOnly
|
|
# #endregion Test.Api.ScenarioAutomation.Rbac
|
|
|
|
|
|
# #region Test.Api.ScenarioAutomation.Sec01 [C:2] [TYPE Class] [SEMANTICS test,api,scenario,automation,rbac,anonymous]
|
|
# @BRIEF SEC-01: operational reads require an authenticated principal; anonymous bearers are 401.
|
|
# @TEST_INVARIANT Api.ScenarioAutomation: every operational read carries _USER; API-key/service
|
|
# principals cannot bypass the OAuth2 bearer dependency.
|
|
class TestScenarioAutomationAnonymousReads:
|
|
_READ_PATHS = (
|
|
"/api/scenario-automation/schedules",
|
|
"/api/scenario-automation/trigger-rules",
|
|
"/api/scenario-automation/policies",
|
|
"/api/scenario-automation/notifications",
|
|
"/api/scenario-automation/metrics",
|
|
"/api/scenario-automation/retention",
|
|
"/api/scenario-automation/quarantine",
|
|
)
|
|
|
|
def test_anonymous_reads_require_authenticated_principal(self, dashboard_testing_client):
|
|
# Remove the auth bypass so the real OAuth2PasswordBearer dependency rejects an
|
|
# anonymous/API-key principal (no Authorization Bearer header) on every operational read.
|
|
app.dependency_overrides.pop(get_current_user, None)
|
|
anonymous = TestClient(app)
|
|
for path in self._READ_PATHS:
|
|
resp = anonymous.get(path)
|
|
assert resp.status_code == 401, (path, resp.text)
|
|
# #endregion Test.Api.ScenarioAutomation.Sec01
|
|
|
|
|
|
# #region Test.Api.ScenarioAutomation.ReadAcl [C:4] [TYPE Class] [SEMANTICS test,api,scenario,automation,rbac,acl]
|
|
# @BRIEF DG-2 (046 T019): six reads require scenario:automation READ plus per-object
|
|
# scenario-ownership ACL; foreign rows vanish from collections without leaking totals.
|
|
# @TEST_INVARIANT Api.ScenarioAutomation.ReadAcl: missing READ -> 403 on every read;
|
|
# owner sees own rows only; admin sees all rows.
|
|
# @TEST_INVARIANT Api.ScenarioAutomation.Schedules: disabled schedule bound to a human-step
|
|
# revision is rejected with the identical code as enabled, with zero side-effect
|
|
# rows (DEF-02) -> VERIFIED_BY: test_disabled_human_schedule_rejected_like_enabled
|
|
def _make_acl_user() -> User:
|
|
role = Role(id="acl-role-test", name="AclRole")
|
|
role.permissions = [Permission(resource="scenario:automation", action="READ")]
|
|
user = User(id="acl-viewer-1", username="acl.viewer", email="acl@test.com")
|
|
user.roles = [role]
|
|
return user
|
|
|
|
|
|
def _make_admin_user() -> User:
|
|
role = Role(id="acl-admin-role-test", name="AclAdmin", is_admin=True)
|
|
user = User(id="acl-admin-1", username="acl.admin", email="acl.admin@test.com")
|
|
user.roles = [role]
|
|
return user
|
|
|
|
|
|
_OWN_SCENARIO = "70460000-0000-4000-8000-0000000000a1"
|
|
_FOREIGN_SCENARIO = "70460000-0000-4000-8000-0000000000b2"
|
|
_HUMAN_SCENARIO = "70460000-0000-4000-8000-0000000000c3"
|
|
_HUMAN_REVISION = "70460000-0000-4000-8000-0000000000c4"
|
|
_ELIGIBLE_SCENARIO = "70460000-0000-4000-8000-0000000000d5"
|
|
_ELIGIBLE_REVISION = "70460000-0000-4000-8000-0000000000d6"
|
|
|
|
|
|
def _seed_acl_rows(db) -> dict[str, str]:
|
|
"""Hardcoded ACL fixture: one own + one foreign row per read collection."""
|
|
db.add(ScenarioRegistryEntry(
|
|
scenario_id=_OWN_SCENARIO, scenario_key="acl-own", name="ACL own", dashboard_id=71,
|
|
environment_ids=["env-preprod-01"], owner_id="acl-viewer-1", owner_username="acl.viewer",
|
|
lifecycle_status="READY", validation_status="valid",
|
|
))
|
|
db.add(ScenarioRegistryEntry(
|
|
scenario_id=_FOREIGN_SCENARIO, scenario_key="acl-foreign", name="ACL foreign", dashboard_id=72,
|
|
environment_ids=["env-preprod-01"], owner_id="foreign-owner-1", owner_username="foreign.owner",
|
|
lifecycle_status="READY", validation_status="valid",
|
|
))
|
|
own_policy = AutomationPolicy(name="acl-policy-own")
|
|
foreign_policy = AutomationPolicy(name="acl-policy-foreign")
|
|
free_policy = AutomationPolicy(name="acl-policy-free")
|
|
db.add_all([own_policy, foreign_policy, free_policy])
|
|
db.flush()
|
|
own_schedule = ScenarioSchedule(
|
|
scenario_id=_OWN_SCENARIO, environment_id="env-preprod-01", cron_expr="0 7 * * *",
|
|
policy_id=own_policy.id,
|
|
)
|
|
foreign_schedule = ScenarioSchedule(
|
|
scenario_id=_FOREIGN_SCENARIO, environment_id="env-preprod-01", cron_expr="0 8 * * *",
|
|
policy_id=foreign_policy.id,
|
|
)
|
|
own_rule = ScenarioTriggerRule(scenario_id=_OWN_SCENARIO, environment_id="env-preprod-01", trigger="api")
|
|
foreign_rule = ScenarioTriggerRule(scenario_id=_FOREIGN_SCENARIO, environment_id="env-preprod-01", trigger="api")
|
|
db.add_all([own_schedule, foreign_schedule, own_rule, foreign_rule])
|
|
own_notification = ScenarioNotificationEvent(
|
|
event_type="run_finished", scenario_id=_OWN_SCENARIO, severity="info", payload={"k": "own"},
|
|
)
|
|
foreign_notification = ScenarioNotificationEvent(
|
|
event_type="run_finished", scenario_id=_FOREIGN_SCENARIO, severity="info", payload={"k": "foreign"},
|
|
)
|
|
db.add_all([own_notification, foreign_notification])
|
|
db.add(ScenarioRun(
|
|
scenario_id=_OWN_SCENARIO, scenario_revision_id="70460000-0000-4000-8000-0000000000e1",
|
|
scenario_content_hash="a" * 64, environment_id="env-preprod-01",
|
|
idempotency_key="acl-own-run-001", trigger_source="scheduled",
|
|
))
|
|
db.add(ScenarioRun(
|
|
scenario_id=_FOREIGN_SCENARIO, scenario_revision_id="70460000-0000-4000-8000-0000000000e2",
|
|
scenario_content_hash="f" * 64, environment_id="env-preprod-01",
|
|
idempotency_key="acl-foreign-run-001", trigger_source="scheduled",
|
|
))
|
|
db.commit()
|
|
return {
|
|
"own_policy": own_policy.id, "foreign_policy": foreign_policy.id, "free_policy": free_policy.id,
|
|
"own_schedule": own_schedule.id, "foreign_schedule": foreign_schedule.id,
|
|
"own_rule": own_rule.id, "foreign_rule": foreign_rule.id,
|
|
"own_notification": own_notification.id, "foreign_notification": foreign_notification.id,
|
|
}
|
|
|
|
|
|
def _drop_acl_rows(db) -> None:
|
|
db.query(ScenarioRun).filter(ScenarioRun.idempotency_key.in_(["acl-own-run-001", "acl-foreign-run-001"])).delete()
|
|
db.query(ScenarioNotificationEvent).filter(ScenarioNotificationEvent.scenario_id.in_([_OWN_SCENARIO, _FOREIGN_SCENARIO])).delete()
|
|
db.query(ScenarioSchedule).filter(ScenarioSchedule.scenario_id.in_([_OWN_SCENARIO, _FOREIGN_SCENARIO])).delete()
|
|
db.query(ScenarioTriggerRule).filter(ScenarioTriggerRule.scenario_id.in_([_OWN_SCENARIO, _FOREIGN_SCENARIO])).delete()
|
|
db.query(AutomationPolicy).filter(AutomationPolicy.name.in_(["acl-policy-own", "acl-policy-foreign", "acl-policy-free"])).delete()
|
|
db.query(ScenarioRegistryEntry).filter(ScenarioRegistryEntry.scenario_id.in_([_OWN_SCENARIO, _FOREIGN_SCENARIO])).delete()
|
|
db.commit()
|
|
|
|
|
|
class TestScenarioAutomationReadAcl:
|
|
_READ_PATHS = (
|
|
"/api/scenario-automation/schedules",
|
|
"/api/scenario-automation/trigger-rules",
|
|
"/api/scenario-automation/policies",
|
|
"/api/scenario-automation/notifications",
|
|
"/api/scenario-automation/metrics",
|
|
"/api/scenario-automation/retention",
|
|
)
|
|
|
|
def test_reads_require_read_permission(self):
|
|
user = _make_user_with_permissions([("scenario:automation", "MANAGE"), ("scenario:automation", "TRIGGER")])
|
|
with _client_for(user) as client:
|
|
for path in self._READ_PATHS:
|
|
resp = client.get(path)
|
|
assert resp.status_code == 403, (path, resp.text)
|
|
|
|
def test_read_acl_filters_foreign_rows_without_totals_leak(self):
|
|
session = SessionLocal()
|
|
try:
|
|
ids = _seed_acl_rows(session)
|
|
finally:
|
|
session.close()
|
|
try:
|
|
with _client_for(_make_acl_user()) as client:
|
|
schedules = client.get("/api/scenario-automation/schedules")
|
|
assert schedules.status_code == 200, schedules.text
|
|
schedule_ids = {item["id"] for item in schedules.json()}
|
|
assert ids["own_schedule"] in schedule_ids
|
|
assert ids["foreign_schedule"] not in schedule_ids
|
|
|
|
rules = client.get("/api/scenario-automation/trigger-rules")
|
|
assert rules.status_code == 200, rules.text
|
|
rule_ids = {item["id"] for item in rules.json()}
|
|
assert ids["own_rule"] in rule_ids
|
|
assert ids["foreign_rule"] not in rule_ids
|
|
|
|
notifications = client.get("/api/scenario-automation/notifications", params={"limit": 100})
|
|
assert notifications.status_code == 200, notifications.text
|
|
notification_ids = {item["id"] for item in notifications.json()}
|
|
assert ids["own_notification"] in notification_ids
|
|
assert ids["foreign_notification"] not in notification_ids
|
|
|
|
policies = client.get("/api/scenario-automation/policies")
|
|
assert policies.status_code == 200, policies.text
|
|
policy_ids = {item["id"] for item in policies.json()}
|
|
assert ids["own_policy"] in policy_ids
|
|
assert ids["free_policy"] in policy_ids
|
|
assert ids["foreign_policy"] not in policy_ids
|
|
|
|
metrics = client.get("/api/scenario-automation/metrics")
|
|
assert metrics.status_code == 200, metrics.text
|
|
body = metrics.json()
|
|
assert body["schedules_total"] == 1
|
|
assert body["trigger_rules_total"] == 1
|
|
assert body["total_runs"] == 1
|
|
|
|
retention = client.get("/api/scenario-automation/retention")
|
|
assert retention.status_code == 200, retention.text
|
|
finally:
|
|
cleanup = SessionLocal()
|
|
try:
|
|
_drop_acl_rows(cleanup)
|
|
finally:
|
|
cleanup.close()
|
|
|
|
def test_admin_read_bypasses_acl_filter(self):
|
|
session = SessionLocal()
|
|
try:
|
|
ids = _seed_acl_rows(session)
|
|
finally:
|
|
session.close()
|
|
try:
|
|
with _client_for(_make_admin_user()) as client:
|
|
schedules = client.get("/api/scenario-automation/schedules")
|
|
assert schedules.status_code == 200, schedules.text
|
|
schedule_ids = {item["id"] for item in schedules.json()}
|
|
assert ids["own_schedule"] in schedule_ids
|
|
assert ids["foreign_schedule"] in schedule_ids
|
|
policies = client.get("/api/scenario-automation/policies")
|
|
policy_ids = {item["id"] for item in policies.json()}
|
|
assert ids["foreign_policy"] in policy_ids
|
|
finally:
|
|
cleanup = SessionLocal()
|
|
try:
|
|
_drop_acl_rows(cleanup)
|
|
finally:
|
|
cleanup.close()
|
|
|
|
def test_disabled_human_schedule_rejected_like_enabled(self):
|
|
setup = SessionLocal()
|
|
try:
|
|
setup.query(ScenarioRegistryEntry).filter(
|
|
ScenarioRegistryEntry.scenario_id.in_([_HUMAN_SCENARIO, _ELIGIBLE_SCENARIO])
|
|
).delete()
|
|
setup.commit()
|
|
setup.add(ScenarioRegistryEntry(
|
|
scenario_id=_HUMAN_SCENARIO, scenario_key="acl-human", name="ACL human fixture",
|
|
dashboard_id=73, environment_ids=["env-preprod-01"],
|
|
owner_id="acl-viewer-1", owner_username="acl.viewer",
|
|
lifecycle_status="READY", validation_status="valid", current_revision_id=_HUMAN_REVISION,
|
|
))
|
|
setup.add(ScenarioRevision(
|
|
revision_id=_HUMAN_REVISION, scenario_id=_HUMAN_SCENARIO, content_hash="c" * 64,
|
|
graph_snapshot={
|
|
"action_registry_version": ACTION_REGISTRY_VERSION,
|
|
"action_registry_hash": action_registry_fingerprint(),
|
|
"steps": [_action_step("human-acl-046", "human", "human_checkpoint")],
|
|
"dependencies": [],
|
|
},
|
|
created_by="acl.viewer", activation_status="current",
|
|
))
|
|
setup.add(ScenarioRegistryEntry(
|
|
scenario_id=_ELIGIBLE_SCENARIO, scenario_key="acl-eligible", name="ACL eligible fixture",
|
|
dashboard_id=74, environment_ids=["env-preprod-01"],
|
|
owner_id="acl-viewer-1", owner_username="acl.viewer",
|
|
lifecycle_status="READY", validation_status="valid", current_revision_id=_ELIGIBLE_REVISION,
|
|
))
|
|
setup.add(ScenarioRevision(
|
|
revision_id=_ELIGIBLE_REVISION, scenario_id=_ELIGIBLE_SCENARIO, content_hash="e" * 64,
|
|
graph_snapshot={
|
|
"action_registry_version": ACTION_REGISTRY_VERSION,
|
|
"action_registry_hash": action_registry_fingerprint(),
|
|
"steps": [_action_step("step-acl-046", "assertion", "structural_assert")],
|
|
"dependencies": [], "environment_ids": ["env-preprod-01"],
|
|
},
|
|
created_by="acl.viewer", activation_status="current",
|
|
))
|
|
setup.commit()
|
|
finally:
|
|
setup.close()
|
|
|
|
user = _make_user_with_permissions([("scenario:automation", "MANAGE")])
|
|
try:
|
|
with _client_for(user) as client:
|
|
enabled = client.post(
|
|
"/api/scenario-automation/schedules",
|
|
json={
|
|
"scenario_id": _HUMAN_SCENARIO, "environment_id": "env-preprod-01",
|
|
"cron_expr": "0 7 * * *", "revision_policy": "pinned",
|
|
"revision_id": _HUMAN_REVISION, "enabled": True,
|
|
},
|
|
)
|
|
assert enabled.status_code == 409, enabled.text
|
|
assert enabled.json()["detail"]["code"] == "AUTOMATION_INELIGIBLE_HUMAN_STEP"
|
|
disabled = client.post(
|
|
"/api/scenario-automation/schedules",
|
|
json={
|
|
"scenario_id": _HUMAN_SCENARIO, "environment_id": "env-preprod-01",
|
|
"cron_expr": "0 7 * * *", "revision_policy": "pinned",
|
|
"revision_id": _HUMAN_REVISION, "enabled": False,
|
|
},
|
|
)
|
|
assert disabled.status_code == 409, disabled.text
|
|
assert disabled.json()["detail"]["code"] == "AUTOMATION_INELIGIBLE_HUMAN_STEP"
|
|
|
|
created = client.post(
|
|
"/api/scenario-automation/schedules",
|
|
json={
|
|
"scenario_id": _ELIGIBLE_SCENARIO, "environment_id": "env-preprod-01",
|
|
"cron_expr": "0 6 * * *", "revision_policy": "pinned",
|
|
"revision_id": _ELIGIBLE_REVISION, "enabled": True,
|
|
},
|
|
)
|
|
assert created.status_code == 201, created.text
|
|
schedule_id = created.json()["id"]
|
|
patch = client.patch(
|
|
f"/api/scenario-automation/schedules/{schedule_id}",
|
|
json={
|
|
"scenario_id": _HUMAN_SCENARIO, "environment_id": "env-preprod-01",
|
|
"cron_expr": "0 6 * * *", "revision_policy": "pinned",
|
|
"revision_id": _HUMAN_REVISION, "enabled": False,
|
|
},
|
|
)
|
|
assert patch.status_code == 409, patch.text
|
|
assert patch.json()["detail"]["code"] == "AUTOMATION_INELIGIBLE_HUMAN_STEP"
|
|
|
|
verify = SessionLocal()
|
|
try:
|
|
assert verify.query(ScenarioSchedule).filter(
|
|
ScenarioSchedule.scenario_id == _HUMAN_SCENARIO
|
|
).count() == 0
|
|
row = verify.query(ScenarioSchedule).filter(ScenarioSchedule.id == schedule_id).first()
|
|
assert row is not None
|
|
assert row.scenario_id == _ELIGIBLE_SCENARIO
|
|
assert row.revision_id == _ELIGIBLE_REVISION
|
|
finally:
|
|
verify.close()
|
|
# Remove the scheduler job through the API so no in-memory APScheduler
|
|
# registration survives the fixture teardown.
|
|
deleted = client.delete(f"/api/scenario-automation/schedules/{schedule_id}")
|
|
assert deleted.status_code == 204
|
|
finally:
|
|
cleanup = SessionLocal()
|
|
try:
|
|
cleanup.query(ScenarioSchedule).filter(
|
|
ScenarioSchedule.scenario_id.in_([_HUMAN_SCENARIO, _ELIGIBLE_SCENARIO])
|
|
).delete()
|
|
cleanup.query(ScenarioRevision).filter(
|
|
ScenarioRevision.scenario_id.in_([_HUMAN_SCENARIO, _ELIGIBLE_SCENARIO])
|
|
).delete()
|
|
cleanup.query(ScenarioRegistryEntry).filter(
|
|
ScenarioRegistryEntry.scenario_id.in_([_HUMAN_SCENARIO, _ELIGIBLE_SCENARIO])
|
|
).delete()
|
|
cleanup.commit()
|
|
finally:
|
|
cleanup.close()
|
|
# #endregion Test.Api.ScenarioAutomation.ReadAcl
|
|
|
|
|
|
# #region Test.Api.ScenarioAutomation.RetentionReceipts [C:3] [TYPE Class] [SEMANTICS test,api,scenario,automation,retention,receipts,acl]
|
|
# @BRIEF 046 T021: the retention read projects deletion receipts under the same DG-2 ACL as the
|
|
# other five operational reads (READ grant + per-object scenario ownership, no totals leak).
|
|
# @TEST_INVARIANT Api.ScenarioAutomation.ReadAcl: foreign deletion receipts are filtered from the
|
|
# projection without leaking their count. -> VERIFIED_BY:
|
|
# test_retention_receipts_projected_with_acl
|
|
class TestScenarioAutomationRetentionReceipts:
|
|
def test_retention_receipts_projected_with_acl(self):
|
|
setup = SessionLocal()
|
|
try:
|
|
setup.query(ScenarioRetentionDeletion).filter(
|
|
ScenarioRetentionDeletion.scenario_id.in_([_OWN_SCENARIO, _FOREIGN_SCENARIO])
|
|
).delete()
|
|
setup.query(ScenarioRegistryEntry).filter(
|
|
ScenarioRegistryEntry.scenario_id.in_([_OWN_SCENARIO, _FOREIGN_SCENARIO])
|
|
).delete()
|
|
setup.add(ScenarioRegistryEntry(
|
|
scenario_id=_OWN_SCENARIO, scenario_key="acl-receipt-own", name="Receipt own",
|
|
dashboard_id=81, environment_ids=["env-preprod-01"],
|
|
owner_id="acl-viewer-1", owner_username="acl.viewer",
|
|
lifecycle_status="READY", validation_status="valid",
|
|
))
|
|
setup.add(ScenarioRegistryEntry(
|
|
scenario_id=_FOREIGN_SCENARIO, scenario_key="acl-receipt-foreign", name="Receipt foreign",
|
|
dashboard_id=82, environment_ids=["env-preprod-01"],
|
|
owner_id="foreign-owner-1", owner_username="foreign.owner",
|
|
lifecycle_status="READY", validation_status="valid",
|
|
))
|
|
setup.add(ScenarioRetentionDeletion(
|
|
target_type="artifact", target_id="receipt-own-001", scenario_id=_OWN_SCENARIO,
|
|
state="deletion_pending", holds_snapshot={"reasons": ["active_operation"]},
|
|
))
|
|
setup.add(ScenarioRetentionDeletion(
|
|
target_type="artifact", target_id="receipt-foreign-001", scenario_id=_FOREIGN_SCENARIO,
|
|
state="tombstoned", holds_snapshot={"reasons": []},
|
|
))
|
|
setup.commit()
|
|
finally:
|
|
setup.close()
|
|
try:
|
|
with _client_for(_make_acl_user()) as client:
|
|
body = client.get("/api/scenario-automation/retention").json()
|
|
assert body["tiers"]["raw_vlm"] == 7
|
|
receipt_targets = {item["target_id"] for item in body["deletions"]}
|
|
assert "receipt-own-001" in receipt_targets
|
|
assert "receipt-foreign-001" not in receipt_targets
|
|
assert body["deletions_total"] == 1
|
|
with _client_for(_make_admin_user()) as client:
|
|
admin_body = client.get("/api/scenario-automation/retention").json()
|
|
admin_targets = {item["target_id"] for item in admin_body["deletions"]}
|
|
assert {"receipt-own-001", "receipt-foreign-001"} <= admin_targets
|
|
finally:
|
|
cleanup = SessionLocal()
|
|
try:
|
|
cleanup.query(ScenarioRetentionDeletion).filter(
|
|
ScenarioRetentionDeletion.scenario_id.in_([_OWN_SCENARIO, _FOREIGN_SCENARIO])
|
|
).delete()
|
|
cleanup.query(ScenarioRegistryEntry).filter(
|
|
ScenarioRegistryEntry.scenario_id.in_([_OWN_SCENARIO, _FOREIGN_SCENARIO])
|
|
).delete()
|
|
cleanup.commit()
|
|
finally:
|
|
cleanup.close()
|
|
# #endregion Test.Api.ScenarioAutomation.RetentionReceipts
|
|
# #region Test.Api.ScenarioAutomation.QuarantineRelease [C:3] [TYPE Class] [SEMANTICS test,api,scenario,automation,quarantine,recovery,operator]
|
|
# @BRIEF Operator-only quarantine listing and CAS release; RBAC forbids a non-MANAGE principal.
|
|
# @RELATION VERIFIES -> [Api.ScenarioAutomation.Quarantine]
|
|
# @TEST_INVARIANT Api.ScenarioAutomation.Quarantine: release requires scenario:automation MANAGE and a matching
|
|
# quarantine version; a released pair re-enables schedules and a second release is 404.
|
|
# @TEST_EDGE missing_manage_scope -> 403 on quarantine release
|
|
class TestScenarioAutomationQuarantineRelease:
|
|
_SCENARIO = "77777777-7777-4777-8777-777777777777"
|
|
_ENV = "env-preprod-01"
|
|
|
|
def test_release_requires_manage_scope(self, monkeypatch, tmp_path):
|
|
monkeypatch.setenv("SCENARIO_POISONED_STORE_PATH", str(tmp_path / "poisoned.jsonl"))
|
|
user = _make_user_with_permissions([("scenario:automation", "TRIGGER")])
|
|
with _client_for(user) as client:
|
|
resp = client.post(
|
|
f"/api/scenario-automation/quarantine/{self._SCENARIO}/release",
|
|
json={"environment_id": self._ENV, "expected_version": 1},
|
|
)
|
|
assert resp.status_code == 403
|
|
|
|
def test_operator_release_roundtrip(self, dashboard_testing_client, monkeypatch, tmp_path):
|
|
client = dashboard_testing_client
|
|
store_path = tmp_path / "poisoned" / "failures.jsonl"
|
|
monkeypatch.setenv("SCENARIO_POISONED_STORE_PATH", str(store_path))
|
|
created = client.post(
|
|
"/api/scenario-automation/schedules",
|
|
json={"scenario_id": self._SCENARIO, "environment_id": self._ENV, "cron_expr": "0 7 * * *"},
|
|
)
|
|
assert created.status_code == 201, created.text
|
|
schedule_id = created.json()["id"]
|
|
disabled = client.patch(
|
|
f"/api/scenario-automation/schedules/{schedule_id}",
|
|
json={"scenario_id": self._SCENARIO, "environment_id": self._ENV, "cron_expr": "0 7 * * *", "enabled": False},
|
|
)
|
|
assert disabled.status_code == 200
|
|
|
|
store = PoisonedRunStore(store_path)
|
|
for _ in range(3):
|
|
store.record_failure(scenario_id=self._SCENARIO, environment_id=self._ENV, error_code="INFRA_TIMEOUT")
|
|
listed = client.get("/api/scenario-automation/quarantine")
|
|
assert listed.status_code == 200
|
|
assert any(
|
|
item["scenario_id"] == self._SCENARIO and item["version"] == 1
|
|
for item in listed.json()
|
|
)
|
|
|
|
released = client.post(
|
|
f"/api/scenario-automation/quarantine/{self._SCENARIO}/release",
|
|
json={"environment_id": self._ENV, "expected_version": 1},
|
|
)
|
|
assert released.status_code == 200, released.text
|
|
assert schedule_id in released.json()["enabled_schedule_ids"]
|
|
|
|
schedules = client.get("/api/scenario-automation/schedules")
|
|
assert next(item for item in schedules.json() if item["id"] == schedule_id)["enabled"] is True
|
|
assert client.get("/api/scenario-automation/quarantine").json() == []
|
|
|
|
again = client.post(
|
|
f"/api/scenario-automation/quarantine/{self._SCENARIO}/release",
|
|
json={"environment_id": self._ENV, "expected_version": 1},
|
|
)
|
|
assert again.status_code == 404
|
|
assert again.json()["detail"]["code"] == "NOT_QUARANTINED"
|
|
client.delete(f"/api/scenario-automation/schedules/{schedule_id}")
|
|
# #endregion Test.Api.ScenarioAutomation.QuarantineRelease
|
|
# #endregion Test.Api.ScenarioAutomation
|