busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(scenario): registry 038.5.0 with disabled-action discipline (AGSCN-FR-025)

Browse Source
This commit is contained in:
busya
2026-09-18 11:39:37 +03:00
parent fcf4e6eb87
commit d29959f6c5
7 changed files with 68 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
backend/src/services/dashboard_testing/scenario/templates/__init__.py
View File

@@ -16,8 +16,11 @@ from typing import Any
PHASES = ("setup", "interact", "observe", "assert", "evidence", "report")
TOOLS = ("browser", "superset_api", "sql_evidence", "transform", "xlsx", "assertion", "screenshot", "report", "artifact", "human", "agent_evaluation")
RISKS = ("read", "browser_interaction", "draft_write", "human")
ACTION_REGISTRY_VERSION = "038.4.0" # additive: canonical browser-actions.md names
ACTION_REGISTRY_VERSION = "038.5.0" # additive: canonical browser-actions.md names + disabled-pending-driver rows
# Registry discipline (AGSCN-FR-025): every registered action MUST be implemented with a canaried
# driver OR carry an explicit `disabled` reason. Compile and RunnerPlan derivation reject a disabled
# action before any I/O; the pre-038.5 runtime BROWSER_ACTION_NOT_SUPPORTED class is forbidden.
# Registered tool/action pairs (action -> phase, risk, required capability).
# Canonical browser names are emitted on new graphs. Legacy aliases stay registered
# so historical descriptors still resolve; the compiler never emits those names.
@@ -27,8 +30,12 @@ REGISTERED_ACTIONS: dict[str, dict[str, Any]] = {
"apply_native_filter": {"tool": "browser", "phase": "interact", "risk": "browser_interaction", "capability": "browser"},
"inspect_filter_state": {"tool": "browser", "phase": "observe", "risk": "read", "capability": "browser"},
"apply_table_filter": {"tool": "browser", "phase": "interact", "risk": "browser_interaction", "capability": "table_filter"},
"pagination": {"tool": "browser", "phase": "interact", "risk": "browser_interaction", "capability": "pagination"},
"navigate_dashboard": {"tool": "browser", "phase": "interact", "risk": "browser_interaction", "capability": "cross_dashboard"},
"pagination": {"tool": "browser", "phase": "interact", "risk": "browser_interaction", "capability": "pagination", "disabled": "pending driver"},
"navigate_dashboard": {"tool": "browser", "phase": "interact", "risk": "browser_interaction", "capability": "cross_dashboard", "disabled": "pending driver"},
"assert_dom": {"tool": "browser", "phase": "observe", "risk": "read", "capability": "browser", "disabled": "pending driver"},
"inspect_filter_options": {"tool": "browser", "phase": "observe", "risk": "read", "capability": "browser", "disabled": "pending driver"},
"navigate_tabs": {"tool": "browser", "phase": "interact", "risk": "browser_interaction", "capability": "browser", "disabled": "pending driver"},
"wait_for_selector": {"tool": "browser", "phase": "observe", "risk": "read", "capability": "browser", "disabled": "pending driver"},
"extract_table": {"tool": "browser", "phase": "observe", "risk": "read", "capability": "browser"},
"scroll_to": {"tool": "browser", "phase": "interact", "risk": "browser_interaction", "capability": "browser"},
"inspect_columns": {"tool": "browser", "phase": "observe", "risk": "read", "capability": "browser"},
@@ -61,6 +68,15 @@ REGISTERED_ACTIONS: dict[str, dict[str, Any]] = {
"evaluate_declared_spec": {"tool": "agent_evaluation", "phase": "assert", "risk": "read", "capability": None},
}
# AGSCN-FR-025: registered-but-unimplemented is forbidden. Each entry above is either implemented
# with a canaried driver or explicitly disabled with a reason. Historical plans pinned to an older
# registry version already fail the strict version check; these maps drive the pre-I/O rejection.
DISABLED_ACTIONS: dict[str, str] = {
action: str(entry["disabled"])
for action, entry in REGISTERED_ACTIONS.items()
if entry.get("disabled")
}
# #region ScenarioGraph.Templates.ActionRegistry [C:5] [TYPE Class] [SEMANTICS scenario,action,registry,descriptor,side-effect]
# @BRIEF Version-pinned 038 ActionRegistry execution metadata shared with 044 RunnerPlan preflight.

15
backend/src/services/dashboard_testing/scenario/validator.py
View File

@@ -159,15 +159,24 @@ def _check_refs(steps: list[dict[str, Any]], result: ScenarioValidationResult) -
# #region ScenarioGraph.Validator.CheckTools [C:2] [TYPE Function] [SEMANTICS scenario,validator,tools]
# @ingroup ScenarioGraph
# @BRIEF Check tool/action pairs against the registered registry.
# @BRIEF Check tool/action pairs against the registered registry, including disabled actions.
# @POST A step whose action carries a registry `disabled` reason receives an ACTION_DISABLED
# error before compile/I/O (AGSCN-FR-025: registered-but-unimplemented is forbidden).
# @REJECTED Letting disabled actions compile and fail later as runtime
# BROWSER_ACTION_NOT_SUPPORTED was rejected — that silent break class is what
# the 038.5.0 discipline exists to prevent.
def _check_tools(steps: list[dict[str, Any]], result: ScenarioValidationResult) -> None:
for s in steps:
action, tool = s.get("action"), s.get("tool")
entry = REGISTERED_ACTIONS.get(action)
if entry is None:
result.errors.append(_err("UNKNOWN_TOOL", f"unregistered action {action!r}", step_id=s["id"], recovery=["remove step", "use registered action"]))
elif entry["tool"] != tool:
result.errors.append(_err("TOOL_MISMATCH", f"action {action!r} registered for {entry['tool']!r}, got {tool!r}", step_id=s["id"]))
else:
if entry["tool"] != tool:
result.errors.append(_err("TOOL_MISMATCH", f"action {action!r} registered for {entry['tool']!r}, got {tool!r}", step_id=s["id"]))
disabled_reason = entry.get("disabled")
if disabled_reason:
result.errors.append(_err("ACTION_DISABLED", f"action {action!r} is disabled: {disabled_reason}", step_id=s["id"], recovery=["remove step", "use implemented action"]))
if tool not in TOOLS:
result.errors.append(_err("UNKNOWN_TOOL", f"unknown tool {tool!r}", step_id=s["id"]))
# #endregion ScenarioGraph.Validator.CheckTools

6
backend/tests/api/test_ready.py
View File

@@ -102,7 +102,7 @@ class TestGetReady:
"superset": {
"state": "ready",
"provider": "superset",
"version": "038.4.0",
"version": "038.5.0",
"capabilities": {
"features": ["dataset_field_read", "superset_metric", "superset_query_envelope"],
"registry_fingerprint": "a" * 64,
@@ -116,7 +116,7 @@ class TestGetReady:
"screenshot": {
"state": "ready",
"provider": "screenshot",
"version": "038.4.0",
"version": "038.5.0",
"capabilities": {"features": ["screenshot"], "registry_fingerprint": "a" * 64},
"dependencies": {
"evidence_storage_ready": True,
@@ -129,7 +129,7 @@ class TestGetReady:
"state": "degraded",
"reason_code": "BROWSER_EXECUTABLE_MISSING",
"provider": "browser",
"version": "038.4.0",
"version": "038.5.0",
"capabilities": {"features": ["browser"], "registry_fingerprint": "a" * 64},
"dependencies": {
"chromium_executable_resolved": False,

4
backend/tests/fixtures/scenario_execution/graph.json vendored
View File

@@ -1,8 +1,8 @@
{
"schema_version": 1,
"compiler_version": "038.1.0",
"action_registry_version": "038.4.0",
"action_registry_hash": "9839099380356b2ebf482eb3b3c34c64320dc88fcc4694288e8d4278291c7310",
"action_registry_version": "038.5.0",
"action_registry_hash": "234807c6fc1fbc97436e62223bce7c4375d617949a38e8369c0cf93f85dd8b82",
"scenario_id": "fi-0080_verify-filters-metric-xlsx",
"revision_hash": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
"objective": {

6
backend/tests/services/dashboard_testing/registry/test_scenario_runner.py
View File

@@ -141,9 +141,9 @@ def test_start_run_pins_revision_content_hash_not_request_hash(seeded_registry):
@pytest.mark.parametrize(
("registry_version", "registry_hash", "action", "error_code"),
[
("038.4.0", action_registry_fingerprint(), "not_registered", "ACTION_DESCRIPTOR_UNKNOWN"),
("038.5.0", action_registry_fingerprint(), "not_registered", "ACTION_DESCRIPTOR_UNKNOWN"),
("038.0.0", action_registry_fingerprint(), "structural_assert", "ACTION_REGISTRY_VERSION_MISMATCH"),
("038.4.0", "0" * 64, "structural_assert", "ACTION_REGISTRY_HASH_MISMATCH"),
("038.5.0", "0" * 64, "structural_assert", "ACTION_REGISTRY_HASH_MISMATCH"),
],
)
def test_action_preflight_rejects_before_run_creation(
@@ -225,7 +225,7 @@ def test_prod_browser_mutation_rejects_before_run_creation(seeded_execution):
revision_id="bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1"
).one()
revision.graph_snapshot = {
"action_registry_version": "038.4.0",
"action_registry_version": "038.5.0",
"action_registry_hash": action_registry_fingerprint(),
"steps": [{
"logical_step_id": "prod-browser-write-044", "tool": "browser", "action": "row_edit",

4
backend/tests/services/dashboard_testing/registry/test_scenario_runner_plan.py
View File

@@ -48,7 +48,7 @@ def test_derived_plan_matches_hardcoded_fixture(seeded_execution):
assert plan["resolved_params"] == expected["resolved_params"]
assert plan["pinned_baselines"] == expected["pinned_baselines"]
assert plan["topological_order"] == expected["topological_order"]
assert plan["action_registry_version"] == "038.4.0"
assert plan["action_registry_version"] == "038.5.0"
assert plan["action_registry_hash"] == action_registry_fingerprint()
assert [plan["executor_mapping"][step_id]["action"] for step_id in plan["topological_order"]] == [
"apply_filters", "execute_metric", "parse_xlsx", "compare_to_baseline",
@@ -131,7 +131,7 @@ def test_malformed_pinned_policy_is_rejected():
# @TEST_EDGE: params absent/other actions -> no param_binding; provider stays in UX-1 current-state mode
# @TEST_EDGE: present-but-invalid param -> typed BROWSER_FILTER_VALUES_INVALID before any run row exists
_APPLY_FILTER_GRAPH = {
"action_registry_version": "038.4.0",
"action_registry_version": "038.5.0",
"action_registry_hash": action_registry_fingerprint(),
"environment_ids": ["env-dev"],
"dependencies": [],

27
backend/tests/services/dashboard_testing/scenario/test_validator.py
View File

@@ -103,4 +103,31 @@ def test_dashboard_context_sql_text_rejected() -> None:
result = validate_scenario(DashboardTestScenario.model_validate(data))
assert result.valid is False
assert "FORBIDDEN_SQL" in _codes(result)
def test_disabled_action_rejected_before_io() -> None:
# AGSCN-FR-025: registered-but-unimplemented actions must not pass validation.
data = _load_raw("scenario_valid.json")
data["steps"][0]["action"] = "pagination"
data["steps"][0]["tool"] = "browser"
result = validate_scenario(DashboardTestScenario.model_validate(data))
assert result.valid is False
assert "ACTION_DISABLED" in _codes(result)
assert any("pagination" in f.message for f in result.errors)
def test_disabled_new_wave_action_rejected() -> None:
# Wave-2 drivers (assert_dom etc.) are registered disabled until their canary gates close.
data = _load_raw("scenario_valid.json")
data["steps"][0]["action"] = "assert_dom"
data["steps"][0]["tool"] = "browser"
result = validate_scenario(DashboardTestScenario.model_validate(data))
assert result.valid is False
assert "ACTION_DISABLED" in _codes(result)
def test_implemented_action_not_flagged_disabled() -> None:
data = _load_raw("scenario_valid.json")
result = validate_scenario(DashboardTestScenario.model_validate(data))
assert "ACTION_DISABLED" not in _codes(result)
# #endregion Test.Scenario.Validator