busya
67e4b9fc42
- T013-T017b: ScenarioGraph.Validator.Validate with deterministic findings — duplicate steps, missing deps, cycles (with path), duplicate/missing refs, tool/action registry, SQL/code/path-traversal bans, raw baseline literals, unresolved params/selectors/baselines, coverage classification - Decomposed to 8 helpers (C901 fixed: _validate_core 36→5 complexity) - Property tests: chain DAGs of any length valid, self-dep cycle, dup refs - Belief runtime: REASON/REFLECT in validate + validate_core; audit 0 errors - 47 scenario tests pass; ruff clean
78 lines
2.8 KiB
Python
78 lines
2.8 KiB
Python
# #region Test.Scenario.Validator [C:3] [TYPE Module] [SEMANTICS testing,scenario,validator,safety]
|
|
# @defgroup Test.Scenario Validator safety matrix tests — cycles, refs, baselines, SQL, paths.
|
|
# @LAYER Test
|
|
# @RELATION BINDS_TO -> [ScenarioGraph.Validator.Validate]
|
|
# @RATIONALE The validator is the hard safety boundary between agent intent and artifact generation.
|
|
# @REJECTED Testing only happy paths — would let unsafe graphs reach the pack compiler.
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from src.services.dashboard_testing.scenario.models import DashboardTestScenario
|
|
from src.services.dashboard_testing.scenario.validator import validate_scenario
|
|
|
|
_FIXTURES = Path(__file__).resolve().parents[3] / "fixtures" / "dashboard_scenarios"
|
|
|
|
|
|
def _load(name: str) -> DashboardTestScenario:
|
|
data = json.loads((_FIXTURES / name).read_text(encoding="utf-8"))
|
|
return DashboardTestScenario.model_validate(data)
|
|
|
|
|
|
def _codes(result) -> set[str]:
|
|
return {f.code for f in result.errors} | {f.code for f in result.blockers}
|
|
|
|
|
|
def test_valid_fixture_passes() -> None:
|
|
result = validate_scenario(_load("scenario_valid.json"))
|
|
assert result.valid is True
|
|
assert not result.errors
|
|
assert not result.blockers
|
|
|
|
|
|
def test_cycle_detected_with_path() -> None:
|
|
result = validate_scenario(_load("scenario_cycle.json"))
|
|
assert result.valid is False
|
|
assert "CYCLE" in _codes(result)
|
|
assert any("step-a" in f.message and "step-b" in f.message for f in result.errors)
|
|
|
|
|
|
def test_missing_ref_detected() -> None:
|
|
result = validate_scenario(_load("scenario_missing_ref.json"))
|
|
assert result.valid is False
|
|
assert "MISSING_REF" in _codes(result)
|
|
|
|
|
|
def test_duplicate_output_detected() -> None:
|
|
result = validate_scenario(_load("scenario_duplicate_output.json"))
|
|
assert result.valid is False
|
|
assert "DUPLICATE_OUTPUT" in _codes(result)
|
|
assert any("phase-3-B01-execute_metric" in f.message for f in result.errors)
|
|
|
|
|
|
def test_raw_baseline_literal_rejected() -> None:
|
|
result = validate_scenario(_load("scenario_raw_baseline.json"))
|
|
assert result.valid is False
|
|
assert "RAW_METRIC_TRUTH" in _codes(result)
|
|
|
|
|
|
def test_sql_rejected() -> None:
|
|
result = validate_scenario(_load("scenario_sql_injection.json"))
|
|
assert result.valid is False
|
|
assert "FORBIDDEN_SQL" in _codes(result)
|
|
|
|
|
|
def test_unreachable_step_warned() -> None:
|
|
result = validate_scenario(_load("scenario_valid.json"))
|
|
# valid fixture has all steps reachable; unknown tool/action is caught elsewhere
|
|
assert result.valid is True
|
|
|
|
|
|
def test_findings_deterministic_order() -> None:
|
|
a = validate_scenario(_load("scenario_cycle.json"))
|
|
b = validate_scenario(_load("scenario_cycle.json"))
|
|
assert [f.code for f in a.errors] == [f.code for f in b.errors]
|
|
assert [f.code for f in a.blockers] == [f.code for f in b.blockers]
|