feat(038): Phase 3 — US2 validator safety matrix
- T013-T017b: ScenarioGraph.Validator.Validate with deterministic findings — duplicate steps, missing deps, cycles (with path), duplicate/missing refs, tool/action registry, SQL/code/path-traversal bans, raw baseline literals, unresolved params/selectors/baselines, coverage classification - Decomposed to 8 helpers (C901 fixed: _validate_core 36→5 complexity) - Property tests: chain DAGs of any length valid, self-dep cycle, dup refs - Belief runtime: REASON/REFLECT in validate + validate_core; audit 0 errors - 47 scenario tests pass; ruff clean
This commit is contained in:
298
backend/src/services/dashboard_testing/scenario/validator.py
Normal file
298
backend/src/services/dashboard_testing/scenario/validator.py
Normal file
@@ -0,0 +1,298 @@
|
||||
# #region ScenarioGraph.Validator.Validate [C:5] [TYPE Function] [SEMANTICS scenario,validator,graph,safety]
|
||||
# @ingroup ScenarioGraph
|
||||
# @BRIEF Return complete deterministic findings for schema, DAG, refs, parameters, baselines, tools, safety, and coverage.
|
||||
# @PRE Candidate graph parses against supported schema version.
|
||||
# @POST Valid is true only with zero errors/blockers; findings are stably ordered and actionable.
|
||||
# @SIDE_EFFECT None.
|
||||
# @SIDE_EFFECT Logging (REASON before validation; REFLECT with finding counts after).
|
||||
# @DATA_CONTRACT DashboardTestScenario -> ScenarioValidationResult
|
||||
# @RATIONALE Validation is a hard safety boundary between agent-produced intent and artifact generation; deterministic findings give the user a recoverable explanation instead of a runtime surprise.
|
||||
# @REJECTED Silent graph repair or best-effort artifact generation — rejected because auto-fixing refs, cycles, or unsafe actions can change business intent without review.
|
||||
# @INVARIANT Cycles, missing/duplicate refs, unregistered tools, SQL, raw metric truth, and path traversal block compilation.
|
||||
# @TEST_EDGE cycle -> error contains cycle path.
|
||||
# @TEST_EDGE duplicate_output -> both producer ids reported.
|
||||
# @TEST_EDGE raw_metric_expected -> forbidden baseline literal error.
|
||||
# @TEST_EDGE unreachable_step -> warning/error according to required coverage.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from ss_tools.shared.cot_logger import log
|
||||
|
||||
from src.core.logger import belief_scope
|
||||
from src.services.dashboard_testing.scenario.models import DashboardTestScenario, Finding
|
||||
from src.services.dashboard_testing.scenario.templates import REGISTERED_ACTIONS, TOOLS
|
||||
|
||||
_SQL_TOKENS = ("select ", "insert ", "update ", "delete from", "drop table", "create table", "select *", "join ")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScenarioValidationResult:
|
||||
valid: bool = False
|
||||
errors: list[Finding] = field(default_factory=list)
|
||||
warnings: list[Finding] = field(default_factory=list)
|
||||
blockers: list[Finding] = field(default_factory=list)
|
||||
coverage: list[dict[str, Any]] = field(default_factory=list)
|
||||
topological_order: list[str] = field(default_factory=list)
|
||||
unresolved_parameters: list[str] = field(default_factory=list)
|
||||
unresolved_selectors: list[str] = field(default_factory=list)
|
||||
unresolved_baselines: list[str] = field(default_factory=list)
|
||||
graph_hash: str = ""
|
||||
|
||||
|
||||
def _err(code: str, msg: str, *, step_id: str | None = None, recovery: list[str] | None = None) -> Finding:
|
||||
return Finding(code=code, severity="error", message=msg, step_id=step_id, recovery_options=recovery or [])
|
||||
|
||||
|
||||
def _warn(code: str, msg: str, *, step_id: str | None = None) -> Finding:
|
||||
return Finding(code=code, severity="warning", message=msg, step_id=step_id)
|
||||
|
||||
|
||||
def _find_cycles(steps: list[dict[str, Any]]) -> list[str]:
|
||||
"""Return one representative cycle path per detected cycle (deterministic order)."""
|
||||
ids = {s["id"] for s in steps}
|
||||
deps: dict[str, list[str]] = {s["id"]: [d for d in s.get("depends_on", []) if d in ids] for s in steps}
|
||||
visited: set[str] = set()
|
||||
stack: list[str] = []
|
||||
cycles: list[str] = []
|
||||
|
||||
def visit(node: str) -> None:
|
||||
if node in stack:
|
||||
idx = stack.index(node)
|
||||
cycles.append(" -> ".join([*stack[idx:], node]))
|
||||
return
|
||||
if node in visited:
|
||||
return
|
||||
visited.add(node)
|
||||
stack.append(node)
|
||||
for d in sorted(deps.get(node, [])):
|
||||
visit(d)
|
||||
stack.pop()
|
||||
|
||||
for node in sorted(ids):
|
||||
visit(node)
|
||||
return sorted(set(cycles))
|
||||
|
||||
|
||||
def _detect_sql(text: str | None) -> bool:
|
||||
if not text:
|
||||
return False
|
||||
lowered = text.lower()
|
||||
return any(tok in lowered for tok in _SQL_TOKENS)
|
||||
|
||||
|
||||
# #region ScenarioGraph.Validator.ValidateCore [C:4] [TYPE Function] [SEMANTICS scenario,validator,checks]
|
||||
# @ingroup ScenarioGraph
|
||||
# @BRIEF Pure validation core — runs all checks and returns findings (no logging).
|
||||
# @POST Returns ScenarioValidationResult with deterministic, ordered findings.
|
||||
def _validate_core(scenario: DashboardTestScenario) -> ScenarioValidationResult:
|
||||
log("ScenarioGraph.Validator.ValidateCore", "REASON", "Starting validation checks",
|
||||
{"scenario_id": scenario.scenario_id})
|
||||
result = _validate_checks(scenario)
|
||||
log("ScenarioGraph.Validator.ValidateCore", "REFLECT", "Validation checks complete",
|
||||
{"valid": result.valid, "errors": len(result.errors)})
|
||||
return result
|
||||
# #endregion ScenarioGraph.Validator.ValidateCore
|
||||
|
||||
|
||||
# #region ScenarioGraph.Validator.ValidateChecks [C:3] [TYPE Function] [SEMANTICS scenario,validator,checks]
|
||||
# @ingroup ScenarioGraph
|
||||
# @BRIEF Pure check orchestration — runs all check helpers and assembles the result.
|
||||
# @POST Returns ScenarioValidationResult with deterministic, ordered findings.
|
||||
def _validate_checks(scenario: DashboardTestScenario) -> ScenarioValidationResult:
|
||||
result = ScenarioValidationResult()
|
||||
steps = [s.model_dump() for s in scenario.steps]
|
||||
id_set = {s["id"] for s in steps}
|
||||
|
||||
_check_structure(steps, id_set, result)
|
||||
_check_refs(steps, result)
|
||||
_check_tools(steps, result)
|
||||
_check_safety(steps, result)
|
||||
_check_expectations(steps, result)
|
||||
_check_artifact_paths(scenario, result)
|
||||
_check_unresolved(scenario, steps, result)
|
||||
_check_coverage(scenario, result)
|
||||
|
||||
result.topological_order = _topological_order(steps)
|
||||
result.valid = not result.errors and not result.blockers
|
||||
result.graph_hash = scenario.revision_hash
|
||||
return result
|
||||
# #endregion ScenarioGraph.Validator.ValidateChecks
|
||||
|
||||
|
||||
# #region ScenarioGraph.Validator.CheckStructure [C:2] [TYPE Function] [SEMANTICS scenario,validator,structure]
|
||||
# @ingroup ScenarioGraph
|
||||
# @BRIEF Check duplicate step ids, unknown dependencies, and dependency cycles.
|
||||
def _check_structure(steps: list[dict[str, Any]], id_set: set[str], result: ScenarioValidationResult) -> None:
|
||||
seen: set[str] = set()
|
||||
for s in steps:
|
||||
if s["id"] in seen:
|
||||
result.errors.append(_err("DUPLICATE_STEP", f"duplicate step id {s['id']}", step_id=s["id"]))
|
||||
seen.add(s["id"])
|
||||
for s in steps:
|
||||
for dep in s.get("depends_on", []):
|
||||
if dep not in id_set:
|
||||
result.errors.append(_err("MISSING_DEPENDENCY", f"step {s['id']} depends on unknown step {dep}", step_id=s["id"]))
|
||||
for cycle in _find_cycles(steps):
|
||||
result.errors.append(_err("CYCLE", f"dependency cycle: {cycle}"))
|
||||
# #endregion ScenarioGraph.Validator.CheckStructure
|
||||
|
||||
|
||||
# #region ScenarioGraph.Validator.CheckRefs [C:2] [TYPE Function] [SEMANTICS scenario,validator,refs]
|
||||
# @ingroup ScenarioGraph
|
||||
# @BRIEF Check ref producer/consumer consistency and duplicate outputs.
|
||||
def _check_refs(steps: list[dict[str, Any]], result: ScenarioValidationResult) -> None:
|
||||
producers: dict[str, str] = {}
|
||||
for s in steps:
|
||||
for out in s.get("outputs", []):
|
||||
name = out["name"]
|
||||
if name in producers:
|
||||
result.errors.append(_err(
|
||||
"DUPLICATE_OUTPUT",
|
||||
f"output ref {name} produced by both {producers[name]} and {s['id']}",
|
||||
step_id=s["id"],
|
||||
))
|
||||
else:
|
||||
producers[name] = s["id"]
|
||||
produced = set(producers)
|
||||
for s in steps:
|
||||
for inp in s.get("inputs", []):
|
||||
name = inp["name"]
|
||||
if name.startswith("step.") and name not in produced:
|
||||
result.errors.append(_err("MISSING_REF", f"step {s['id']} consumes missing output ref {name}", step_id=s["id"]))
|
||||
# #endregion ScenarioGraph.Validator.CheckRefs
|
||||
|
||||
|
||||
# #region ScenarioGraph.Validator.CheckTools [C:2] [TYPE Function] [SEMANTICS scenario,validator,tools]
|
||||
# @ingroup ScenarioGraph
|
||||
# @BRIEF Check tool/action pairs against the registered registry.
|
||||
def _check_tools(steps: list[dict[str, Any]], result: ScenarioValidationResult) -> None:
|
||||
for s in steps:
|
||||
action, tool = s.get("action"), s.get("tool")
|
||||
entry = REGISTERED_ACTIONS.get(action)
|
||||
if entry is None:
|
||||
result.errors.append(_err("UNKNOWN_TOOL", f"unregistered action {action!r}", step_id=s["id"], recovery=["remove step", "use registered action"]))
|
||||
elif entry["tool"] != tool:
|
||||
result.errors.append(_err("TOOL_MISMATCH", f"action {action!r} registered for {entry['tool']!r}, got {tool!r}", step_id=s["id"]))
|
||||
if tool not in TOOLS:
|
||||
result.errors.append(_err("UNKNOWN_TOOL", f"unknown tool {tool!r}", step_id=s["id"]))
|
||||
# #endregion ScenarioGraph.Validator.CheckTools
|
||||
|
||||
|
||||
# #region ScenarioGraph.Validator.CheckSafety [C:2] [TYPE Function] [SEMANTICS scenario,validator,safety]
|
||||
# @ingroup ScenarioGraph
|
||||
# @BRIEF Check SQL, executable code, and forbidden actions in step text and tooling.
|
||||
def _check_safety(steps: list[dict[str, Any]], result: ScenarioValidationResult) -> None:
|
||||
code_tokens = ("import os", "subprocess", "shell=True", "__import__")
|
||||
for s in steps:
|
||||
if _detect_sql(s.get("description")) or _detect_sql(s.get("title")):
|
||||
result.errors.append(_err("FORBIDDEN_SQL", f"step {s['id']} contains SQL", step_id=s["id"]))
|
||||
if s.get("tool") == "superset_api" and s.get("action") in {"raw_sql", "execute_sql", "run_query"}:
|
||||
result.errors.append(_err("FORBIDDEN_SQL", f"step {s['id']} uses forbidden SQL action", step_id=s["id"]))
|
||||
if s.get("description") and any(tok in s["description"].lower() for tok in code_tokens):
|
||||
result.errors.append(_err("FORBIDDEN_CODE", f"step {s['id']} contains executable code", step_id=s["id"]))
|
||||
# #endregion ScenarioGraph.Validator.CheckSafety
|
||||
|
||||
|
||||
# #region ScenarioGraph.Validator.CheckExpectations [C:2] [TYPE Function] [SEMANTICS scenario,validator,expectations]
|
||||
# @ingroup ScenarioGraph
|
||||
# @BRIEF Check baseline refs and reject raw numeric truth in expectations.
|
||||
def _check_expectations(steps: list[dict[str, Any]], result: ScenarioValidationResult) -> None:
|
||||
literal_hints = ("must equal", "equals", "= ", "==")
|
||||
for s in steps:
|
||||
exp = s.get("expected", {})
|
||||
if exp.get("kind") == "baseline_ref" and not exp.get("ref"):
|
||||
result.errors.append(_err("RAW_METRIC_TRUTH", f"step {s['id']} embeds raw numeric truth without baseline ref", step_id=s["id"]))
|
||||
desc = (exp.get("description") or "")
|
||||
if exp.get("kind") == "structural" and any(ch.isdigit() for ch in desc) and any(tok in desc.lower() for tok in literal_hints):
|
||||
result.errors.append(_err("RAW_METRIC_TRUTH", f"step {s['id']} embeds expected numeric literal", step_id=s["id"]))
|
||||
ref = exp.get("ref")
|
||||
if ref and (ref.startswith("baseline.") or ref.startswith("candidate.")) and not ref.split(".", 1)[1]:
|
||||
result.errors.append(_err("MISSING_BASELINE", f"step {s['id']} has empty baseline ref", step_id=s["id"]))
|
||||
# #endregion ScenarioGraph.Validator.CheckExpectations
|
||||
|
||||
|
||||
# #region ScenarioGraph.Validator.CheckArtifactPaths [C:1] [TYPE Function] [SEMANTICS scenario,validator,paths]
|
||||
# @ingroup ScenarioGraph
|
||||
# @BRIEF Reject absolute and parent-traversing artifact paths.
|
||||
def _check_artifact_paths(scenario: DashboardTestScenario, result: ScenarioValidationResult) -> None:
|
||||
for plan in scenario.artifact_plan:
|
||||
path = plan.relative_path_template
|
||||
if path.startswith("/") or ".." in path:
|
||||
result.errors.append(_err("PATH_TRAVERSAL", f"artifact path unsafe: {path}"))
|
||||
# #endregion ScenarioGraph.Validator.CheckArtifactPaths
|
||||
|
||||
|
||||
# #region ScenarioGraph.Validator.CheckUnresolved [C:2] [TYPE Function] [SEMANTICS scenario,validator,unresolved]
|
||||
# @ingroup ScenarioGraph
|
||||
# @BRIEF Collect unresolved parameters, selectors, and baselines into errors + result lists.
|
||||
def _check_unresolved(scenario: DashboardTestScenario, steps: list[dict[str, Any]], result: ScenarioValidationResult) -> None:
|
||||
for p in scenario.parameters:
|
||||
if p.required and p.status == "unresolved":
|
||||
result.unresolved_parameters.append(p.name)
|
||||
for s in steps:
|
||||
if s.get("automation_status") == "needs_selector":
|
||||
result.unresolved_selectors.append(s["id"])
|
||||
result.errors.append(_err("NEEDS_SELECTOR", f"step {s['id']} requires a UI selector", step_id=s["id"], recovery=["provide selector hint", "convert to human checkpoint"]))
|
||||
for s in steps:
|
||||
if s.get("automation_status") == "needs_baseline":
|
||||
result.unresolved_baselines.append(s["id"])
|
||||
result.errors.append(_err("NEEDS_BASELINE", f"step {s['id']} requires a baseline", step_id=s["id"], recovery=["run 037 baseline discovery", "mark pending"]))
|
||||
# #endregion ScenarioGraph.Validator.CheckUnresolved
|
||||
|
||||
|
||||
# #region ScenarioGraph.Validator.CheckCoverage [C:1] [TYPE Function] [SEMANTICS scenario,validator,coverage]
|
||||
# @ingroup ScenarioGraph
|
||||
# @BRIEF Validate checklist coverage classifications.
|
||||
def _check_coverage(scenario: DashboardTestScenario, result: ScenarioValidationResult) -> None:
|
||||
valid_classes = {"automated", "human_checkpoint", "unsupported", "needs_context"}
|
||||
for c in scenario.checklist_coverage:
|
||||
if c.classification not in valid_classes:
|
||||
result.errors.append(_err("BAD_COVERAGE", f"case {c.case_id} has invalid classification {c.classification!r}"))
|
||||
# #endregion ScenarioGraph.Validator.CheckCoverage
|
||||
|
||||
|
||||
# #region ScenarioGraph.Validator.TopologicalOrder [C:2] [TYPE Function] [SEMANTICS scenario,topological,dag]
|
||||
# @ingroup ScenarioGraph
|
||||
# @BRIEF Deterministic topological order via Kahn's algorithm with sorted tie-breakers.
|
||||
# @POST Returns ordered step ids; cycles excluded from output.
|
||||
def _topological_order(steps: list[dict[str, Any]]) -> list[str]:
|
||||
ids = [s["id"] for s in steps]
|
||||
id_set = set(ids)
|
||||
indeg = {s["id"]: 0 for s in steps}
|
||||
adj: dict[str, list[str]] = {s["id"]: [] for s in steps}
|
||||
for s in steps:
|
||||
for d in s.get("depends_on", []):
|
||||
if d in id_set:
|
||||
adj[d].append(s["id"])
|
||||
indeg[s["id"]] += 1
|
||||
ready = sorted([n for n, d in indeg.items() if d == 0])
|
||||
order: list[str] = []
|
||||
while ready:
|
||||
n = ready.pop(0)
|
||||
order.append(n)
|
||||
for m in sorted(adj[n]):
|
||||
indeg[m] -= 1
|
||||
if indeg[m] == 0:
|
||||
ready.append(m)
|
||||
return order
|
||||
# #endregion ScenarioGraph.Validator.TopologicalOrder
|
||||
|
||||
|
||||
def validate_scenario(scenario: DashboardTestScenario) -> ScenarioValidationResult:
|
||||
"""Validate a candidate graph and return all deterministic findings."""
|
||||
log("ScenarioGraph.Validator.Validate", "REASON", "Validating scenario graph",
|
||||
{"scenario_id": scenario.scenario_id, "steps": len(scenario.steps)})
|
||||
with belief_scope("ScenarioGraph.Validator.Validate", "Validating scenario"):
|
||||
try:
|
||||
result = _validate_core(scenario)
|
||||
except Exception as e:
|
||||
log("ScenarioGraph.Validator.Validate", "EXPLORE", "Validation crashed",
|
||||
error=str(e))
|
||||
raise
|
||||
log("ScenarioGraph.Validator.Validate", "REFLECT", "Validation complete",
|
||||
{"valid": result.valid, "errors": len(result.errors), "warnings": len(result.warnings)})
|
||||
return result
|
||||
# #endregion ScenarioGraph.Validator.Validate
|
||||
@@ -0,0 +1,77 @@
|
||||
# #region Test.Scenario.Validator [C:3] [TYPE Module] [SEMANTICS testing,scenario,validator,safety]
|
||||
# @defgroup Test.Scenario Validator safety matrix tests — cycles, refs, baselines, SQL, paths.
|
||||
# @LAYER Test
|
||||
# @RELATION BINDS_TO -> [ScenarioGraph.Validator.Validate]
|
||||
# @RATIONALE The validator is the hard safety boundary between agent intent and artifact generation.
|
||||
# @REJECTED Testing only happy paths — would let unsafe graphs reach the pack compiler.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from src.services.dashboard_testing.scenario.models import DashboardTestScenario
|
||||
from src.services.dashboard_testing.scenario.validator import validate_scenario
|
||||
|
||||
_FIXTURES = Path(__file__).resolve().parents[3] / "fixtures" / "dashboard_scenarios"
|
||||
|
||||
|
||||
def _load(name: str) -> DashboardTestScenario:
|
||||
data = json.loads((_FIXTURES / name).read_text(encoding="utf-8"))
|
||||
return DashboardTestScenario.model_validate(data)
|
||||
|
||||
|
||||
def _codes(result) -> set[str]:
|
||||
return {f.code for f in result.errors} | {f.code for f in result.blockers}
|
||||
|
||||
|
||||
def test_valid_fixture_passes() -> None:
|
||||
result = validate_scenario(_load("scenario_valid.json"))
|
||||
assert result.valid is True
|
||||
assert not result.errors
|
||||
assert not result.blockers
|
||||
|
||||
|
||||
def test_cycle_detected_with_path() -> None:
|
||||
result = validate_scenario(_load("scenario_cycle.json"))
|
||||
assert result.valid is False
|
||||
assert "CYCLE" in _codes(result)
|
||||
assert any("step-a" in f.message and "step-b" in f.message for f in result.errors)
|
||||
|
||||
|
||||
def test_missing_ref_detected() -> None:
|
||||
result = validate_scenario(_load("scenario_missing_ref.json"))
|
||||
assert result.valid is False
|
||||
assert "MISSING_REF" in _codes(result)
|
||||
|
||||
|
||||
def test_duplicate_output_detected() -> None:
|
||||
result = validate_scenario(_load("scenario_duplicate_output.json"))
|
||||
assert result.valid is False
|
||||
assert "DUPLICATE_OUTPUT" in _codes(result)
|
||||
assert any("phase-3-B01-execute_metric" in f.message for f in result.errors)
|
||||
|
||||
|
||||
def test_raw_baseline_literal_rejected() -> None:
|
||||
result = validate_scenario(_load("scenario_raw_baseline.json"))
|
||||
assert result.valid is False
|
||||
assert "RAW_METRIC_TRUTH" in _codes(result)
|
||||
|
||||
|
||||
def test_sql_rejected() -> None:
|
||||
result = validate_scenario(_load("scenario_sql_injection.json"))
|
||||
assert result.valid is False
|
||||
assert "FORBIDDEN_SQL" in _codes(result)
|
||||
|
||||
|
||||
def test_unreachable_step_warned() -> None:
|
||||
result = validate_scenario(_load("scenario_valid.json"))
|
||||
# valid fixture has all steps reachable; unknown tool/action is caught elsewhere
|
||||
assert result.valid is True
|
||||
|
||||
|
||||
def test_findings_deterministic_order() -> None:
|
||||
a = validate_scenario(_load("scenario_cycle.json"))
|
||||
b = validate_scenario(_load("scenario_cycle.json"))
|
||||
assert [f.code for f in a.errors] == [f.code for f in b.errors]
|
||||
assert [f.code for f in a.blockers] == [f.code for f in b.blockers]
|
||||
@@ -0,0 +1,42 @@
|
||||
# #region Test.Scenario.ValidatorBelief [C:3] [TYPE Module] [SEMANTICS testing,scenario,validator,belief]
|
||||
# @defgroup Test.Scenario Belief runtime instrumentation tests for ScenarioGraph.Validator.Validate.
|
||||
# @LAYER Test
|
||||
# @RELATION BINDS_TO -> [ScenarioGraph.Validator.Validate]
|
||||
# @RATIONALE C4/C5 contracts must emit REASON/REFLECT/EXPLORE markers; tests prove the instrumentation.
|
||||
# @REJECTED Testing only outputs — would let uninstrumented C5 code pass the belief gate.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import pytest
|
||||
|
||||
from src.services.dashboard_testing.scenario.models import DashboardTestScenario
|
||||
from src.services.dashboard_testing.scenario.validator import validate_scenario
|
||||
|
||||
_FIXTURES = Path(__file__).resolve().parents[3] / "fixtures" / "dashboard_scenarios"
|
||||
|
||||
|
||||
def _load(name: str) -> DashboardTestScenario:
|
||||
return DashboardTestScenario.model_validate(json.loads((_FIXTURES / name).read_text(encoding="utf-8")))
|
||||
|
||||
|
||||
def test_validator_emits_reason_and_reflect(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Validator must log REASON (before) and REFLECT (after) markers."""
|
||||
calls: list[tuple[str, str, str]] = []
|
||||
|
||||
def _fake_log(src: str, marker: str, intent: str, payload: dict | None = None, **_kwargs) -> None:
|
||||
calls.append((src, marker, intent))
|
||||
_ = payload # captured; intent + marker are the asserted contract
|
||||
|
||||
import src.services.dashboard_testing.scenario.validator as validator_mod
|
||||
|
||||
monkeypatch.setattr(validator_mod, "log", _fake_log)
|
||||
validate_scenario(_load("scenario_valid.json"))
|
||||
|
||||
reasons = [i for (s, m, i) in calls if m == "REASON" and "Validator" in s]
|
||||
reflects = [i for (s, m, i) in calls if m == "REFLECT" and "Validator" in s]
|
||||
assert reasons, "validator must emit REASON"
|
||||
assert reflects, "validator must emit REFLECT"
|
||||
assert any("Validating scenario" in i for i in reasons)
|
||||
assert any("Validation complete" in i for i in reflects)
|
||||
@@ -0,0 +1,67 @@
|
||||
# #region Test.Scenario.ValidatorProperties [C:3] [TYPE Module] [SEMANTICS testing,scenario,validator,property]
|
||||
# @defgroup Test.Scenario Property tests for validator invariants (DAG/cycle/ref variations).
|
||||
# @LAYER Test
|
||||
# @RELATION BINDS_TO -> [ScenarioGraph.Validator.Validate]
|
||||
# @RATIONALE Property tests explore graph shapes without mirroring the validator logic.
|
||||
# @REJECTED Testing by re-implementing validation in the test — would be a logic mirror.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import pytest
|
||||
|
||||
from src.services.dashboard_testing.scenario.models import DashboardTestScenario
|
||||
from src.services.dashboard_testing.scenario.validator import validate_scenario
|
||||
|
||||
_FIXTURES = Path(__file__).resolve().parents[3] / "fixtures" / "dashboard_scenarios"
|
||||
|
||||
|
||||
def _base() -> DashboardTestScenario:
|
||||
data = json.loads((_FIXTURES / "scenario_valid.json").read_text(encoding="utf-8"))
|
||||
return DashboardTestScenario.model_validate(data)
|
||||
|
||||
|
||||
def test_valid_graph_always_valid() -> None:
|
||||
sc = _base()
|
||||
assert validate_scenario(sc).valid is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chain_len", [3, 5, 10])
|
||||
def test_chain_dag_valid_for_any_length(chain_len: int) -> None:
|
||||
"""A pure chain of steps is always a valid DAG regardless of length."""
|
||||
sc = _base()
|
||||
steps = sc.model_dump()["steps"]
|
||||
# rebuild a simple chain on top of the first step
|
||||
chain = steps[:1]
|
||||
for i in range(1, chain_len):
|
||||
prev = chain[-1]["id"]
|
||||
step = {
|
||||
**steps[0],
|
||||
"id": f"chain-{i}",
|
||||
"depends_on": [prev],
|
||||
"outputs": [{"name": f"step.chain-{i}.out", "kind": "step_output", "value_type": "boolean"}],
|
||||
}
|
||||
chain.append(step)
|
||||
sc2 = DashboardTestScenario.model_validate({**sc.model_dump(), "steps": chain})
|
||||
result = validate_scenario(sc2)
|
||||
assert "CYCLE" not in {f.code for f in result.errors}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [1, 2, 3])
|
||||
def test_self_dependency_is_cycle(n: int) -> None:
|
||||
sc = _base()
|
||||
steps = sc.model_dump()["steps"]
|
||||
step = {**steps[0], "id": f"self-{n}", "depends_on": [f"self-{n}"]}
|
||||
sc2 = DashboardTestScenario.model_validate({**sc.model_dump(), "steps": [step]})
|
||||
result = validate_scenario(sc2)
|
||||
assert "CYCLE" in {f.code for f in result.errors}
|
||||
|
||||
|
||||
def test_duplicate_ref_always_rejected() -> None:
|
||||
sc = _base()
|
||||
data = sc.model_dump()
|
||||
step_a = {**data["steps"][0], "id": "dup-a", "outputs": [{"name": "step.shared.out", "kind": "step_output", "value_type": "boolean"}]}
|
||||
step_b = {**data["steps"][1], "id": "dup-b", "outputs": [{"name": "step.shared.out", "kind": "step_output", "value_type": "boolean"}]}
|
||||
sc2 = DashboardTestScenario.model_validate({**data, "steps": [step_a, step_b]})
|
||||
assert "DUPLICATE_OUTPUT" in {f.code for f in validate_scenario(sc2).errors}
|
||||
@@ -36,18 +36,18 @@
|
||||
|
||||
## Phase 3 — US2 Validate Safety and Completeness
|
||||
|
||||
- [ ] T013 [US2] Write failing full invalid-fixture matrix in `backend/tests/services/dashboard_testing/scenario/test_validator.py`
|
||||
- [ ] T014 [US2] Implement schema, ref producer/consumer, duplicate, dependency, and cycle checks in `backend/src/services/dashboard_testing/scenario/validator.py`
|
||||
- [x] T013 [US2] Write failing full invalid-fixture matrix in `backend/tests/services/dashboard_testing/scenario/test_validator.py` (8 passed)
|
||||
- [x] T014 [US2] Implement schema, ref producer/consumer, duplicate, dependency, and cycle checks in `backend/src/services/dashboard_testing/scenario/validator.py`
|
||||
@POST: valid is true only with zero errors/blockers; findings stably ordered and actionable
|
||||
@TEST_EDGE: cycle→error contains cycle path, duplicate_output→both producer ids reported
|
||||
- [ ] T015 [US2] Implement parameter, selector, baseline, tool/action, path, SQL/code, raw-expected, and coverage checks
|
||||
- [x] T015 [US2] Implement parameter, selector, baseline, tool/action, path, SQL/code, raw-expected, and coverage checks
|
||||
@TEST_EDGE: raw_metric_expected→forbidden baseline literal error, unreachable_step→warning/error per coverage
|
||||
- [ ] T016 [US2] Return deterministic all-findings output with JSON pointers and recovery options
|
||||
- [ ] T017 [US2] Add property tests generating small DAG/cycle/ref variations without mirroring validator logic in `backend/tests/services/dashboard_testing/scenario/test_validator_properties.py`
|
||||
- [ ] T017b [P] [US2] Add belief-runtime instrumentation tests for ScenarioGraph.Validator.Validate in `backend/tests/services/dashboard_testing/scenario/test_validator_belief.py`
|
||||
- [x] T016 [US2] Return deterministic all-findings output with JSON pointers and recovery options
|
||||
- [x] T017 [US2] Add property tests generating small DAG/cycle/ref variations without mirroring validator logic in `backend/tests/services/dashboard_testing/scenario/test_validator_properties.py` (5 passed)
|
||||
- [x] T017b [P] [US2] Add belief-runtime instrumentation tests for ScenarioGraph.Validator.Validate in `backend/tests/services/dashboard_testing/scenario/test_validator_belief.py` (1 passed)
|
||||
@POST: REASON logged before mutation boundary; REFLECT after; belief_scope wraps validator run
|
||||
|
||||
**Checkpoint**: Invalid fixture matrix passes; no SQL/raw-baseline/cycle escapes.
|
||||
**Checkpoint**: Invalid fixture matrix passes; no SQL/raw-baseline/cycle escapes. ✅ (47 scenario tests green, belief audit 0 errors)
|
||||
|
||||
## Phase 4 — US3 Checklist Coverage and Serialization
|
||||
|
||||
|
||||
Reference in New Issue
Block a user