feat(scenario): live baseline pin via published Gitea catalog — publisher POST/PUT contract, run.sh key-wipe fix, T045/T046 closed

This commit is contained in:
2026-09-11 14:07:20 +03:00
parent 1411c03f9b
commit 7d3fb8a770
7 changed files with 418 additions and 21 deletions

View File

@@ -25,6 +25,7 @@ from typing import Any
from urllib.parse import quote
import httpx
from dotenv import load_dotenv
REQUIRED_PIN_KEYS = ("baseline_set_id", "baseline_set_version", "release_id", "baseline_family", "catalog_digest")
DEFAULT_PATH_TEMPLATE = "catalogs/{baseline_set_id}/v{version}.json"
@@ -42,7 +43,9 @@ class PublishError(RuntimeError):
# #region Tooling.PublishCatalog.Validate [C:3] [TYPE Function] [SEMANTICS catalog,validate,fail-closed]
# @BRIEF Parse and shape-check catalog bytes before any network I/O.
# @POST Returns the parsed catalog dict; raises PublishError(PUBLISH_CATALOG_INVALID) on any
# malformed JSON, non-object payload, or missing/shape-invalid baseline_pin.
# malformed JSON, non-object payload, or missing/shape-invalid pin identity. Two deployment
# shapes are lawful: the 044 contract-refresh artifact (identity nested under `baseline_pin`)
# and the 037 published envelope (identity at top level, consumed by the 044 resolver).
def validate_catalog(raw: bytes) -> dict[str, Any]:
try:
payload = json.loads(raw.decode("utf-8"))
@@ -52,7 +55,7 @@ def validate_catalog(raw: bytes) -> dict[str, Any]:
raise PublishError("PUBLISH_CATALOG_INVALID", "catalog payload must be a JSON object")
pin = payload.get("baseline_pin")
if not isinstance(pin, dict):
raise PublishError("PUBLISH_CATALOG_INVALID", "catalog is missing a baseline_pin object")
pin = {key: payload.get(key) for key in REQUIRED_PIN_KEYS}
missing = [key for key in REQUIRED_PIN_KEYS if not isinstance(pin.get(key), str) or not pin.get(key)]
if missing:
raise PublishError("PUBLISH_CATALOG_INVALID", f"baseline_pin missing keys: {', '.join(missing)}")
@@ -62,10 +65,21 @@ def validate_catalog(raw: bytes) -> dict[str, Any]:
# #region Tooling.PublishCatalog.Path [C:2] [TYPE Function] [SEMANTICS catalog,path,template]
# @BRIEF Render the repository path from the template and pin identity.
# @POST Accepts every template alias in use across the deployment contract
# ({baseline_set_id}/{version} publisher-side, {baseline_set}/{baseline_set_version} loader-side)
# so one operator template works for both; unknown placeholders are a typed invalid-catalog error.
def catalog_path(baseline_set_id: str, version: str, template: str | None = None) -> str:
rendered = (template or os.environ.get("PUBLISHED_CATALOG_PATH_TEMPLATE") or DEFAULT_PATH_TEMPLATE).format(
baseline_set_id=quote(baseline_set_id, safe=""), version=quote(version, safe=""),
)
selected = template or os.environ.get("PUBLISHED_CATALOG_PATH_TEMPLATE") or DEFAULT_PATH_TEMPLATE
kwargs = {
"baseline_set_id": quote(baseline_set_id, safe=""),
"version": quote(version, safe=""),
"baseline_set": quote(baseline_set_id, safe=""),
"baseline_set_version": quote(version, safe=""),
}
try:
rendered = selected.format(**kwargs)
except (KeyError, IndexError) as exc:
raise PublishError("PUBLISH_CATALOG_PATH_INVALID", f"path template placeholder unknown: {exc}") from exc
return rendered.lstrip("/")
# #endregion Tooling.PublishCatalog.Path
@@ -102,8 +116,10 @@ def fetch_existing_sha(client: Any, target: GiteaTarget) -> str | None:
# #region Tooling.PublishCatalog.Put [C:3] [TYPE Function] [SEMANTICS gitea,contents,publish,put]
# @ingroup Tooling
# @BRIEF PUT the catalog bytes as a create (no sha) or update (with sha) commit.
# @BRIEF Commit the catalog bytes: POST creates a new file, PUT (with sha) updates an existing one.
# @POST Returns the Gitea commit metadata on 200/201; typed auth/conflict/unavailable errors otherwise.
# @RATIONALE The Gitea contents API requires sha for every PUT (update) and rejects PUT on a missing
# file with 422 "[SHA]: Required" — creation must go through POST (verified live 2026-09-11).
def publish_catalog_bytes(client: Any, target: GiteaTarget, raw: bytes, message: str) -> dict[str, Any]:
existing_sha = fetch_existing_sha(client, target)
body: dict[str, Any] = {
@@ -113,13 +129,15 @@ def publish_catalog_bytes(client: Any, target: GiteaTarget, raw: bytes, message:
}
if existing_sha is not None:
body["sha"] = existing_sha
response = client.put(target.contents_url(target.path), json=body)
response = client.put(target.contents_url(target.path), json=body)
else:
response = client.post(target.contents_url(target.path), json=body)
if response.status_code in (401, 403):
raise PublishError("PUBLISH_AUTH_FAILED", f"Gitea rejected the token (HTTP {response.status_code})")
if response.status_code == 409:
raise PublishError("PUBLISH_CONFLICT", "the catalog file changed concurrently; re-run to update the new sha")
if response.status_code not in (200, 201):
raise PublishError("PUBLISH_UNAVAILABLE", f"unexpected PUT status {response.status_code}: {response.text[:200]}")
raise PublishError("PUBLISH_UNAVAILABLE", f"unexpected write status {response.status_code}: {response.text[:200]}")
payload = response.json()
return payload if isinstance(payload, dict) else {}
# #endregion Tooling.PublishCatalog.Put
@@ -140,11 +158,13 @@ def main(argv: list[str] | None = None) -> int:
base_url = os.environ.get("PUBLISHED_CATALOG_GITEA_URL", "").rstrip("/")
token = os.environ.get("PUBLISHED_CATALOG_GITEA_TOKEN", "")
repo = os.environ.get("PUBLISHED_CATALOG_GITEA_REPO", "")
ref = os.environ.get("PUBLISHED_CATALOG_GITEA_REF", "main")
# Canonical env keys are the loader's (PUBLISHED_CATALOG_REPO/REF, commit bbbd4ccf); the
# GITEA_-prefixed spellings remain accepted so one deployment config feeds both surfaces.
repo = (os.environ.get("PUBLISHED_CATALOG_REPO") or os.environ.get("PUBLISHED_CATALOG_GITEA_REPO") or "").strip().strip("/")
ref = (os.environ.get("PUBLISHED_CATALOG_REF") or os.environ.get("PUBLISHED_CATALOG_GITEA_REF") or "").strip() or "master"
missing = [name for name, value in (
("PUBLISHED_CATALOG_GITEA_URL", base_url), ("PUBLISHED_CATALOG_GITEA_TOKEN", token),
("PUBLISHED_CATALOG_GITEA_REPO", repo),
("PUBLISHED_CATALOG_REPO", repo),
) if not value]
if missing:
print(json.dumps({"result": "failed", "code": "PUBLISH_CONFIG_MISSING",
@@ -155,7 +175,9 @@ def main(argv: list[str] | None = None) -> int:
with open(args.catalog, "rb") as handle:
raw = handle.read()
catalog = validate_catalog(raw)
pin = catalog["baseline_pin"]
pin = catalog.get("baseline_pin") if isinstance(catalog.get("baseline_pin"), dict) else {
key: catalog.get(key) for key in REQUIRED_PIN_KEYS
}
target = GiteaTarget(
base_url=base_url, repo=repo, ref=ref, token=token,
path=catalog_path(args.baseline_set, args.version),
@@ -182,6 +204,9 @@ def main(argv: list[str] | None = None) -> int:
if __name__ == "__main__":
# CLI mode loads backend/.env (the deployment secret boundary); module imports stay env-neutral
# so the offline test-suite never picks up deployment secrets.
load_dotenv(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), ".env"))
sys.exit(main())
# #endregion Tooling.PublishCatalog.Main
# #endregion Tooling.PublishCatalog

View File

@@ -48,11 +48,13 @@ class _FakeResponse:
class _FakeClient:
def __init__(self, get_response, put_response) -> None:
def __init__(self, get_response, put_response, post_response=None) -> None:
self.get_response = get_response
self.put_response = put_response
self.post_response = post_response or put_response
self.gets: list[tuple[str, dict]] = []
self.puts: list[tuple[str, dict]] = []
self.posts: list[tuple[str, dict]] = []
def get(self, url, params=None):
self.gets.append((url, params or {}))
@@ -62,6 +64,10 @@ class _FakeClient:
self.puts.append((url, json or {}))
return self.put_response
def post(self, url, json=None):
self.posts.append((url, json or {}))
return self.post_response
def __enter__(self):
return self
@@ -91,15 +97,20 @@ def test_catalog_path_renders_template_and_default():
# #region Test.Tooling.PublishCatalog.Publish [C:2] [TYPE Function]
def test_publish_creates_when_absent_and_updates_with_sha():
create_client = _FakeClient(_FakeResponse(404), _FakeResponse(201, {"content": {"path": "x"}}))
# Gitea contract: create is POST (no sha), update is PUT (sha required) — a PUT on a missing
# file is rejected 422 "[SHA]: Required" (verified live 2026-09-11).
create_client = _FakeClient(_FakeResponse(404), _FakeResponse(422, {"message": "[SHA]: Required"}), _FakeResponse(201, {"content": {"path": "x"}}))
assert fetch_existing_sha(create_client, _TARGET) is None
publish_catalog_bytes(create_client, _TARGET, _RAW, "create")
assert "sha" not in create_client.puts[0][1]
assert base64.b64decode(create_client.puts[0][1]["content"]) == _RAW
assert not create_client.puts
assert len(create_client.posts) == 1
assert "sha" not in create_client.posts[0][1]
assert base64.b64decode(create_client.posts[0][1]["content"]) == _RAW
update_client = _FakeClient(_FakeResponse(200, {"sha": "existing-sha"}), _FakeResponse(200, {}))
assert fetch_existing_sha(update_client, _TARGET) == "existing-sha"
publish_catalog_bytes(update_client, _TARGET, _RAW, "update")
assert not update_client.posts
assert update_client.puts[0][1]["sha"] == "existing-sha"
@@ -136,6 +147,7 @@ def test_main_missing_config_exits_2_without_network(monkeypatch, tmp_path):
catalog_file.write_bytes(_RAW)
monkeypatch.delenv("PUBLISHED_CATALOG_GITEA_URL", raising=False)
monkeypatch.delenv("PUBLISHED_CATALOG_GITEA_TOKEN", raising=False)
monkeypatch.delenv("PUBLISHED_CATALOG_REPO", raising=False)
monkeypatch.delenv("PUBLISHED_CATALOG_GITEA_REPO", raising=False)
assert main(["--catalog", str(catalog_file), "--baseline-set", "ss-prod-visual", "--version", "1"]) == 2
@@ -147,7 +159,7 @@ def test_main_publishes_end_to_end_with_fake_client(monkeypatch, tmp_path):
catalog_file.write_bytes(_RAW)
monkeypatch.setenv("PUBLISHED_CATALOG_GITEA_URL", "http://gitea.test")
monkeypatch.setenv("PUBLISHED_CATALOG_GITEA_TOKEN", "token-value")
monkeypatch.setenv("PUBLISHED_CATALOG_GITEA_REPO", "o/r")
monkeypatch.setenv("PUBLISHED_CATALOG_REPO", "o/r")
seen: dict = {}
class _InstantClient(_FakeClient):
@@ -170,7 +182,7 @@ def test_main_types_transport_errors(monkeypatch, tmp_path, capsys):
catalog_file.write_bytes(_RAW)
monkeypatch.setenv("PUBLISHED_CATALOG_GITEA_URL", "http://gitea.test")
monkeypatch.setenv("PUBLISHED_CATALOG_GITEA_TOKEN", "token-value")
monkeypatch.setenv("PUBLISHED_CATALOG_GITEA_REPO", "o/r")
monkeypatch.setenv("PUBLISHED_CATALOG_REPO", "o/r")
class _BoomClient(_FakeClient):
def __init__(self, **kwargs) -> None:

View File

@@ -0,0 +1,69 @@
# Live canary v4 — published-catalog baseline pin (T045/T046-pin) — 2026-09-11
## Objective
Close the last PAT-blocked track: publish the 044 catalog snapshot to Gitea, prove the complete
baseline pin end-to-end on the live stand (`runner_plan.baseline_pin` + `AgentEvaluation.baseline_pin`),
and run the publish-failure canary (050 T045).
Client: `specs/044-dashboard-scenario-execution/prototype/live_canary_v4_baseline_pin.py`
(REST pattern of canary v2 + envelope publication via the `Tooling.PublishCatalog` functions).
## Published envelope
- File: `catalogs/ss-prod-visual/v1.json` in `busya/ss-tools` (Gitea `gitea.bebesh.ru`, branch `master`).
- Envelope shape: the resolver-canonical published envelope — `baseline_set_id`, `baseline_set_version`,
`release_id`, `baseline_family`, `catalog_digest` at top level + `catalog_revision`
(`publication.state=published`, 2 `entry_revisions`). The 044 contract-refresh fixture nests the
identity under `baseline_pin`; the publisher accepts both shapes, the canary publishes the hoisted
envelope the 044 resolver consumes.
- Commits: create `e41d6ad2` (POST), update-sha proof `3bb2c63e` (PUT with stored sha),
final envelope `4d5b7e4c` — all three flows exercised live.
## Publish-failure canary (T045)
Bogus-token run against the real Gitea: `{"result":"failed","code":"PUBLISH_AUTH_FAILED",
"detail":"Gitea rejected the token (HTTP 401)"}`, exit 1 — typed fail-closed, no partial state.
## Baseline-pinned run
Run **`ace916a0-84a8-4f15-befe-860c1fa964ac`**, scenario `live-canary-v4-1fe64a17`.
| Proof | Result |
|---|---|
| Start with explicit selector (`ss-prod-visual` / `1`), graph declares `compare_to_baseline` | `resolve_baseline_pin` resolved the pin **from the published bytes** (`PUBLISHED_CATALOG_*` → raw content API) — no working-tree YAML, no client bytes |
| `runner_plan.baseline_pin` | full pin (`catalog_digest=aaaa…`, release `1111…`, 2 entries) |
| `AgentEvaluation.baseline_pin` | **identical to the plan pin** (`pin match: true`) — the walker stamping (commit `792bb125`) proven live |
| `open-dashboard` (browser) | passed, 1 artifact ref |
| `evaluate-visual` (multimodal judge) | passed, 1 artifact ref, verdict `pass` |
| terminal | `inconclusive` — honest: the `baseline-semantic` policy marks every comparison without a bound evaluation as typed `EVALUATION_UNAVAILABLE` (capture/compare steps carry no evaluation by graph design) |
## Production defects found and fixed this packet
1. **Gitea contents contract**: create is `POST`, update is `PUT` (sha required) — a PUT on a missing
file returns `422 "[SHA]: Required"`. `publish_catalog_bytes` now routes create→POST / update→PUT
(pinned by `tests/scripts/test_publish_catalog.py`).
2. **run.sh wiped deployment keys on every restart**: `ensure_mcp_oauth_key` (a) validated the stored
key with the system `python3` (no `cryptography` → always "invalid" → regenerate) and (b) its
rewrite helper swallowed every line after `MCP_JWT_PRIVATE_KEY=` until an `-----END PRIVATE KEY-----`
marker that a single-line escaped key never emits — deleting anything appended below it.
Fix: venv-python validation fallback + single-line in-place replacement (`run.sh`).
3. **Binding principal/surface mismatch (operational)**: REST starts act as `str(user.id)`, MCP starts
as the JWT subject — one binding principal cannot serve both. Resolved by registering a second
binding `ss-prod-d11-rest-001` (principal = sha256(user id)) through the D5 admin API; the MCP
binding `ss-prod-d11-live-001` (principal = sha256("admin")) stays for MCP starts.
## Statuses
- **050 T045**: CLOSED — publish success + update-sha flow + typed 401 canary + offline typed tests +
consume-path fail-closed (`bbbd4ccf`) + every prerequisite externally reachable (T029m CLOSED).
- **050 T046**: baseline-pin part CLOSED by this trace (complete pin, no raw repair); the row keeps
its frontend-controls evidence from 2026-09-10.
- **044 quickstart live-pin**: CLOSED (was "blocked on a valid Gitea PAT").
## Remaining
- `Expected.threshold` (038 schema authority) — separate architect packet.
- Structural curation (8 oversized contracts, `verification_service.py` 407, 50 naked defs) — separate
curator packet.
- Recommendation: rotate the Gitea PAT (it transited the chat) and keep it only in `backend/.env`.

19
run.sh
View File

@@ -322,6 +322,20 @@ raise SystemExit(0 if isinstance(key, RSAPrivateKey) and key.key_size >= 2048 el
return
fi
if [ -n "$key" ] && MCP_JWT_PRIVATE_KEY="$key" "$PROJECT_ROOT/backend/.venv/bin/python" -c '
import os
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey
key = serialization.load_pem_private_key(os.environ["MCP_JWT_PRIVATE_KEY"].replace("\\n", "\n").encode(), password=None)
raise SystemExit(0 if isinstance(key, RSAPrivateKey) and key.key_size >= 2048 else 1)
' 2>/dev/null; then
# The system python3 often lacks `cryptography`; the backend venv always has it. Without
# this fallback a perfectly valid stored key was regenerated on EVERY start (2026-09-11).
export MCP_JWT_PRIVATE_KEY="$key"
echo "MCP OAuth key preflight: existing MCP_JWT_PRIVATE_KEY reused from $ENV_FILE (venv python)"
return
fi
if ! command -v openssl >/dev/null 2>&1; then
echo "Error: openssl is required to generate MCP_JWT_PRIVATE_KEY for local development." >&2
exit 1
@@ -346,6 +360,11 @@ result: list[str] = []
skipping_pem = False
for line in source.splitlines():
if line.startswith("MCP_JWT_PRIVATE_KEY="):
if "-----END PRIVATE KEY-----" in line:
# Single-line escaped assignment: replace in place; never swallow the lines after it
# (deployment keys appended below this line must survive the rewrite, 2026-09-11).
result.append(f"MCP_JWT_PRIVATE_KEY={encoded_key}")
continue
skipping_pem = True
continue
if skipping_pem:

View File

@@ -0,0 +1,272 @@
"""Live canary v4 — published-catalog baseline pin end-to-end (T045/T046-pin evidence).
Extends the proven canary v2 pattern with the D7 track:
1. The graph declares a `compare_to_baseline` assertion step, so the start path requires a pin.
2. The start request carries the explicit selector (`baseline_set`/`baseline_set_version`); the pin
is resolved from the PUBLISHED catalog bytes in Gitea (PUBLISHED_CATALOG_* env) — never client bytes.
3. The walker stamps the resolved pin into the persisted `AgentEvaluation.baseline_pin`
(ScenarioExecution.BaselineResolver.StampEvaluation) — the live proof of server-owned identity.
Expected honest outcomes: `open-dashboard`/`capture-evidence` pass live; `compare-to-baseline` is a
deterministic dashboard-identity assertion (the published fixture is a validation snapshot, so no
image comparison is claimed); `evaluate-visual` runs the multimodal judge.
"""
import json
import os
import sys
import time
import uuid
from pathlib import Path
from dotenv import load_dotenv
_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
load_dotenv(os.path.join(_REPO_ROOT, "backend", ".env"))
sys.path.insert(0, os.path.join(_REPO_ROOT, "backend"))
import httpx # noqa: E402
from src.core.database import SessionLocal # noqa: E402
from src.models.scenario_evaluation import AgentEvaluation # noqa: E402
from src.models.scenario_registry import ScenarioRegistryEntry, ScenarioRevision # noqa: E402
from src.models.scenario_run import ScenarioRun, ScenarioStepRun # noqa: E402
from src.scripts.publish_catalog import ( # noqa: E402
GiteaTarget,
catalog_path,
publish_catalog_bytes,
validate_catalog,
)
from src.services.dashboard_testing.scenario.templates import ( # noqa: E402
ACTION_REGISTRY_VERSION,
action_registry_fingerprint,
resolve_action_descriptor,
)
BASE = os.environ.get("SS_REPLAY_BASE", "http://127.0.0.1:8000")
ENVIRONMENT_ID = "ss-prod"
DASHBOARD_ID = 11
ACTOR = "admin"
BASELINE_SET = "ss-prod-visual"
BASELINE_SET_VERSION = "1"
LLM_PROVIDER_ID = "a13fcc27-03d3-42a7-afb4-b4d77e31e805"
LLM_MODEL = "qwen3.8-flash"
POLL_TIMEOUT_SECONDS = 300
_TERMINAL_STATUSES = frozenset({"passed", "failed", "blocked", "inconclusive", "cancelled"})
def descriptor(action: str, tool: str) -> dict:
return resolve_action_descriptor(
tool=tool, action=action,
registry_version=ACTION_REGISTRY_VERSION,
registry_hash=action_registry_fingerprint(),
).snapshot()
# #endregion ScenarioExecution.LiveCanaryV4.Env
# #region ScenarioExecution.LiveCanaryV4.Seed [C:4] [TYPE Function] [SEMANTICS scenario,baseline,pin,seed]
# @ingroup ScenarioExecution
# @BRIEF Seed the baseline-pinned canary graph: open → capture → compare_to_baseline → evaluate.
# @POST Returns (scenario_id, revision_id); the compare step makes the start path require a pin.
def seed_scenario() -> tuple[str, str]:
scenario_id = f"live-canary-v4-{uuid.uuid4().hex[:8]}"
revision_id = f"rev-{uuid.uuid4().hex[:12]}"
spec = {
"schema_version": 1,
"spec_id": f"spec-{uuid.uuid4().hex[:8]}",
"provider_id": LLM_PROVIDER_ID,
"provider_version": "1",
"model_id": LLM_MODEL,
"model_version": "2026-09",
"prompt_template_id": "agent-evaluation-prompt",
"prompt_template_version": "1.0.0",
"prompt_template_hash": "b" * 64,
"evidence_refs": ["capture-evidence"],
"comparison_refs": [str(uuid.uuid4())],
"output_schema": "agent-evaluation.schema.json",
"decision_policy": {"policy_id": "baseline-semantic", "version": "1.0.0"},
"limits": {"timeout_ms": 60000, "max_images": 8, "max_input_tokens": 32000,
"max_output_tokens": 2000, "max_cost": "1.00", "currency": "USD"},
"trust_policy_hash": "c" * 64,
"criteria": [
{"criterion_id": "crit-dashboard-visible", "criterion_kind": "semantic",
"description": "Sales Dashboard rendered with visible charts and no error banner",
"comparison_id": None},
],
}
graph = {
"action_registry_version": ACTION_REGISTRY_VERSION,
"action_registry_hash": action_registry_fingerprint(),
"environment_ids": [ENVIRONMENT_ID],
"steps": [
{"logical_step_id": "open-dashboard", "tool": "browser", "action": "open_dashboard",
"environment_id": ENVIRONMENT_ID, "dashboard_id": DASHBOARD_ID,
"action_descriptor": descriptor("open_dashboard", "browser")},
{"logical_step_id": "capture-evidence", "tool": "screenshot", "action": "capture_screenshot",
"environment_id": ENVIRONMENT_ID, "dashboard_id": DASHBOARD_ID,
"action_descriptor": descriptor("capture_screenshot", "screenshot")},
{"logical_step_id": "compare-to-baseline", "tool": "assertion", "action": "compare_to_baseline",
"environment_id": ENVIRONMENT_ID, "dashboard_id": DASHBOARD_ID,
# Deterministic dashboard-identity assertion: the published fixture is a validation
# snapshot (placeholder digests), so no image comparison is claimed — the pinned
# linkage is proven by the resolved pin, not by this step's PASS.
"expected": {"dashboard_id": DASHBOARD_ID}, "actual": {"dashboard_id": DASHBOARD_ID},
"policy_type": "exact",
"action_descriptor": descriptor("compare_to_baseline", "assertion")},
{"logical_step_id": "evaluate-visual", "tool": "agent_evaluation", "action": "evaluate_declared_spec",
"environment_id": ENVIRONMENT_ID, "dashboard_id": DASHBOARD_ID,
"agent_evaluation_spec": spec,
"action_descriptor": descriptor("evaluate_declared_spec", "agent_evaluation")},
],
"dependencies": [
{"source": "open-dashboard", "target": "capture-evidence"},
{"source": "capture-evidence", "target": "compare-to-baseline"},
{"source": "capture-evidence", "target": "evaluate-visual"},
],
}
with SessionLocal() as db:
db.add(ScenarioRevision(
revision_id=revision_id, scenario_id=scenario_id,
content_hash=action_registry_fingerprint(), graph_snapshot=graph,
execution_template_hash="", template_version="v1", schema_version=1,
compatibility_family="default", change_summary={}, created_by=ACTOR,
activation_status="current",
))
db.add(ScenarioRegistryEntry(
scenario_id=scenario_id, scenario_key=scenario_id,
name="Live canary v4 baseline pin 2026-09-11", dashboard_id=DASHBOARD_ID,
environment_ids=[ENVIRONMENT_ID], owner_id=ACTOR, owner_username=ACTOR,
lifecycle_status="DRAFT", validation_status="valid",
current_revision_id=revision_id,
))
db.commit()
return scenario_id, revision_id
# #endregion ScenarioExecution.LiveCanaryV4.Seed
# #region ScenarioExecution.LiveCanaryV4.Publish [C:4] [TYPE Function] [SEMANTICS baseline,catalog,gitea,publish,envelope]
# @ingroup ScenarioExecution
# @BRIEF Publish the resolver-canonical envelope (identity top-level + catalog_revision) to Gitea.
# @PRE Deployment env supplies PUBLISHED_CATALOG_GITEA_URL/TOKEN and PUBLISHED_CATALOG_REPO/REF.
# @POST Returns the published commit sha; the envelope is validated before any network call and the
# update path reuses the stored sha (create=POST, update=PUT per the Gitea contents contract).
# @INVARIANT The published bytes are the resolver-canonical envelope — the fixture's baseline_pin
# identity hoisted to top-level plus its catalog_revision — so the start-path resolver
# consumes exactly what this packet published (no working-tree YAML, no client bytes).
def publish_envelope() -> dict:
fixture = json.loads((Path(_REPO_ROOT) / "specs/044-dashboard-scenario-execution/fixtures/production-contract-refresh.json").read_text(encoding="utf-8"))
pin = fixture["baseline_pin"]
envelope = {
"baseline_set_id": pin["baseline_set_id"],
"baseline_set_version": pin["baseline_set_version"],
"release_id": pin["release_id"],
"baseline_family": pin["baseline_family"],
"catalog_digest": pin["catalog_digest"],
"catalog_revision": fixture["catalog_revision"],
}
raw = json.dumps(envelope, indent=1, ensure_ascii=False).encode("utf-8")
validate_catalog(raw)
target = GiteaTarget(
base_url=os.environ["PUBLISHED_CATALOG_GITEA_URL"].rstrip("/"),
repo=os.environ["PUBLISHED_CATALOG_REPO"],
ref=os.environ.get("PUBLISHED_CATALOG_REF", "master"),
token=os.environ["PUBLISHED_CATALOG_GITEA_TOKEN"],
path=catalog_path(pin["baseline_set_id"], pin["baseline_set_version"]),
)
headers = {"Authorization": f"token {os.environ['PUBLISHED_CATALOG_GITEA_TOKEN']}"}
with httpx.Client(base_url=target.base_url, headers=headers, timeout=30.0) as client:
commit = publish_catalog_bytes(client, target, raw, "Publish 044 published-envelope snapshot ss-prod-visual v1 (canary v4)")
return {"path": target.path, "commit_sha": str((commit.get("commit") or {}).get("sha") or "")}
# #endregion ScenarioExecution.LiveCanaryV4.Publish
# #region ScenarioExecution.LiveCanaryV4.Run [C:3] [TYPE Function] [SEMANTICS scenario,baseline,pin,gate,report]
# @ingroup ScenarioExecution
# @BRIEF Start the pinned run (selector -> published catalog), approve the PROD gate, poll, verify pin.
# @POST Fails closed unless runner_plan.baseline_pin and the persisted AgentEvaluation.baseline_pin
# both carry the resolved published pin; a missing pin is a ReplayError-grade failure.
def main() -> None:
publication = publish_envelope()
print(f"[canary-v4] published: {publication['path']} commit={publication['commit_sha'][:12]}")
scenario_id, revision_id = seed_scenario()
print(f"[canary-v4] scenario={scenario_id} revision={revision_id}")
with httpx.Client(base_url=BASE, timeout=30) as client:
login = client.post("/api/auth/login", data={"username": ACTOR, "password": ACTOR})
login.raise_for_status()
client.headers["Authorization"] = f"Bearer {login.json()['access_token']}"
start = client.post("/api/scenario-runs", json={
"scenario_id": scenario_id, "revision_id": revision_id,
"environment_id": ENVIRONMENT_ID,
"baseline_set": BASELINE_SET, "baseline_set_version": BASELINE_SET_VERSION,
}, headers={"Idempotency-Key": f"canary-v4-{uuid.uuid4().hex[:12]}"})
print(f"[canary-v4] REST start: HTTP {start.status_code}")
start.raise_for_status()
run = start.json()
run_id = run["id"]
print(f"[canary-v4] run={run_id} status={run['status']} phase={run['phase']}")
if run["status"] == "pending_approval":
approve = client.post(f"/api/scenario-runs/{run_id}/approval/decision",
json={"decision": "approve", "comment": "live canary v4 baseline pin"})
print(f"[canary-v4] approve: HTTP {approve.status_code} -> {approve.json().get('status')}")
approve.raise_for_status()
deadline = time.monotonic() + POLL_TIMEOUT_SECONDS
status = "queued"
detail: dict = {}
while time.monotonic() < deadline:
detail = client.get(f"/api/scenario-runs/{run_id}").json()
status = detail["status"]
if status in _TERMINAL_STATUSES:
print(f"[canary-v4] terminal: {status}")
break
time.sleep(5)
else:
print("[canary-v4] poll timeout")
with SessionLocal() as db:
run_row = db.get(ScenarioRun, run_id)
plan_pin = (run_row.runner_plan or {}).get("baseline_pin") if run_row else None
evaluation = db.query(AgentEvaluation).filter(AgentEvaluation.scenario_run_id == run_id).first()
steps = db.query(ScenarioStepRun).filter(ScenarioStepRun.run_id == run_id).all()
report = {
"run_id": run_id,
"scenario_id": scenario_id,
"publication": publication,
"baseline_selector": {"baseline_set": BASELINE_SET, "baseline_set_version": BASELINE_SET_VERSION},
"plan_baseline_pin": plan_pin,
"evaluation_baseline_pin": evaluation.baseline_pin if evaluation else None,
"evaluation_id": evaluation.evaluation_id if evaluation else None,
"evaluation_verdict": evaluation.verdict if evaluation else None,
"final_status": status,
"final_phase": detail.get("phase"),
"error_code": detail.get("error_code"),
"steps": [
{
"logical_step_id": s.logical_step_id,
"status": s.status,
"error_code": s.error_code,
"artifact_refs_count": len(s.artifact_refs or []),
"step_outcome": {
k: v for k, v in (s.step_outcome or {}).items()
if k in ("status", "reason_code", "sha256", "verdict", "confidence",
"comparison_status", "screenshot_count")
},
}
for s in steps
],
}
print("[canary-v4] RESULT")
print(json.dumps(report, indent=1, ensure_ascii=False, default=str))
result_path = os.environ.get("SS_REPLAY_RESULT", "/tmp/kilo/live_canary_v4_result.json")
with open(result_path, "w") as handle:
json.dump(report, handle, indent=1, default=str)
# Fail-closed pin proof: the whole packet exists to prove the published-catalog pin end-to-end.
assert isinstance(plan_pin, dict) and plan_pin.get("catalog_digest"), "runner_plan pin missing"
assert evaluation is not None and evaluation.baseline_pin, "AgentEvaluation baseline_pin not stamped"
if __name__ == "__main__":
main()
# #endregion ScenarioExecution.LiveCanaryV4.Run

View File

@@ -26,7 +26,7 @@
| Visual chain | Compiler emits browser→capture→comparison→evaluation→policy (Slice E) | One-action templates; live Playwright canary PASSED 2026-09-10 (browser `open_dashboard` + screenshot capture vs ss-prod dashboard 11; comparison/evaluation steps not in the canary graph) |
| Live providers | Registered browser/screenshot + multimodal LLM | Binding `ss-prod-d11-live-001` resolved **server-side at REST start** (no binding in the request body); `/api/ready`: browser/screenshot ready, bindings 1; canary v2 live trace: `docs/reports/agentic-runtime-live-canary-v2-2026-09-10.md` |
**Open live gaps (2026-09-10, for the next agent):** ~~multimodal image attachment (prompts are text-only → visual verdicts stay inconclusive); `baseline_pin` in the persisted `AgentEvaluation` record is `{}` for baseline-backed plans; screenshot/browser evidence MIME is hardcoded (`image/jpeg`/`image/png`) and would mismatch a WebP archive~~ → all three CLOSED 2026-09-10/11 (commit `792bb125`): multimodal evidence attachment (`evaluation_images.py` + `_evaluation_messages`), server-authoritative `baseline_pin` stamping (`baseline_resolver.stamp_baseline_pin`, live canary v3 `9e6f59c0`: `verdict=pass`, visual explanation), per-ref MIME sniffing (`mime_sniff.py`). Still open: live baseline pin is blocked on a valid Gitea PAT (publisher ready: `backend/src/scripts/publish_catalog.py`).
**Open live gaps (2026-09-10, for the next agent):** ~~multimodal image attachment; `baseline_pin={}` for baseline-backed plans; hardcoded evidence MIME~~ → all three CLOSED 2026-09-10/11 (commit `792bb125`). ~~Live baseline pin blocked on a valid Gitea PAT~~ → **CLOSED 2026-09-11** (canary v4, run `ace916a0…`): the 037 envelope published to Gitea (`catalogs/ss-prod-visual/v1.json`, commit `4d5b7e4c`), selector-driven resolution from published bytes → `runner_plan.baseline_pin` == `AgentEvaluation.baseline_pin` — full trace: `docs/reports/agentic-runtime-live-canary-v4-baseline-pin-2026-09-11.md`. No open live gaps remain; residual debt is structural (oversized contracts / naked defs) and `Expected.threshold` (038 schema authority).
The former agent-driven product-UI flow (dashboard → `/agent` workspace → agent-generated scenario) is SUPERSEDED. 044 is a headless execution engine; agent interaction is external MCP (050). Product UI (045) is read-only evidence plus human approvals.

View File

@@ -93,7 +93,7 @@ Historical [x] rows above retain only their dated local/transport evidence; they
Contract: [Contract-complete public parity](contracts/modules.md).
- [ ] T044 [P0/P1/P2] REST/MCP lifecycle/read/auth errors and disabled automation validation are identical; service principal cannot decide human gate. Implement at the existing 050 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11):** offline parity tests green; live REST-only canary v2 exercised start→gate→approve→dispatch (`4eebfab3`); the live MCP-external replay (T029m CLOSED, `docs/2026-09-11-sales-prod-mcp-replay.md`) exercised the same lifecycle through `/mcp` with the identical typed start/gate/terminal outcomes — remaining: dedicated REST-vs-MCP error-shape parity fixtures for this matrix.
- [ ] T045 [P0/P1/P2] consume/publish failures return typed errors/pending state with no legacy fallback; every prerequisite is externally MCP-reachable. Implement at the existing 050 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11):** the caller-side published-catalog source returns typed `None`/fail-closed (commit `bbbd4ccf`); the publisher helper landed 2026-09-11 (`backend/src/scripts/publish_catalog.py`, pre-validate + Gitea contents PUT, typed auth/conflict/unavailable; `tests/scripts/test_publish_catalog.py` offline-green). Gitea publication path remains BLOCKED on a valid PAT. No live publish failure canary yet.
- [ ] T046 [P0/P1/P2] Fresh external-client chain preserves authoritative context and complete baseline pin without raw ORM/REST repair; no frontend agent controls/routes/requests. Implement at the existing 050 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11):** a fresh REST client chain (canary v2 `4eebfab3`) AND the fresh MCP-external chain (T029m CLOSED) both preserve the server-resolved binding and the verified authoritative context live (`context_authority=verified` at the register boundary; binding adopted server-side and exercised by live providers). The complete baseline pin is still blocked on the Gitea PAT.
- [x] T045 [P0/P1/P2] consume/publish failures return typed errors/pending state with no legacy fallback; every prerequisite is externally MCP-reachable. Implement at the existing 050 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11): CLOSED.** Consume path typed fail-closed (`bbbd4ccf`); publisher `backend/src/scripts/publish_catalog.py` with pre-validation before any network I/O and typed auth/conflict/unavailable (offline `tests/scripts/test_publish_catalog.py`); LIVE evidence (`docs/reports/agentic-runtime-live-canary-v4-baseline-pin-2026-09-11.md`): create `e41d6ad2` (POST), update-sha `3bb2c63e` (PUT), publish-failure canary with a bogus token → typed `PUBLISH_AUTH_FAILED` (HTTP 401, exit 1, no partial state); every prerequisite externally reachable (T029m CLOSED). Defects found and fixed live: Gitea create=POST/update=PUT contract; `run.sh` wiping deployment keys below the MCP_JWT line on every start (venv-python validation fallback + single-line in-place rewrite).
- [x] T046 [P0/P1/P2] Fresh external-client chain preserves authoritative context and complete baseline pin without raw ORM/REST repair; no frontend agent controls/routes/requests. Implement at the existing 050 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11): CLOSED.** Fresh REST chain (canary v2 `4eebfab3`) and fresh MCP-external chain (T029m) preserve the server-resolved binding and verified authoritative context (`context_authority=verified`); the complete baseline pin is proven live by canary v4 (run `ace916a0…`, `docs/reports/agentic-runtime-live-canary-v4-baseline-pin-2026-09-11.md`): selector-driven resolution from the PUBLISHED Gitea envelope → `runner_plan.baseline_pin` == `AgentEvaluation.baseline_pin` (walker stamping, commit `792bb125`), no raw ORM/REST repair; frontend agent controls remain absent (2026-09-10 evidence).
Frontend boundary for this package: manual CRUD/editor, human review/approval, monitoring and read-only evidence/evaluation only; all agent interaction is external MCP. No agent chat/prompt/assistant editing/proposal generation/workspace/start/handoff controls. Runtime removal is OPEN, not performed by this spec refresh. Optional approved performance baseline is outside scope.