feat(scenario): live baseline pin via published Gitea catalog — publisher POST/PUT contract, run.sh key-wipe fix, T045/T046 closed
This commit is contained in:
@@ -25,6 +25,7 @@ from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from dotenv import load_dotenv
|
||||
|
||||
REQUIRED_PIN_KEYS = ("baseline_set_id", "baseline_set_version", "release_id", "baseline_family", "catalog_digest")
|
||||
DEFAULT_PATH_TEMPLATE = "catalogs/{baseline_set_id}/v{version}.json"
|
||||
@@ -42,7 +43,9 @@ class PublishError(RuntimeError):
|
||||
# #region Tooling.PublishCatalog.Validate [C:3] [TYPE Function] [SEMANTICS catalog,validate,fail-closed]
|
||||
# @BRIEF Parse and shape-check catalog bytes before any network I/O.
|
||||
# @POST Returns the parsed catalog dict; raises PublishError(PUBLISH_CATALOG_INVALID) on any
|
||||
# malformed JSON, non-object payload, or missing/shape-invalid baseline_pin.
|
||||
# malformed JSON, non-object payload, or missing/shape-invalid pin identity. Two deployment
|
||||
# shapes are lawful: the 044 contract-refresh artifact (identity nested under `baseline_pin`)
|
||||
# and the 037 published envelope (identity at top level, consumed by the 044 resolver).
|
||||
def validate_catalog(raw: bytes) -> dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
@@ -52,7 +55,7 @@ def validate_catalog(raw: bytes) -> dict[str, Any]:
|
||||
raise PublishError("PUBLISH_CATALOG_INVALID", "catalog payload must be a JSON object")
|
||||
pin = payload.get("baseline_pin")
|
||||
if not isinstance(pin, dict):
|
||||
raise PublishError("PUBLISH_CATALOG_INVALID", "catalog is missing a baseline_pin object")
|
||||
pin = {key: payload.get(key) for key in REQUIRED_PIN_KEYS}
|
||||
missing = [key for key in REQUIRED_PIN_KEYS if not isinstance(pin.get(key), str) or not pin.get(key)]
|
||||
if missing:
|
||||
raise PublishError("PUBLISH_CATALOG_INVALID", f"baseline_pin missing keys: {', '.join(missing)}")
|
||||
@@ -62,10 +65,21 @@ def validate_catalog(raw: bytes) -> dict[str, Any]:
|
||||
|
||||
# #region Tooling.PublishCatalog.Path [C:2] [TYPE Function] [SEMANTICS catalog,path,template]
|
||||
# @BRIEF Render the repository path from the template and pin identity.
|
||||
# @POST Accepts every template alias in use across the deployment contract
|
||||
# ({baseline_set_id}/{version} publisher-side, {baseline_set}/{baseline_set_version} loader-side)
|
||||
# so one operator template works for both; unknown placeholders are a typed invalid-catalog error.
|
||||
def catalog_path(baseline_set_id: str, version: str, template: str | None = None) -> str:
|
||||
rendered = (template or os.environ.get("PUBLISHED_CATALOG_PATH_TEMPLATE") or DEFAULT_PATH_TEMPLATE).format(
|
||||
baseline_set_id=quote(baseline_set_id, safe=""), version=quote(version, safe=""),
|
||||
)
|
||||
selected = template or os.environ.get("PUBLISHED_CATALOG_PATH_TEMPLATE") or DEFAULT_PATH_TEMPLATE
|
||||
kwargs = {
|
||||
"baseline_set_id": quote(baseline_set_id, safe=""),
|
||||
"version": quote(version, safe=""),
|
||||
"baseline_set": quote(baseline_set_id, safe=""),
|
||||
"baseline_set_version": quote(version, safe=""),
|
||||
}
|
||||
try:
|
||||
rendered = selected.format(**kwargs)
|
||||
except (KeyError, IndexError) as exc:
|
||||
raise PublishError("PUBLISH_CATALOG_PATH_INVALID", f"path template placeholder unknown: {exc}") from exc
|
||||
return rendered.lstrip("/")
|
||||
# #endregion Tooling.PublishCatalog.Path
|
||||
|
||||
@@ -102,8 +116,10 @@ def fetch_existing_sha(client: Any, target: GiteaTarget) -> str | None:
|
||||
|
||||
# #region Tooling.PublishCatalog.Put [C:3] [TYPE Function] [SEMANTICS gitea,contents,publish,put]
|
||||
# @ingroup Tooling
|
||||
# @BRIEF PUT the catalog bytes as a create (no sha) or update (with sha) commit.
|
||||
# @BRIEF Commit the catalog bytes: POST creates a new file, PUT (with sha) updates an existing one.
|
||||
# @POST Returns the Gitea commit metadata on 200/201; typed auth/conflict/unavailable errors otherwise.
|
||||
# @RATIONALE The Gitea contents API requires sha for every PUT (update) and rejects PUT on a missing
|
||||
# file with 422 "[SHA]: Required" — creation must go through POST (verified live 2026-09-11).
|
||||
def publish_catalog_bytes(client: Any, target: GiteaTarget, raw: bytes, message: str) -> dict[str, Any]:
|
||||
existing_sha = fetch_existing_sha(client, target)
|
||||
body: dict[str, Any] = {
|
||||
@@ -113,13 +129,15 @@ def publish_catalog_bytes(client: Any, target: GiteaTarget, raw: bytes, message:
|
||||
}
|
||||
if existing_sha is not None:
|
||||
body["sha"] = existing_sha
|
||||
response = client.put(target.contents_url(target.path), json=body)
|
||||
response = client.put(target.contents_url(target.path), json=body)
|
||||
else:
|
||||
response = client.post(target.contents_url(target.path), json=body)
|
||||
if response.status_code in (401, 403):
|
||||
raise PublishError("PUBLISH_AUTH_FAILED", f"Gitea rejected the token (HTTP {response.status_code})")
|
||||
if response.status_code == 409:
|
||||
raise PublishError("PUBLISH_CONFLICT", "the catalog file changed concurrently; re-run to update the new sha")
|
||||
if response.status_code not in (200, 201):
|
||||
raise PublishError("PUBLISH_UNAVAILABLE", f"unexpected PUT status {response.status_code}: {response.text[:200]}")
|
||||
raise PublishError("PUBLISH_UNAVAILABLE", f"unexpected write status {response.status_code}: {response.text[:200]}")
|
||||
payload = response.json()
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
# #endregion Tooling.PublishCatalog.Put
|
||||
@@ -140,11 +158,13 @@ def main(argv: list[str] | None = None) -> int:
|
||||
|
||||
base_url = os.environ.get("PUBLISHED_CATALOG_GITEA_URL", "").rstrip("/")
|
||||
token = os.environ.get("PUBLISHED_CATALOG_GITEA_TOKEN", "")
|
||||
repo = os.environ.get("PUBLISHED_CATALOG_GITEA_REPO", "")
|
||||
ref = os.environ.get("PUBLISHED_CATALOG_GITEA_REF", "main")
|
||||
# Canonical env keys are the loader's (PUBLISHED_CATALOG_REPO/REF, commit bbbd4ccf); the
|
||||
# GITEA_-prefixed spellings remain accepted so one deployment config feeds both surfaces.
|
||||
repo = (os.environ.get("PUBLISHED_CATALOG_REPO") or os.environ.get("PUBLISHED_CATALOG_GITEA_REPO") or "").strip().strip("/")
|
||||
ref = (os.environ.get("PUBLISHED_CATALOG_REF") or os.environ.get("PUBLISHED_CATALOG_GITEA_REF") or "").strip() or "master"
|
||||
missing = [name for name, value in (
|
||||
("PUBLISHED_CATALOG_GITEA_URL", base_url), ("PUBLISHED_CATALOG_GITEA_TOKEN", token),
|
||||
("PUBLISHED_CATALOG_GITEA_REPO", repo),
|
||||
("PUBLISHED_CATALOG_REPO", repo),
|
||||
) if not value]
|
||||
if missing:
|
||||
print(json.dumps({"result": "failed", "code": "PUBLISH_CONFIG_MISSING",
|
||||
@@ -155,7 +175,9 @@ def main(argv: list[str] | None = None) -> int:
|
||||
with open(args.catalog, "rb") as handle:
|
||||
raw = handle.read()
|
||||
catalog = validate_catalog(raw)
|
||||
pin = catalog["baseline_pin"]
|
||||
pin = catalog.get("baseline_pin") if isinstance(catalog.get("baseline_pin"), dict) else {
|
||||
key: catalog.get(key) for key in REQUIRED_PIN_KEYS
|
||||
}
|
||||
target = GiteaTarget(
|
||||
base_url=base_url, repo=repo, ref=ref, token=token,
|
||||
path=catalog_path(args.baseline_set, args.version),
|
||||
@@ -182,6 +204,9 @@ def main(argv: list[str] | None = None) -> int:
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# CLI mode loads backend/.env (the deployment secret boundary); module imports stay env-neutral
|
||||
# so the offline test-suite never picks up deployment secrets.
|
||||
load_dotenv(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), ".env"))
|
||||
sys.exit(main())
|
||||
# #endregion Tooling.PublishCatalog.Main
|
||||
# #endregion Tooling.PublishCatalog
|
||||
|
||||
@@ -48,11 +48,13 @@ class _FakeResponse:
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, get_response, put_response) -> None:
|
||||
def __init__(self, get_response, put_response, post_response=None) -> None:
|
||||
self.get_response = get_response
|
||||
self.put_response = put_response
|
||||
self.post_response = post_response or put_response
|
||||
self.gets: list[tuple[str, dict]] = []
|
||||
self.puts: list[tuple[str, dict]] = []
|
||||
self.posts: list[tuple[str, dict]] = []
|
||||
|
||||
def get(self, url, params=None):
|
||||
self.gets.append((url, params or {}))
|
||||
@@ -62,6 +64,10 @@ class _FakeClient:
|
||||
self.puts.append((url, json or {}))
|
||||
return self.put_response
|
||||
|
||||
def post(self, url, json=None):
|
||||
self.posts.append((url, json or {}))
|
||||
return self.post_response
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
@@ -91,15 +97,20 @@ def test_catalog_path_renders_template_and_default():
|
||||
|
||||
# #region Test.Tooling.PublishCatalog.Publish [C:2] [TYPE Function]
|
||||
def test_publish_creates_when_absent_and_updates_with_sha():
|
||||
create_client = _FakeClient(_FakeResponse(404), _FakeResponse(201, {"content": {"path": "x"}}))
|
||||
# Gitea contract: create is POST (no sha), update is PUT (sha required) — a PUT on a missing
|
||||
# file is rejected 422 "[SHA]: Required" (verified live 2026-09-11).
|
||||
create_client = _FakeClient(_FakeResponse(404), _FakeResponse(422, {"message": "[SHA]: Required"}), _FakeResponse(201, {"content": {"path": "x"}}))
|
||||
assert fetch_existing_sha(create_client, _TARGET) is None
|
||||
publish_catalog_bytes(create_client, _TARGET, _RAW, "create")
|
||||
assert "sha" not in create_client.puts[0][1]
|
||||
assert base64.b64decode(create_client.puts[0][1]["content"]) == _RAW
|
||||
assert not create_client.puts
|
||||
assert len(create_client.posts) == 1
|
||||
assert "sha" not in create_client.posts[0][1]
|
||||
assert base64.b64decode(create_client.posts[0][1]["content"]) == _RAW
|
||||
|
||||
update_client = _FakeClient(_FakeResponse(200, {"sha": "existing-sha"}), _FakeResponse(200, {}))
|
||||
assert fetch_existing_sha(update_client, _TARGET) == "existing-sha"
|
||||
publish_catalog_bytes(update_client, _TARGET, _RAW, "update")
|
||||
assert not update_client.posts
|
||||
assert update_client.puts[0][1]["sha"] == "existing-sha"
|
||||
|
||||
|
||||
@@ -136,6 +147,7 @@ def test_main_missing_config_exits_2_without_network(monkeypatch, tmp_path):
|
||||
catalog_file.write_bytes(_RAW)
|
||||
monkeypatch.delenv("PUBLISHED_CATALOG_GITEA_URL", raising=False)
|
||||
monkeypatch.delenv("PUBLISHED_CATALOG_GITEA_TOKEN", raising=False)
|
||||
monkeypatch.delenv("PUBLISHED_CATALOG_REPO", raising=False)
|
||||
monkeypatch.delenv("PUBLISHED_CATALOG_GITEA_REPO", raising=False)
|
||||
assert main(["--catalog", str(catalog_file), "--baseline-set", "ss-prod-visual", "--version", "1"]) == 2
|
||||
|
||||
@@ -147,7 +159,7 @@ def test_main_publishes_end_to_end_with_fake_client(monkeypatch, tmp_path):
|
||||
catalog_file.write_bytes(_RAW)
|
||||
monkeypatch.setenv("PUBLISHED_CATALOG_GITEA_URL", "http://gitea.test")
|
||||
monkeypatch.setenv("PUBLISHED_CATALOG_GITEA_TOKEN", "token-value")
|
||||
monkeypatch.setenv("PUBLISHED_CATALOG_GITEA_REPO", "o/r")
|
||||
monkeypatch.setenv("PUBLISHED_CATALOG_REPO", "o/r")
|
||||
seen: dict = {}
|
||||
|
||||
class _InstantClient(_FakeClient):
|
||||
@@ -170,7 +182,7 @@ def test_main_types_transport_errors(monkeypatch, tmp_path, capsys):
|
||||
catalog_file.write_bytes(_RAW)
|
||||
monkeypatch.setenv("PUBLISHED_CATALOG_GITEA_URL", "http://gitea.test")
|
||||
monkeypatch.setenv("PUBLISHED_CATALOG_GITEA_TOKEN", "token-value")
|
||||
monkeypatch.setenv("PUBLISHED_CATALOG_GITEA_REPO", "o/r")
|
||||
monkeypatch.setenv("PUBLISHED_CATALOG_REPO", "o/r")
|
||||
|
||||
class _BoomClient(_FakeClient):
|
||||
def __init__(self, **kwargs) -> None:
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
# Live canary v4 — published-catalog baseline pin (T045/T046-pin) — 2026-09-11
|
||||
|
||||
## Objective
|
||||
|
||||
Close the last PAT-blocked track: publish the 044 catalog snapshot to Gitea, prove the complete
|
||||
baseline pin end-to-end on the live stand (`runner_plan.baseline_pin` + `AgentEvaluation.baseline_pin`),
|
||||
and run the publish-failure canary (050 T045).
|
||||
|
||||
Client: `specs/044-dashboard-scenario-execution/prototype/live_canary_v4_baseline_pin.py`
|
||||
(REST pattern of canary v2 + envelope publication via the `Tooling.PublishCatalog` functions).
|
||||
|
||||
## Published envelope
|
||||
|
||||
- File: `catalogs/ss-prod-visual/v1.json` in `busya/ss-tools` (Gitea `gitea.bebesh.ru`, branch `master`).
|
||||
- Envelope shape: the resolver-canonical published envelope — `baseline_set_id`, `baseline_set_version`,
|
||||
`release_id`, `baseline_family`, `catalog_digest` at top level + `catalog_revision`
|
||||
(`publication.state=published`, 2 `entry_revisions`). The 044 contract-refresh fixture nests the
|
||||
identity under `baseline_pin`; the publisher accepts both shapes, the canary publishes the hoisted
|
||||
envelope the 044 resolver consumes.
|
||||
- Commits: create `e41d6ad2` (POST), update-sha proof `3bb2c63e` (PUT with stored sha),
|
||||
final envelope `4d5b7e4c` — all three flows exercised live.
|
||||
|
||||
## Publish-failure canary (T045)
|
||||
|
||||
Bogus-token run against the real Gitea: `{"result":"failed","code":"PUBLISH_AUTH_FAILED",
|
||||
"detail":"Gitea rejected the token (HTTP 401)"}`, exit 1 — typed fail-closed, no partial state.
|
||||
|
||||
## Baseline-pinned run
|
||||
|
||||
Run **`ace916a0-84a8-4f15-befe-860c1fa964ac`**, scenario `live-canary-v4-1fe64a17`.
|
||||
|
||||
| Proof | Result |
|
||||
|---|---|
|
||||
| Start with explicit selector (`ss-prod-visual` / `1`), graph declares `compare_to_baseline` | `resolve_baseline_pin` resolved the pin **from the published bytes** (`PUBLISHED_CATALOG_*` → raw content API) — no working-tree YAML, no client bytes |
|
||||
| `runner_plan.baseline_pin` | full pin (`catalog_digest=aaaa…`, release `1111…`, 2 entries) |
|
||||
| `AgentEvaluation.baseline_pin` | **identical to the plan pin** (`pin match: true`) — the walker stamping (commit `792bb125`) proven live |
|
||||
| `open-dashboard` (browser) | passed, 1 artifact ref |
|
||||
| `evaluate-visual` (multimodal judge) | passed, 1 artifact ref, verdict `pass` |
|
||||
| terminal | `inconclusive` — honest: the `baseline-semantic` policy marks every comparison without a bound evaluation as typed `EVALUATION_UNAVAILABLE` (capture/compare steps carry no evaluation by graph design) |
|
||||
|
||||
## Production defects found and fixed this packet
|
||||
|
||||
1. **Gitea contents contract**: create is `POST`, update is `PUT` (sha required) — a PUT on a missing
|
||||
file returns `422 "[SHA]: Required"`. `publish_catalog_bytes` now routes create→POST / update→PUT
|
||||
(pinned by `tests/scripts/test_publish_catalog.py`).
|
||||
2. **run.sh wiped deployment keys on every restart**: `ensure_mcp_oauth_key` (a) validated the stored
|
||||
key with the system `python3` (no `cryptography` → always "invalid" → regenerate) and (b) its
|
||||
rewrite helper swallowed every line after `MCP_JWT_PRIVATE_KEY=` until an `-----END PRIVATE KEY-----`
|
||||
marker that a single-line escaped key never emits — deleting anything appended below it.
|
||||
Fix: venv-python validation fallback + single-line in-place replacement (`run.sh`).
|
||||
3. **Binding principal/surface mismatch (operational)**: REST starts act as `str(user.id)`, MCP starts
|
||||
as the JWT subject — one binding principal cannot serve both. Resolved by registering a second
|
||||
binding `ss-prod-d11-rest-001` (principal = sha256(user id)) through the D5 admin API; the MCP
|
||||
binding `ss-prod-d11-live-001` (principal = sha256("admin")) stays for MCP starts.
|
||||
|
||||
## Statuses
|
||||
|
||||
- **050 T045**: CLOSED — publish success + update-sha flow + typed 401 canary + offline typed tests +
|
||||
consume-path fail-closed (`bbbd4ccf`) + every prerequisite externally reachable (T029m CLOSED).
|
||||
- **050 T046**: baseline-pin part CLOSED by this trace (complete pin, no raw repair); the row keeps
|
||||
its frontend-controls evidence from 2026-09-10.
|
||||
- **044 quickstart live-pin**: CLOSED (was "blocked on a valid Gitea PAT").
|
||||
|
||||
## Remaining
|
||||
|
||||
- `Expected.threshold` (038 schema authority) — separate architect packet.
|
||||
- Structural curation (8 oversized contracts, `verification_service.py` 407, 50 naked defs) — separate
|
||||
curator packet.
|
||||
- Recommendation: rotate the Gitea PAT (it transited the chat) and keep it only in `backend/.env`.
|
||||
19
run.sh
19
run.sh
@@ -322,6 +322,20 @@ raise SystemExit(0 if isinstance(key, RSAPrivateKey) and key.key_size >= 2048 el
|
||||
return
|
||||
fi
|
||||
|
||||
if [ -n "$key" ] && MCP_JWT_PRIVATE_KEY="$key" "$PROJECT_ROOT/backend/.venv/bin/python" -c '
|
||||
import os
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey
|
||||
key = serialization.load_pem_private_key(os.environ["MCP_JWT_PRIVATE_KEY"].replace("\\n", "\n").encode(), password=None)
|
||||
raise SystemExit(0 if isinstance(key, RSAPrivateKey) and key.key_size >= 2048 else 1)
|
||||
' 2>/dev/null; then
|
||||
# The system python3 often lacks `cryptography`; the backend venv always has it. Without
|
||||
# this fallback a perfectly valid stored key was regenerated on EVERY start (2026-09-11).
|
||||
export MCP_JWT_PRIVATE_KEY="$key"
|
||||
echo "MCP OAuth key preflight: existing MCP_JWT_PRIVATE_KEY reused from $ENV_FILE (venv python)"
|
||||
return
|
||||
fi
|
||||
|
||||
if ! command -v openssl >/dev/null 2>&1; then
|
||||
echo "Error: openssl is required to generate MCP_JWT_PRIVATE_KEY for local development." >&2
|
||||
exit 1
|
||||
@@ -346,6 +360,11 @@ result: list[str] = []
|
||||
skipping_pem = False
|
||||
for line in source.splitlines():
|
||||
if line.startswith("MCP_JWT_PRIVATE_KEY="):
|
||||
if "-----END PRIVATE KEY-----" in line:
|
||||
# Single-line escaped assignment: replace in place; never swallow the lines after it
|
||||
# (deployment keys appended below this line must survive the rewrite, 2026-09-11).
|
||||
result.append(f"MCP_JWT_PRIVATE_KEY={encoded_key}")
|
||||
continue
|
||||
skipping_pem = True
|
||||
continue
|
||||
if skipping_pem:
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
"""Live canary v4 — published-catalog baseline pin end-to-end (T045/T046-pin evidence).
|
||||
|
||||
Extends the proven canary v2 pattern with the D7 track:
|
||||
1. The graph declares a `compare_to_baseline` assertion step, so the start path requires a pin.
|
||||
2. The start request carries the explicit selector (`baseline_set`/`baseline_set_version`); the pin
|
||||
is resolved from the PUBLISHED catalog bytes in Gitea (PUBLISHED_CATALOG_* env) — never client bytes.
|
||||
3. The walker stamps the resolved pin into the persisted `AgentEvaluation.baseline_pin`
|
||||
(ScenarioExecution.BaselineResolver.StampEvaluation) — the live proof of server-owned identity.
|
||||
|
||||
Expected honest outcomes: `open-dashboard`/`capture-evidence` pass live; `compare-to-baseline` is a
|
||||
deterministic dashboard-identity assertion (the published fixture is a validation snapshot, so no
|
||||
image comparison is claimed); `evaluate-visual` runs the multimodal judge.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
load_dotenv(os.path.join(_REPO_ROOT, "backend", ".env"))
|
||||
sys.path.insert(0, os.path.join(_REPO_ROOT, "backend"))
|
||||
|
||||
import httpx # noqa: E402
|
||||
|
||||
from src.core.database import SessionLocal # noqa: E402
|
||||
from src.models.scenario_evaluation import AgentEvaluation # noqa: E402
|
||||
from src.models.scenario_registry import ScenarioRegistryEntry, ScenarioRevision # noqa: E402
|
||||
from src.models.scenario_run import ScenarioRun, ScenarioStepRun # noqa: E402
|
||||
from src.scripts.publish_catalog import ( # noqa: E402
|
||||
GiteaTarget,
|
||||
catalog_path,
|
||||
publish_catalog_bytes,
|
||||
validate_catalog,
|
||||
)
|
||||
from src.services.dashboard_testing.scenario.templates import ( # noqa: E402
|
||||
ACTION_REGISTRY_VERSION,
|
||||
action_registry_fingerprint,
|
||||
resolve_action_descriptor,
|
||||
)
|
||||
|
||||
BASE = os.environ.get("SS_REPLAY_BASE", "http://127.0.0.1:8000")
|
||||
ENVIRONMENT_ID = "ss-prod"
|
||||
DASHBOARD_ID = 11
|
||||
ACTOR = "admin"
|
||||
BASELINE_SET = "ss-prod-visual"
|
||||
BASELINE_SET_VERSION = "1"
|
||||
LLM_PROVIDER_ID = "a13fcc27-03d3-42a7-afb4-b4d77e31e805"
|
||||
LLM_MODEL = "qwen3.8-flash"
|
||||
POLL_TIMEOUT_SECONDS = 300
|
||||
_TERMINAL_STATUSES = frozenset({"passed", "failed", "blocked", "inconclusive", "cancelled"})
|
||||
|
||||
|
||||
def descriptor(action: str, tool: str) -> dict:
|
||||
return resolve_action_descriptor(
|
||||
tool=tool, action=action,
|
||||
registry_version=ACTION_REGISTRY_VERSION,
|
||||
registry_hash=action_registry_fingerprint(),
|
||||
).snapshot()
|
||||
# #endregion ScenarioExecution.LiveCanaryV4.Env
|
||||
|
||||
|
||||
# #region ScenarioExecution.LiveCanaryV4.Seed [C:4] [TYPE Function] [SEMANTICS scenario,baseline,pin,seed]
|
||||
# @ingroup ScenarioExecution
|
||||
# @BRIEF Seed the baseline-pinned canary graph: open → capture → compare_to_baseline → evaluate.
|
||||
# @POST Returns (scenario_id, revision_id); the compare step makes the start path require a pin.
|
||||
def seed_scenario() -> tuple[str, str]:
|
||||
scenario_id = f"live-canary-v4-{uuid.uuid4().hex[:8]}"
|
||||
revision_id = f"rev-{uuid.uuid4().hex[:12]}"
|
||||
spec = {
|
||||
"schema_version": 1,
|
||||
"spec_id": f"spec-{uuid.uuid4().hex[:8]}",
|
||||
"provider_id": LLM_PROVIDER_ID,
|
||||
"provider_version": "1",
|
||||
"model_id": LLM_MODEL,
|
||||
"model_version": "2026-09",
|
||||
"prompt_template_id": "agent-evaluation-prompt",
|
||||
"prompt_template_version": "1.0.0",
|
||||
"prompt_template_hash": "b" * 64,
|
||||
"evidence_refs": ["capture-evidence"],
|
||||
"comparison_refs": [str(uuid.uuid4())],
|
||||
"output_schema": "agent-evaluation.schema.json",
|
||||
"decision_policy": {"policy_id": "baseline-semantic", "version": "1.0.0"},
|
||||
"limits": {"timeout_ms": 60000, "max_images": 8, "max_input_tokens": 32000,
|
||||
"max_output_tokens": 2000, "max_cost": "1.00", "currency": "USD"},
|
||||
"trust_policy_hash": "c" * 64,
|
||||
"criteria": [
|
||||
{"criterion_id": "crit-dashboard-visible", "criterion_kind": "semantic",
|
||||
"description": "Sales Dashboard rendered with visible charts and no error banner",
|
||||
"comparison_id": None},
|
||||
],
|
||||
}
|
||||
graph = {
|
||||
"action_registry_version": ACTION_REGISTRY_VERSION,
|
||||
"action_registry_hash": action_registry_fingerprint(),
|
||||
"environment_ids": [ENVIRONMENT_ID],
|
||||
"steps": [
|
||||
{"logical_step_id": "open-dashboard", "tool": "browser", "action": "open_dashboard",
|
||||
"environment_id": ENVIRONMENT_ID, "dashboard_id": DASHBOARD_ID,
|
||||
"action_descriptor": descriptor("open_dashboard", "browser")},
|
||||
{"logical_step_id": "capture-evidence", "tool": "screenshot", "action": "capture_screenshot",
|
||||
"environment_id": ENVIRONMENT_ID, "dashboard_id": DASHBOARD_ID,
|
||||
"action_descriptor": descriptor("capture_screenshot", "screenshot")},
|
||||
{"logical_step_id": "compare-to-baseline", "tool": "assertion", "action": "compare_to_baseline",
|
||||
"environment_id": ENVIRONMENT_ID, "dashboard_id": DASHBOARD_ID,
|
||||
# Deterministic dashboard-identity assertion: the published fixture is a validation
|
||||
# snapshot (placeholder digests), so no image comparison is claimed — the pinned
|
||||
# linkage is proven by the resolved pin, not by this step's PASS.
|
||||
"expected": {"dashboard_id": DASHBOARD_ID}, "actual": {"dashboard_id": DASHBOARD_ID},
|
||||
"policy_type": "exact",
|
||||
"action_descriptor": descriptor("compare_to_baseline", "assertion")},
|
||||
{"logical_step_id": "evaluate-visual", "tool": "agent_evaluation", "action": "evaluate_declared_spec",
|
||||
"environment_id": ENVIRONMENT_ID, "dashboard_id": DASHBOARD_ID,
|
||||
"agent_evaluation_spec": spec,
|
||||
"action_descriptor": descriptor("evaluate_declared_spec", "agent_evaluation")},
|
||||
],
|
||||
"dependencies": [
|
||||
{"source": "open-dashboard", "target": "capture-evidence"},
|
||||
{"source": "capture-evidence", "target": "compare-to-baseline"},
|
||||
{"source": "capture-evidence", "target": "evaluate-visual"},
|
||||
],
|
||||
}
|
||||
with SessionLocal() as db:
|
||||
db.add(ScenarioRevision(
|
||||
revision_id=revision_id, scenario_id=scenario_id,
|
||||
content_hash=action_registry_fingerprint(), graph_snapshot=graph,
|
||||
execution_template_hash="", template_version="v1", schema_version=1,
|
||||
compatibility_family="default", change_summary={}, created_by=ACTOR,
|
||||
activation_status="current",
|
||||
))
|
||||
db.add(ScenarioRegistryEntry(
|
||||
scenario_id=scenario_id, scenario_key=scenario_id,
|
||||
name="Live canary v4 baseline pin 2026-09-11", dashboard_id=DASHBOARD_ID,
|
||||
environment_ids=[ENVIRONMENT_ID], owner_id=ACTOR, owner_username=ACTOR,
|
||||
lifecycle_status="DRAFT", validation_status="valid",
|
||||
current_revision_id=revision_id,
|
||||
))
|
||||
db.commit()
|
||||
return scenario_id, revision_id
|
||||
# #endregion ScenarioExecution.LiveCanaryV4.Seed
|
||||
|
||||
|
||||
# #region ScenarioExecution.LiveCanaryV4.Publish [C:4] [TYPE Function] [SEMANTICS baseline,catalog,gitea,publish,envelope]
|
||||
# @ingroup ScenarioExecution
|
||||
# @BRIEF Publish the resolver-canonical envelope (identity top-level + catalog_revision) to Gitea.
|
||||
# @PRE Deployment env supplies PUBLISHED_CATALOG_GITEA_URL/TOKEN and PUBLISHED_CATALOG_REPO/REF.
|
||||
# @POST Returns the published commit sha; the envelope is validated before any network call and the
|
||||
# update path reuses the stored sha (create=POST, update=PUT per the Gitea contents contract).
|
||||
# @INVARIANT The published bytes are the resolver-canonical envelope — the fixture's baseline_pin
|
||||
# identity hoisted to top-level plus its catalog_revision — so the start-path resolver
|
||||
# consumes exactly what this packet published (no working-tree YAML, no client bytes).
|
||||
def publish_envelope() -> dict:
|
||||
fixture = json.loads((Path(_REPO_ROOT) / "specs/044-dashboard-scenario-execution/fixtures/production-contract-refresh.json").read_text(encoding="utf-8"))
|
||||
pin = fixture["baseline_pin"]
|
||||
envelope = {
|
||||
"baseline_set_id": pin["baseline_set_id"],
|
||||
"baseline_set_version": pin["baseline_set_version"],
|
||||
"release_id": pin["release_id"],
|
||||
"baseline_family": pin["baseline_family"],
|
||||
"catalog_digest": pin["catalog_digest"],
|
||||
"catalog_revision": fixture["catalog_revision"],
|
||||
}
|
||||
raw = json.dumps(envelope, indent=1, ensure_ascii=False).encode("utf-8")
|
||||
validate_catalog(raw)
|
||||
target = GiteaTarget(
|
||||
base_url=os.environ["PUBLISHED_CATALOG_GITEA_URL"].rstrip("/"),
|
||||
repo=os.environ["PUBLISHED_CATALOG_REPO"],
|
||||
ref=os.environ.get("PUBLISHED_CATALOG_REF", "master"),
|
||||
token=os.environ["PUBLISHED_CATALOG_GITEA_TOKEN"],
|
||||
path=catalog_path(pin["baseline_set_id"], pin["baseline_set_version"]),
|
||||
)
|
||||
headers = {"Authorization": f"token {os.environ['PUBLISHED_CATALOG_GITEA_TOKEN']}"}
|
||||
with httpx.Client(base_url=target.base_url, headers=headers, timeout=30.0) as client:
|
||||
commit = publish_catalog_bytes(client, target, raw, "Publish 044 published-envelope snapshot ss-prod-visual v1 (canary v4)")
|
||||
return {"path": target.path, "commit_sha": str((commit.get("commit") or {}).get("sha") or "")}
|
||||
# #endregion ScenarioExecution.LiveCanaryV4.Publish
|
||||
|
||||
|
||||
# #region ScenarioExecution.LiveCanaryV4.Run [C:3] [TYPE Function] [SEMANTICS scenario,baseline,pin,gate,report]
|
||||
# @ingroup ScenarioExecution
|
||||
# @BRIEF Start the pinned run (selector -> published catalog), approve the PROD gate, poll, verify pin.
|
||||
# @POST Fails closed unless runner_plan.baseline_pin and the persisted AgentEvaluation.baseline_pin
|
||||
# both carry the resolved published pin; a missing pin is a ReplayError-grade failure.
|
||||
def main() -> None:
|
||||
publication = publish_envelope()
|
||||
print(f"[canary-v4] published: {publication['path']} commit={publication['commit_sha'][:12]}")
|
||||
scenario_id, revision_id = seed_scenario()
|
||||
print(f"[canary-v4] scenario={scenario_id} revision={revision_id}")
|
||||
|
||||
with httpx.Client(base_url=BASE, timeout=30) as client:
|
||||
login = client.post("/api/auth/login", data={"username": ACTOR, "password": ACTOR})
|
||||
login.raise_for_status()
|
||||
client.headers["Authorization"] = f"Bearer {login.json()['access_token']}"
|
||||
|
||||
start = client.post("/api/scenario-runs", json={
|
||||
"scenario_id": scenario_id, "revision_id": revision_id,
|
||||
"environment_id": ENVIRONMENT_ID,
|
||||
"baseline_set": BASELINE_SET, "baseline_set_version": BASELINE_SET_VERSION,
|
||||
}, headers={"Idempotency-Key": f"canary-v4-{uuid.uuid4().hex[:12]}"})
|
||||
print(f"[canary-v4] REST start: HTTP {start.status_code}")
|
||||
start.raise_for_status()
|
||||
run = start.json()
|
||||
run_id = run["id"]
|
||||
print(f"[canary-v4] run={run_id} status={run['status']} phase={run['phase']}")
|
||||
|
||||
if run["status"] == "pending_approval":
|
||||
approve = client.post(f"/api/scenario-runs/{run_id}/approval/decision",
|
||||
json={"decision": "approve", "comment": "live canary v4 baseline pin"})
|
||||
print(f"[canary-v4] approve: HTTP {approve.status_code} -> {approve.json().get('status')}")
|
||||
approve.raise_for_status()
|
||||
|
||||
deadline = time.monotonic() + POLL_TIMEOUT_SECONDS
|
||||
status = "queued"
|
||||
detail: dict = {}
|
||||
while time.monotonic() < deadline:
|
||||
detail = client.get(f"/api/scenario-runs/{run_id}").json()
|
||||
status = detail["status"]
|
||||
if status in _TERMINAL_STATUSES:
|
||||
print(f"[canary-v4] terminal: {status}")
|
||||
break
|
||||
time.sleep(5)
|
||||
else:
|
||||
print("[canary-v4] poll timeout")
|
||||
|
||||
with SessionLocal() as db:
|
||||
run_row = db.get(ScenarioRun, run_id)
|
||||
plan_pin = (run_row.runner_plan or {}).get("baseline_pin") if run_row else None
|
||||
evaluation = db.query(AgentEvaluation).filter(AgentEvaluation.scenario_run_id == run_id).first()
|
||||
steps = db.query(ScenarioStepRun).filter(ScenarioStepRun.run_id == run_id).all()
|
||||
report = {
|
||||
"run_id": run_id,
|
||||
"scenario_id": scenario_id,
|
||||
"publication": publication,
|
||||
"baseline_selector": {"baseline_set": BASELINE_SET, "baseline_set_version": BASELINE_SET_VERSION},
|
||||
"plan_baseline_pin": plan_pin,
|
||||
"evaluation_baseline_pin": evaluation.baseline_pin if evaluation else None,
|
||||
"evaluation_id": evaluation.evaluation_id if evaluation else None,
|
||||
"evaluation_verdict": evaluation.verdict if evaluation else None,
|
||||
"final_status": status,
|
||||
"final_phase": detail.get("phase"),
|
||||
"error_code": detail.get("error_code"),
|
||||
"steps": [
|
||||
{
|
||||
"logical_step_id": s.logical_step_id,
|
||||
"status": s.status,
|
||||
"error_code": s.error_code,
|
||||
"artifact_refs_count": len(s.artifact_refs or []),
|
||||
"step_outcome": {
|
||||
k: v for k, v in (s.step_outcome or {}).items()
|
||||
if k in ("status", "reason_code", "sha256", "verdict", "confidence",
|
||||
"comparison_status", "screenshot_count")
|
||||
},
|
||||
}
|
||||
for s in steps
|
||||
],
|
||||
}
|
||||
print("[canary-v4] RESULT")
|
||||
print(json.dumps(report, indent=1, ensure_ascii=False, default=str))
|
||||
result_path = os.environ.get("SS_REPLAY_RESULT", "/tmp/kilo/live_canary_v4_result.json")
|
||||
with open(result_path, "w") as handle:
|
||||
json.dump(report, handle, indent=1, default=str)
|
||||
# Fail-closed pin proof: the whole packet exists to prove the published-catalog pin end-to-end.
|
||||
assert isinstance(plan_pin, dict) and plan_pin.get("catalog_digest"), "runner_plan pin missing"
|
||||
assert evaluation is not None and evaluation.baseline_pin, "AgentEvaluation baseline_pin not stamped"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
# #endregion ScenarioExecution.LiveCanaryV4.Run
|
||||
@@ -26,7 +26,7 @@
|
||||
| Visual chain | Compiler emits browser→capture→comparison→evaluation→policy (Slice E) | One-action templates; live Playwright canary PASSED 2026-09-10 (browser `open_dashboard` + screenshot capture vs ss-prod dashboard 11; comparison/evaluation steps not in the canary graph) |
|
||||
| Live providers | Registered browser/screenshot + multimodal LLM | Binding `ss-prod-d11-live-001` resolved **server-side at REST start** (no binding in the request body); `/api/ready`: browser/screenshot ready, bindings 1; canary v2 live trace: `docs/reports/agentic-runtime-live-canary-v2-2026-09-10.md` |
|
||||
|
||||
**Open live gaps (2026-09-10, for the next agent):** ~~multimodal image attachment (prompts are text-only → visual verdicts stay inconclusive); `baseline_pin` in the persisted `AgentEvaluation` record is `{}` for baseline-backed plans; screenshot/browser evidence MIME is hardcoded (`image/jpeg`/`image/png`) and would mismatch a WebP archive~~ → all three CLOSED 2026-09-10/11 (commit `792bb125`): multimodal evidence attachment (`evaluation_images.py` + `_evaluation_messages`), server-authoritative `baseline_pin` stamping (`baseline_resolver.stamp_baseline_pin`, live canary v3 `9e6f59c0`: `verdict=pass`, visual explanation), per-ref MIME sniffing (`mime_sniff.py`). Still open: live baseline pin is blocked on a valid Gitea PAT (publisher ready: `backend/src/scripts/publish_catalog.py`).
|
||||
**Open live gaps (2026-09-10, for the next agent):** ~~multimodal image attachment; `baseline_pin={}` for baseline-backed plans; hardcoded evidence MIME~~ → all three CLOSED 2026-09-10/11 (commit `792bb125`). ~~Live baseline pin blocked on a valid Gitea PAT~~ → **CLOSED 2026-09-11** (canary v4, run `ace916a0…`): the 037 envelope published to Gitea (`catalogs/ss-prod-visual/v1.json`, commit `4d5b7e4c`), selector-driven resolution from published bytes → `runner_plan.baseline_pin` == `AgentEvaluation.baseline_pin` — full trace: `docs/reports/agentic-runtime-live-canary-v4-baseline-pin-2026-09-11.md`. No open live gaps remain; residual debt is structural (oversized contracts / naked defs) and `Expected.threshold` (038 schema authority).
|
||||
|
||||
The former agent-driven product-UI flow (dashboard → `/agent` workspace → agent-generated scenario) is SUPERSEDED. 044 is a headless execution engine; agent interaction is external MCP (050). Product UI (045) is read-only evidence plus human approvals.
|
||||
|
||||
|
||||
@@ -93,7 +93,7 @@ Historical [x] rows above retain only their dated local/transport evidence; they
|
||||
Contract: [Contract-complete public parity](contracts/modules.md).
|
||||
|
||||
- [ ] T044 [P0/P1/P2] REST/MCP lifecycle/read/auth errors and disabled automation validation are identical; service principal cannot decide human gate. Implement at the existing 050 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11):** offline parity tests green; live REST-only canary v2 exercised start→gate→approve→dispatch (`4eebfab3`); the live MCP-external replay (T029m CLOSED, `docs/2026-09-11-sales-prod-mcp-replay.md`) exercised the same lifecycle through `/mcp` with the identical typed start/gate/terminal outcomes — remaining: dedicated REST-vs-MCP error-shape parity fixtures for this matrix.
|
||||
- [ ] T045 [P0/P1/P2] consume/publish failures return typed errors/pending state with no legacy fallback; every prerequisite is externally MCP-reachable. Implement at the existing 050 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11):** the caller-side published-catalog source returns typed `None`/fail-closed (commit `bbbd4ccf`); the publisher helper landed 2026-09-11 (`backend/src/scripts/publish_catalog.py`, pre-validate + Gitea contents PUT, typed auth/conflict/unavailable; `tests/scripts/test_publish_catalog.py` offline-green). Gitea publication path remains BLOCKED on a valid PAT. No live publish failure canary yet.
|
||||
- [ ] T046 [P0/P1/P2] Fresh external-client chain preserves authoritative context and complete baseline pin without raw ORM/REST repair; no frontend agent controls/routes/requests. Implement at the existing 050 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11):** a fresh REST client chain (canary v2 `4eebfab3`) AND the fresh MCP-external chain (T029m CLOSED) both preserve the server-resolved binding and the verified authoritative context live (`context_authority=verified` at the register boundary; binding adopted server-side and exercised by live providers). The complete baseline pin is still blocked on the Gitea PAT.
|
||||
- [x] T045 [P0/P1/P2] consume/publish failures return typed errors/pending state with no legacy fallback; every prerequisite is externally MCP-reachable. Implement at the existing 050 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11): CLOSED.** Consume path typed fail-closed (`bbbd4ccf`); publisher `backend/src/scripts/publish_catalog.py` with pre-validation before any network I/O and typed auth/conflict/unavailable (offline `tests/scripts/test_publish_catalog.py`); LIVE evidence (`docs/reports/agentic-runtime-live-canary-v4-baseline-pin-2026-09-11.md`): create `e41d6ad2` (POST), update-sha `3bb2c63e` (PUT), publish-failure canary with a bogus token → typed `PUBLISH_AUTH_FAILED` (HTTP 401, exit 1, no partial state); every prerequisite externally reachable (T029m CLOSED). Defects found and fixed live: Gitea create=POST/update=PUT contract; `run.sh` wiping deployment keys below the MCP_JWT line on every start (venv-python validation fallback + single-line in-place rewrite).
|
||||
- [x] T046 [P0/P1/P2] Fresh external-client chain preserves authoritative context and complete baseline pin without raw ORM/REST repair; no frontend agent controls/routes/requests. Implement at the existing 050 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11): CLOSED.** Fresh REST chain (canary v2 `4eebfab3`) and fresh MCP-external chain (T029m) preserve the server-resolved binding and verified authoritative context (`context_authority=verified`); the complete baseline pin is proven live by canary v4 (run `ace916a0…`, `docs/reports/agentic-runtime-live-canary-v4-baseline-pin-2026-09-11.md`): selector-driven resolution from the PUBLISHED Gitea envelope → `runner_plan.baseline_pin` == `AgentEvaluation.baseline_pin` (walker stamping, commit `792bb125`), no raw ORM/REST repair; frontend agent controls remain absent (2026-09-10 evidence).
|
||||
|
||||
Frontend boundary for this package: manual CRUD/editor, human review/approval, monitoring and read-only evidence/evaluation only; all agent interaction is external MCP. No agent chat/prompt/assistant editing/proposal generation/workspace/start/handoff controls. Runtime removal is OPEN, not performed by this spec refresh. Optional approved performance baseline is outside scope.
|
||||
|
||||
Reference in New Issue
Block a user