From de5e098a1beb088f87b98336abd539ddcbcfda82 Mon Sep 17 00:00:00 2001 From: busya Date: Fri, 11 Sep 2026 15:09:16 +0300 Subject: [PATCH] =?UTF-8?q?fix(scenario):=20review=20hardening=20=E2=80=94?= =?UTF-8?q?=20run.sh=20key=20reuse,=20resolver-canonical=20publisher=20env?= =?UTF-8?q?elope,=20strict=20pin=20asserts,=20honest=20T045/T046=20statuse?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/src/scripts/publish_catalog.py | 104 +++++++++++------- .../execution/published_catalog_source.py | 7 +- backend/tests/scripts/test_publish_catalog.py | 102 ++++++++++++++--- ...-live-canary-v4-baseline-pin-2026-09-11.md | 36 +++++- run.sh | 31 +++--- .../prototype/live_canary_v4_baseline_pin.py | 51 +++++++-- .../prototype/live_mcp_replay.py | 44 ++++++-- .../044-dashboard-scenario-execution/tasks.md | 4 +- .../traceability.md | 2 +- specs/050-mcp-interface/tasks.md | 4 +- 10 files changed, 278 insertions(+), 107 deletions(-) diff --git a/backend/src/scripts/publish_catalog.py b/backend/src/scripts/publish_catalog.py index 9919be8af..ffc8a0652 100644 --- a/backend/src/scripts/publish_catalog.py +++ b/backend/src/scripts/publish_catalog.py @@ -1,17 +1,21 @@ # #region Tooling.PublishCatalog [C:4] [TYPE Module] [SEMANTICS cli,gitea,catalog,publish,baseline,release] -# @defgroup Tooling D7-prep publication helper: push one validated 044 catalog snapshot to Gitea. -# @BRIEF CLI: validate catalog bytes fail-closed, then PUT them to the Gitea contents API. +# @defgroup Tooling Publication helper: push one resolver-canonical 037 envelope to Gitea. +# @BRIEF CLI: validate the published envelope (top-level identity + catalog_revision), then write it. # @RELATION CALLS -> [ScenarioExecution.BaselineResolver.LoadPublished] -# @INVARIANT Catalog bytes are validated (JSON + baseline_pin shape) BEFORE any network call; an -# invalid catalog never reaches Gitea. Secrets live only in environment variables. +# @INVARIANT Catalog bytes are validated as the resolver-canonical published envelope BEFORE any +# network call; an invalid or nested-only artifact never reaches Gitea. Secrets live only +# in environment variables and are never printed. # @INVARIANT Errors are typed and fail-closed (auth/conflict/unavailable/config/catalog); the script -# never fabricates a successful publication and never prints the token. -# @RATIONALE T6/T045: the published-catalog source (`PUBLISHED_CATALOG_*` env) reads catalog bytes -# from Gitea at start time; without this helper there was no trusted way to publish them. -# @REJECTED Uploading without pre-validation, or auto-creating missing repos, was rejected — the -# publisher must stay a thin, auditable transport for already-validated snapshots. -# @PRE Environment: PUBLISHED_CATALOG_GITEA_URL, PUBLISHED_CATALOG_GITEA_TOKEN, -# PUBLISHED_CATALOG_GITEA_REPO (owner/repo), optional PUBLISHED_CATALOG_GITEA_REF (default main), +# never fabricates a successful publication. +# @RATIONALE T6/T045: the published-catalog source (`PUBLISHED_CATALOG_*` env) reads bytes from +# Gitea at start time; only the resolver-canonical envelope (top-level identity + +# `catalog_revision`) becomes a runnable pin, so publishing anything else is a footgun. +# @REJECTED Accepting the nested 044 contract-refresh artifact was rejected after the live defect: +# it publishes bytes the resolver rejects (BASELINE_NOT_PUBLISHED), so the operator must +# hoist identity first (see the canary envelope builder). +# @PRE Environment: PUBLISHED_CATALOG_GITEA_URL, PUBLISHED_CATALOG_GITEA_TOKEN, PUBLISHED_CATALOG_REPO +# (owner/repo; the legacy PUBLISHED_CATALOG_GITEA_REPO spelling is accepted), optional +# PUBLISHED_CATALOG_REF (legacy PUBLISHED_CATALOG_GITEA_REF; default main, matching the loader), # optional PUBLISHED_CATALOG_PATH_TEMPLATE (default catalogs/{baseline_set_id}/v{version}.json). from __future__ import annotations @@ -41,11 +45,13 @@ class PublishError(RuntimeError): # #region Tooling.PublishCatalog.Validate [C:3] [TYPE Function] [SEMANTICS catalog,validate,fail-closed] -# @BRIEF Parse and shape-check catalog bytes before any network I/O. -# @POST Returns the parsed catalog dict; raises PublishError(PUBLISH_CATALOG_INVALID) on any -# malformed JSON, non-object payload, or missing/shape-invalid pin identity. Two deployment -# shapes are lawful: the 044 contract-refresh artifact (identity nested under `baseline_pin`) -# and the 037 published envelope (identity at top level, consumed by the 044 resolver). +# @BRIEF Parse and shape-check the resolver-canonical published envelope before any network I/O. +# @POST Returns the parsed envelope; raises PublishError(PUBLISH_CATALOG_INVALID) on malformed JSON, +# a non-object payload, missing/shape-invalid top-level identity keys, or a missing +# `catalog_revision` object. A nested-only contract-refresh artifact is rejected explicitly. +# @INVARIANT The validated shape is exactly what ScenarioExecution.BaselineResolver consumes +# (_split_published_snapshot + _require_selector_match + _require_pin_identity): top-level +# identity plus a `catalog_revision` dict. Publishing any other shape cannot become a pin. def validate_catalog(raw: bytes) -> dict[str, Any]: try: payload = json.loads(raw.decode("utf-8")) @@ -53,12 +59,18 @@ def validate_catalog(raw: bytes) -> dict[str, Any]: raise PublishError("PUBLISH_CATALOG_INVALID", f"catalog bytes are not JSON: {exc}") from exc if not isinstance(payload, dict): raise PublishError("PUBLISH_CATALOG_INVALID", "catalog payload must be a JSON object") - pin = payload.get("baseline_pin") - if not isinstance(pin, dict): - pin = {key: payload.get(key) for key in REQUIRED_PIN_KEYS} - missing = [key for key in REQUIRED_PIN_KEYS if not isinstance(pin.get(key), str) or not pin.get(key)] + nested = payload.get("baseline_pin") + if isinstance(nested, dict): + raise PublishError( + "PUBLISH_CATALOG_INVALID", + "published envelope requires top-level identity + catalog_revision; " + "hoist the nested baseline_pin identity before publishing", + ) + missing = [key for key in REQUIRED_PIN_KEYS if not isinstance(payload.get(key), str) or not payload.get(key)] if missing: - raise PublishError("PUBLISH_CATALOG_INVALID", f"baseline_pin missing keys: {', '.join(missing)}") + raise PublishError("PUBLISH_CATALOG_INVALID", f"top-level identity missing keys: {', '.join(missing)}") + if not isinstance(payload.get("catalog_revision"), dict): + raise PublishError("PUBLISH_CATALOG_INVALID", "published envelope is missing a catalog_revision object") return payload # #endregion Tooling.PublishCatalog.Validate @@ -66,20 +78,22 @@ def validate_catalog(raw: bytes) -> dict[str, Any]: # #region Tooling.PublishCatalog.Path [C:2] [TYPE Function] [SEMANTICS catalog,path,template] # @BRIEF Render the repository path from the template and pin identity. # @POST Accepts every template alias in use across the deployment contract -# ({baseline_set_id}/{version} publisher-side, {baseline_set}/{baseline_set_version} loader-side) -# so one operator template works for both; unknown placeholders are a typed invalid-catalog error. +# ({baseline_set_id}/{version} publisher-side, {baseline_set}/{baseline_set_version} +# loader-side) so one operator template works for both surfaces. Returns the RAW path; +# URL-encoding happens exactly once in GiteaTarget.contents_url. A malformed template is a +# typed PUBLISH_CATALOG_PATH_INVALID, never an untyped traceback. def catalog_path(baseline_set_id: str, version: str, template: str | None = None) -> str: selected = template or os.environ.get("PUBLISHED_CATALOG_PATH_TEMPLATE") or DEFAULT_PATH_TEMPLATE kwargs = { - "baseline_set_id": quote(baseline_set_id, safe=""), - "version": quote(version, safe=""), - "baseline_set": quote(baseline_set_id, safe=""), - "baseline_set_version": quote(version, safe=""), + "baseline_set_id": baseline_set_id, + "version": version, + "baseline_set": baseline_set_id, + "baseline_set_version": version, } try: rendered = selected.format(**kwargs) - except (KeyError, IndexError) as exc: - raise PublishError("PUBLISH_CATALOG_PATH_INVALID", f"path template placeholder unknown: {exc}") from exc + except (KeyError, IndexError, ValueError) as exc: + raise PublishError("PUBLISH_CATALOG_PATH_INVALID", f"path template invalid: {exc}") from exc return rendered.lstrip("/") # #endregion Tooling.PublishCatalog.Path @@ -93,7 +107,8 @@ class GiteaTarget: path: str def contents_url(self, path: str) -> str: - return f"/api/v1/repos/{self.repo}/contents/{quote(path)}" + # Single URL-encoding point: catalog_path returns the raw rendered path. + return f"/api/v1/repos/{self.repo}/contents/{quote(path, safe='/')}" # #region Tooling.PublishCatalog.Fetch [C:3] [TYPE Function] [SEMANTICS gitea,contents,sha,fetch] @@ -134,7 +149,12 @@ def publish_catalog_bytes(client: Any, target: GiteaTarget, raw: bytes, message: response = client.post(target.contents_url(target.path), json=body) if response.status_code in (401, 403): raise PublishError("PUBLISH_AUTH_FAILED", f"Gitea rejected the token (HTTP {response.status_code})") - if response.status_code == 409: + # 409 is a stale-sha conflict; Gitea also reports a create race as 400/422 ("file already + # exists" / "[SHA]: Required") — both are retryable concurrent change, not unavailability. + body_text = (response.text or "").lower() + if response.status_code == 409 or ( + response.status_code in (400, 422) and ("already exists" in body_text or "[sha]" in body_text) + ): raise PublishError("PUBLISH_CONFLICT", "the catalog file changed concurrently; re-run to update the new sha") if response.status_code not in (200, 201): raise PublishError("PUBLISH_UNAVAILABLE", f"unexpected write status {response.status_code}: {response.text[:200]}") @@ -149,19 +169,20 @@ def publish_catalog_bytes(client: Any, target: GiteaTarget, raw: bytes, message: # @POST Exit 0 with {"result":"published", ...} on success; exit 1 with {"result":"failed","code":...} # on typed failure; exit 2 with PUBLISH_CONFIG_MISSING when the environment is not configured. def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description="Publish a validated 044 catalog snapshot to Gitea.") - parser.add_argument("--catalog", required=True, help="path to the catalog snapshot JSON") + parser = argparse.ArgumentParser(description="Publish a validated 037 published envelope to Gitea.") + parser.add_argument("--catalog", required=True, help="path to the published envelope JSON") parser.add_argument("--baseline-set", required=True, help="baseline_set_id used in the path template") parser.add_argument("--version", required=True, help="baseline_set_version used in the path template") - parser.add_argument("--message", default="Publish 044 catalog snapshot", help="commit message") + parser.add_argument("--message", default="Publish 037 published envelope", help="commit message") args = parser.parse_args(argv) - base_url = os.environ.get("PUBLISHED_CATALOG_GITEA_URL", "").rstrip("/") - token = os.environ.get("PUBLISHED_CATALOG_GITEA_TOKEN", "") + base_url = os.environ.get("PUBLISHED_CATALOG_GITEA_URL", "").strip().rstrip("/") + token = os.environ.get("PUBLISHED_CATALOG_GITEA_TOKEN", "").strip() # Canonical env keys are the loader's (PUBLISHED_CATALOG_REPO/REF, commit bbbd4ccf); the # GITEA_-prefixed spellings remain accepted so one deployment config feeds both surfaces. repo = (os.environ.get("PUBLISHED_CATALOG_REPO") or os.environ.get("PUBLISHED_CATALOG_GITEA_REPO") or "").strip().strip("/") - ref = (os.environ.get("PUBLISHED_CATALOG_REF") or os.environ.get("PUBLISHED_CATALOG_GITEA_REF") or "").strip() or "master" + # Default matches published_catalog_source (main) so an unset ref cannot split write/read branches. + ref = (os.environ.get("PUBLISHED_CATALOG_REF") or os.environ.get("PUBLISHED_CATALOG_GITEA_REF") or "").strip() or "main" missing = [name for name, value in ( ("PUBLISHED_CATALOG_GITEA_URL", base_url), ("PUBLISHED_CATALOG_GITEA_TOKEN", token), ("PUBLISHED_CATALOG_REPO", repo), @@ -174,10 +195,7 @@ def main(argv: list[str] | None = None) -> int: try: with open(args.catalog, "rb") as handle: raw = handle.read() - catalog = validate_catalog(raw) - pin = catalog.get("baseline_pin") if isinstance(catalog.get("baseline_pin"), dict) else { - key: catalog.get(key) for key in REQUIRED_PIN_KEYS - } + envelope = validate_catalog(raw) target = GiteaTarget( base_url=base_url, repo=repo, ref=ref, token=token, path=catalog_path(args.baseline_set, args.version), @@ -187,7 +205,9 @@ def main(argv: list[str] | None = None) -> int: commit = publish_catalog_bytes(client, target, raw, args.message) print(json.dumps({ "result": "published", "path": target.path, "ref": target.ref, - "baseline_set_id": pin["baseline_set_id"], "baseline_set_version": pin["baseline_set_version"], + "baseline_set_id": envelope["baseline_set_id"], + "baseline_set_version": envelope["baseline_set_version"], + "catalog_digest": envelope["catalog_digest"], "commit": commit.get("commit") or commit.get("content") or {}, })) return 0 diff --git a/backend/src/services/dashboard_testing/execution/published_catalog_source.py b/backend/src/services/dashboard_testing/execution/published_catalog_source.py index 2932ddf2e..daf3e5b9d 100644 --- a/backend/src/services/dashboard_testing/execution/published_catalog_source.py +++ b/backend/src/services/dashboard_testing/execution/published_catalog_source.py @@ -77,10 +77,13 @@ def load_published_catalog_from_config( if not set_id or not set_version: return None try: + # Accept both placeholder families so a single PUBLISHED_CATALOG_PATH_TEMPLATE works for the + # publisher ({baseline_set_id}/{version}) and this loader ({baseline_set}/{baseline_set_version}). relative_path = config["path_template"].format( - baseline_set=set_id, baseline_set_version=set_version + baseline_set=set_id, baseline_set_version=set_version, + baseline_set_id=set_id, version=set_version, ) - except (KeyError, IndexError): + except (KeyError, IndexError, ValueError): logger.explore( "Published catalog path template is malformed", src=_SRC, error_code="PUBLISHED_CATALOG_PATH_INVALID", diff --git a/backend/tests/scripts/test_publish_catalog.py b/backend/tests/scripts/test_publish_catalog.py index 5af3b0ca1..dc82ca22f 100644 --- a/backend/tests/scripts/test_publish_catalog.py +++ b/backend/tests/scripts/test_publish_catalog.py @@ -4,9 +4,9 @@ # @RELATION VERIFIES -> [Tooling.PublishCatalog] # @TEST_CONTRACT: validate_catalog -> parsed dict or PUBLISH_CATALOG_INVALID # @TEST_EDGE: missing-env config -> exit 2 without network -# @TEST_INVARIANT Tooling.PublishCatalog: an invalid catalog never reaches the network; create and -# update PUTs carry the right sha semantics; auth/conflict failures are typed. -# -> VERIFIED_BY: publish flow tests below +# @TEST_INVARIANT Tooling.PublishCatalog: an invalid or nested-only catalog never reaches the network; +# create is POST (no sha) and update is PUT (sha) per the Gitea contract; auth, +# create-race and transport failures are typed. -> VERIFIED_BY: tests below import base64 import json import os @@ -25,18 +25,22 @@ from src.scripts.publish_catalog import ( # noqa: E402 validate_catalog, ) -_CATALOG = { - "catalog_revision": "rev-001", - "baseline_pin": { - "baseline_set_id": "ss-prod-visual", "baseline_set_version": "1", - "release_id": "ss-prod-initial", "baseline_family": "visual", - "catalog_digest": "a" * 64, - }, +_ENVELOPE = { + "baseline_set_id": "ss-prod-visual", "baseline_set_version": "1", + "release_id": "11111111-1111-4111-8111-111111111111", "baseline_family": "a" * 64, + "catalog_digest": "a" * 64, + "catalog_revision": {"catalog_revision_id": "rev-001", "publication": {"state": "published"}}, } -_RAW = json.dumps(_CATALOG).encode() +_RAW = json.dumps(_ENVELOPE).encode() +_NESTED_ONLY = json.dumps({ + "catalog_revision": {"catalog_revision_id": "rev-001"}, + "baseline_pin": {"baseline_set_id": "ss-prod-visual"}, +}).encode() _TARGET = GiteaTarget(base_url="http://gitea.test", repo="o/r", ref="main", token="t", path="catalogs/ss-prod-visual/v1.json") +# #region Test.Tooling.PublishCatalog.Fakes [C:2] [TYPE Class] +# @BRIEF Minimal httpx-client doubles capturing GET/POST/PUT calls for offline assertions. class _FakeResponse: def __init__(self, status_code: int, payload=None, text: str = "") -> None: self.status_code = status_code @@ -77,10 +81,22 @@ class _FakeClient: # #region Test.Tooling.PublishCatalog.Validate [C:2] [TYPE Function] -def test_validate_catalog_accepts_wellformed_and_rejects_malformed(): - assert validate_catalog(_RAW)["baseline_pin"]["catalog_digest"] == "a" * 64 +def test_validate_catalog_accepts_envelope_and_rejects_nested_only_and_malformed(): + assert validate_catalog(_RAW)["catalog_digest"] == "a" * 64 - for bad in (b"not-json", b"[]", json.dumps({"baseline_pin": {}}).encode(), json.dumps({"baseline_pin": {"baseline_set_id": "x"}}).encode()): + # A nested-only contract-refresh artifact must be rejected: the resolver cannot consume it. + try: + validate_catalog(_NESTED_ONLY) + except PublishError as exc: + assert exc.code == "PUBLISH_CATALOG_INVALID" and "top-level identity" in exc.detail + else: + raise AssertionError("nested-only catalog must be rejected") + + for bad in ( + b"not-json", b"[]", + json.dumps({**_ENVELOPE, "release_id": ""}).encode(), + json.dumps({k: v for k, v in _ENVELOPE.items() if k != "catalog_revision"}).encode(), + ): try: validate_catalog(bad) except PublishError as exc: @@ -89,9 +105,17 @@ def test_validate_catalog_accepts_wellformed_and_rejects_malformed(): raise AssertionError(f"catalog {bad[:30]!r} must be rejected") -def test_catalog_path_renders_template_and_default(): +def test_catalog_path_renders_aliases_and_rejects_bad_template(): assert catalog_path("ss-prod-visual", "1", "catalogs/{baseline_set_id}/v{version}.json") == "catalogs/ss-prod-visual/v1.json" + assert catalog_path("ss-prod-visual", "1", "catalogs/{baseline_set}/v{baseline_set_version}.json") == "catalogs/ss-prod-visual/v1.json" assert catalog_path("ss-prod-visual", "1", None) == "catalogs/ss-prod-visual/v1.json" + for bad_template in ("catalogs/{unknown}/v1.json", "catalogs/{baseline_set_id/v1.json"): + try: + catalog_path("ss-prod-visual", "1", bad_template) + except PublishError as exc: + assert exc.code == "PUBLISH_CATALOG_PATH_INVALID" + else: + raise AssertionError(f"template {bad_template!r} must be rejected") # #endregion Test.Tooling.PublishCatalog.Validate @@ -138,6 +162,28 @@ def test_publish_failures_are_typed(): assert exc.code == "PUBLISH_UNAVAILABLE" else: raise AssertionError("500 must be PUBLISH_UNAVAILABLE") + + # A create race (file appeared between GET 404 and POST) is a conflict, not unavailability. + race_client = _FakeClient( + _FakeResponse(404), + _FakeResponse(422, {"message": "[SHA]: Required"}, text="[SHA]: Required"), + _FakeResponse(422, {"message": "file already exists"}, text="file already exists"), + ) + try: + publish_catalog_bytes(race_client, _TARGET, _RAW, "race") + except PublishError as exc: + assert exc.code == "PUBLISH_CONFLICT" + else: + raise AssertionError("create race must be PUBLISH_CONFLICT") + + # A POST auth failure is typed the same as a GET/PUT auth failure. + post_auth_client = _FakeClient(_FakeResponse(404), _FakeResponse(200, {}), _FakeResponse(401, text="bad token")) + try: + publish_catalog_bytes(post_auth_client, _TARGET, _RAW, "auth") + except PublishError as exc: + assert exc.code == "PUBLISH_AUTH_FAILED" + else: + raise AssertionError("POST 401 must be PUBLISH_AUTH_FAILED") # #endregion Test.Tooling.PublishCatalog.Publish @@ -173,6 +219,32 @@ def test_main_publishes_end_to_end_with_fake_client(monkeypatch, tmp_path): assert code == 0 assert seen["kwargs"]["base_url"] == "http://gitea.test" assert seen["kwargs"]["headers"] == {"Authorization": "token token-value"} + + +def test_main_creates_via_post_when_absent(monkeypatch, tmp_path): + import src.scripts.publish_catalog as module + + catalog_file = tmp_path / "catalog.json" + catalog_file.write_bytes(_RAW) + monkeypatch.setenv("PUBLISHED_CATALOG_GITEA_URL", "http://gitea.test") + monkeypatch.setenv("PUBLISHED_CATALOG_GITEA_TOKEN", "token-value") + monkeypatch.setenv("PUBLISHED_CATALOG_REPO", "o/r") + created: dict = {} + + class _CreateClient(_FakeClient): + def __init__(self, **kwargs) -> None: + super().__init__(_FakeResponse(404), _FakeResponse(200, {}), _FakeResponse(201, {"commit": {"id": "c-new"}})) + + def post(self, url, json=None): + created.setdefault("posts", []).append((url, json or {})) + return self.post_response + + monkeypatch.setattr(module.httpx, "Client", lambda **kwargs: _CreateClient(**kwargs)) + code = main(["--catalog", str(catalog_file), "--baseline-set", "ss-prod-visual", "--version", "1"]) + assert code == 0 + assert len(created["posts"]) == 1 and "sha" not in created["posts"][0][1] + + def test_main_types_transport_errors(monkeypatch, tmp_path, capsys): import httpx diff --git a/docs/reports/agentic-runtime-live-canary-v4-baseline-pin-2026-09-11.md b/docs/reports/agentic-runtime-live-canary-v4-baseline-pin-2026-09-11.md index dd18768c8..fbbe045f7 100644 --- a/docs/reports/agentic-runtime-live-canary-v4-baseline-pin-2026-09-11.md +++ b/docs/reports/agentic-runtime-live-canary-v4-baseline-pin-2026-09-11.md @@ -55,15 +55,41 @@ Run **`ace916a0-84a8-4f15-befe-860c1fa964ac`**, scenario `live-canary-v4-1fe64a1 ## Statuses -- **050 T045**: CLOSED — publish success + update-sha flow + typed 401 canary + offline typed tests + - consume-path fail-closed (`bbbd4ccf`) + every prerequisite externally reachable (T029m CLOSED). -- **050 T046**: baseline-pin part CLOSED by this trace (complete pin, no raw repair); the row keeps - its frontend-controls evidence from 2026-09-10. -- **044 quickstart live-pin**: CLOSED (was "blocked on a valid Gitea PAT"). +- **050 T046**: CLOSED — complete baseline pin proven through fresh external chains: the + MCP-external `--baseline` chain carries a full `runner_plan.baseline_pin` from the published + envelope (no raw ORM), and the REST canary proves the walker stamps it into + `AgentEvaluation.baseline_pin` (strict equality). +- **050 T045**: OPEN — partially delivered. Consume path + CLI publisher are typed and live-proven, + but MCPX-FR-030 requires a curated MCP `publish_baseline_catalog` operation on the 037 + publication-worker contract (authorized intent, `expected_branch_head` CAS, commit/receipt, + reconcile); no such MCP tool exists, so "every prerequisite is externally MCP-reachable" is not + yet true for publication. +- **044 quickstart live-pin**: CLOSED for the pin resolution + stamping (044's own T043 row should + inherit this evidence; see the review-response note below). + +## Review response (orthogonal review + semantic/spec audit, 2026-09-11) + +Independent code review, GRACE-Poly semantic audit and spec-conformance audit were run on the +packet; the following were found and fixed: + +| Severity | Finding | Fix | +|---|---|---| +| major | `run.sh` key-reuse fix was ineffective: `for line in $(grep ...)` word-splits the PEM header, truncating the key → regeneration every start, and the new in-place branch appended a duplicate line (`.env` had grown to 4 `MCP_JWT_PRIVATE_KEY=` lines) | prefer the already-exported `$MCP_JWT_PRIVATE_KEY`; rewrite helper replaces the first single-line assignment and drops duplicates; existing duplicates collapsed to one (verified: "reused", 1 line) | +| major | Publisher defaulted `ref=master` while the loader defaults `main` → unset ref splits write/read branches | publisher default aligned to `main`; `@PRE` documents the canonical env keys | +| major | Canary "pin identity" proof asserted truthiness only | strict asserts: `plan_pin == AgentEvaluation.baseline_pin` and set/version/release/digest/revision equality against the published envelope | +| high | Publisher accepted the nested contract-refresh artifact the resolver rejects (the live `BASELINE_NOT_PUBLISHED` root cause) and validated weaker than the 044 validator | `validate_catalog` now requires the resolver-canonical envelope (top-level identity + `catalog_revision`) and rejects nested-only with a typed message; tests updated | +| high | T045/T046 were marked CLOSED while MCPX-FR-030's publish tool was absent and the pin proof used raw-ORM seeding | T045 reverted to OPEN with the precise residual; T046 re-argued on the MCP `--baseline` chain (no raw ORM) + REST record-stamping proof | +| medium | Path-template aliases existed only publisher-side; malformed templates raised untyped `ValueError`; double URL-encoding | loader accepts both alias families; `ValueError` typed as `PUBLISH_CATALOG_PATH_INVALID`; raw path rendered once, encoded once in `contents_url` | +| medium | Create race typed as `PUBLISH_UNAVAILABLE` | Gitea "already exists"/"[SHA]" 400/422 mapped to `PUBLISH_CONFLICT` | +| medium | Canary v4 lost its opening `#region ...Env` anchor (`descriptor()` naked); no module contract | module + Env/Helpers regions restored; strict asserts added (INV_3/INV_1) | +| low | env values not stripped; missing tests for envelope/alias/create-main/POST-auth; stale `@TEST_INVARIANT` | `.strip()` on token/repo/ref; added coverage; test contract text synced to POST/PUT | ## Remaining +- **050 T045** — curated MCP `publish_baseline_catalog` + 037 publication-worker contract. - `Expected.threshold` (038 schema authority) — separate architect packet. - Structural curation (8 oversized contracts, `verification_service.py` 407, 50 naked defs) — separate curator packet. +- Server-side `git push` blocked by a cron script polluting the gitea stdout (`/var/tmp/gitea_cron_health`, + `/var/tmp/.sys_health_s3`); publication itself reached the server via the Gitea API. - Recommendation: rotate the Gitea PAT (it transited the chat) and keep it only in `backend/.env`. diff --git a/run.sh b/run.sh index d228726b7..322faf5ed 100755 --- a/run.sh +++ b/run.sh @@ -300,15 +300,10 @@ ensure_jwt_secret # continue to provide MCP_JWT_PRIVATE_KEY explicitly from their secret manager. ensure_mcp_oauth_key() { local ENV_FILE="backend/.env" - local key="" - - if [ -f "$ENV_FILE" ]; then - # The PEM is stored on one dotenv line with literal \n separators and is expanded - # by src.services.mcp_keys at runtime. - for line in $(grep "^MCP_JWT_PRIVATE_KEY=" "$ENV_FILE" | head -1); do - key="${line#MCP_JWT_PRIVATE_KEY=}" - done - fi + # load_dotenv (line 47) already exported the full value using `IFS= read -r -d ''`, so prefer + # the environment over a `for line in $(grep ...)` extraction: the unquoted grep splits the PEM + # header on its spaces, truncates the key, and forced a regeneration on EVERY start (2026-09-11). + local key="${MCP_JWT_PRIVATE_KEY:-}" if [ -n "$key" ] && MCP_JWT_PRIVATE_KEY="$key" python3 -c ' import os @@ -329,8 +324,9 @@ from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey key = serialization.load_pem_private_key(os.environ["MCP_JWT_PRIVATE_KEY"].replace("\\n", "\n").encode(), password=None) raise SystemExit(0 if isinstance(key, RSAPrivateKey) and key.key_size >= 2048 else 1) ' 2>/dev/null; then - # The system python3 often lacks `cryptography`; the backend venv always has it. Without - # this fallback a perfectly valid stored key was regenerated on EVERY start (2026-09-11). + # Defensive fallback for hosts whose system python3 lacks `cryptography`; the backend venv + # always has it. (The 2026-09-11 regeneration storm was not a cryptography gap but the + # word-split extraction above, now replaced by the exported environment value.) export MCP_JWT_PRIVATE_KEY="$key" echo "MCP OAuth key preflight: existing MCP_JWT_PRIVATE_KEY reused from $ENV_FILE (venv python)" return @@ -357,14 +353,18 @@ path = Path(sys.argv[1]) encoded_key = sys.argv[2] source = path.read_text(encoding="utf-8") if path.exists() else "" result: list[str] = [] +replaced = False skipping_pem = False for line in source.splitlines(): if line.startswith("MCP_JWT_PRIVATE_KEY="): if "-----END PRIVATE KEY-----" in line: - # Single-line escaped assignment: replace in place; never swallow the lines after it - # (deployment keys appended below this line must survive the rewrite, 2026-09-11). - result.append(f"MCP_JWT_PRIVATE_KEY={encoded_key}") + # Single-line escaped assignment: replace the FIRST one in place and drop duplicates; + # never append a second copy (unbounded .env growth, 2026-09-11). + if not replaced: + result.append(f"MCP_JWT_PRIVATE_KEY={encoded_key}") + replaced = True continue + # Legacy multiline PEM: skip the whole block, append the escaped key once at the end. skipping_pem = True continue if skipping_pem: @@ -372,7 +372,8 @@ for line in source.splitlines(): skipping_pem = False continue result.append(line) -result.append(f"MCP_JWT_PRIVATE_KEY={encoded_key}") +if not replaced: + result.append(f"MCP_JWT_PRIVATE_KEY={encoded_key}") path.write_text("\n".join(result) + "\n", encoding="utf-8") PY export MCP_JWT_PRIVATE_KEY="$new_key" diff --git a/specs/044-dashboard-scenario-execution/prototype/live_canary_v4_baseline_pin.py b/specs/044-dashboard-scenario-execution/prototype/live_canary_v4_baseline_pin.py index a3580475c..d6db23b72 100644 --- a/specs/044-dashboard-scenario-execution/prototype/live_canary_v4_baseline_pin.py +++ b/specs/044-dashboard-scenario-execution/prototype/live_canary_v4_baseline_pin.py @@ -11,6 +11,19 @@ Expected honest outcomes: `open-dashboard`/`capture-evidence` pass live; `compar deterministic dashboard-identity assertion (the published fixture is a validation snapshot, so no image comparison is claimed); `evaluate-visual` runs the multimodal judge. """ +# #region ScenarioExecution.LiveCanaryV4 [C:4] [TYPE Module] [SEMANTICS scenario,baseline,pin,gitea,canary,live] +# @defgroup ScenarioExecution Live proof that a published Gitea envelope resolves into plan + record pins. +# @BRIEF Publish the resolver-canonical envelope, start with the selector, and assert both pins match. +# @RELATION VERIFIES -> [ScenarioExecution.BaselineResolver.Resolve] +# @RELATION VERIFIES -> [ScenarioExecution.BaselineResolver.StampEvaluation] +# @RELATION CALLS -> [Tooling.PublishCatalog] +# @INVARIANT The run is fail-closed: the script asserts `runner_plan.baseline_pin` equals +# `AgentEvaluation.baseline_pin` and that both carry the published envelope identity; a +# missing or divergent pin fails the canary instead of being reported as success. +# @RATIONALE T045/T046: the pin must come from published bytes at start time and be stamped into the +# immutable evaluation record without any raw ORM/REST repair. +# @REJECTED Asserting pin truthiness only was rejected after review — the proof must compare the +# server-resolved plan pin against the persisted record pin and the published identity. import json import os import sys @@ -54,13 +67,15 @@ POLL_TIMEOUT_SECONDS = 300 _TERMINAL_STATUSES = frozenset({"passed", "failed", "blocked", "inconclusive", "cancelled"}) +# #region ScenarioExecution.LiveCanaryV4.Helpers [C:1] [TYPE Function] [SEMANTICS descriptor,registry] +# @BRIEF Resolve the server-owned action descriptor snapshot for a graph step. def descriptor(action: str, tool: str) -> dict: return resolve_action_descriptor( tool=tool, action=action, registry_version=ACTION_REGISTRY_VERSION, registry_hash=action_registry_fingerprint(), ).snapshot() -# #endregion ScenarioExecution.LiveCanaryV4.Env +# #endregion ScenarioExecution.LiveCanaryV4.Helpers # #region ScenarioExecution.LiveCanaryV4.Seed [C:4] [TYPE Function] [SEMANTICS scenario,baseline,pin,seed] @@ -147,8 +162,8 @@ def seed_scenario() -> tuple[str, str]: # @ingroup ScenarioExecution # @BRIEF Publish the resolver-canonical envelope (identity top-level + catalog_revision) to Gitea. # @PRE Deployment env supplies PUBLISHED_CATALOG_GITEA_URL/TOKEN and PUBLISHED_CATALOG_REPO/REF. -# @POST Returns the published commit sha; the envelope is validated before any network call and the -# update path reuses the stored sha (create=POST, update=PUT per the Gitea contents contract). +# @POST Returns the published path, commit sha and envelope identity (set/version/release/digest/ +# revision) so the caller can assert the resolved pin against the published generation. # @INVARIANT The published bytes are the resolver-canonical envelope — the fixture's baseline_pin # identity hoisted to top-level plus its catalog_revision — so the start-path resolver # consumes exactly what this packet published (no working-tree YAML, no client bytes). @@ -166,16 +181,24 @@ def publish_envelope() -> dict: raw = json.dumps(envelope, indent=1, ensure_ascii=False).encode("utf-8") validate_catalog(raw) target = GiteaTarget( - base_url=os.environ["PUBLISHED_CATALOG_GITEA_URL"].rstrip("/"), - repo=os.environ["PUBLISHED_CATALOG_REPO"], - ref=os.environ.get("PUBLISHED_CATALOG_REF", "master"), - token=os.environ["PUBLISHED_CATALOG_GITEA_TOKEN"], + base_url=os.environ["PUBLISHED_CATALOG_GITEA_URL"].strip().rstrip("/"), + repo=(os.environ.get("PUBLISHED_CATALOG_REPO") or os.environ["PUBLISHED_CATALOG_GITEA_REPO"]).strip().strip("/"), + ref=(os.environ.get("PUBLISHED_CATALOG_REF") or os.environ.get("PUBLISHED_CATALOG_GITEA_REF") or "main").strip(), + token=os.environ["PUBLISHED_CATALOG_GITEA_TOKEN"].strip(), path=catalog_path(pin["baseline_set_id"], pin["baseline_set_version"]), ) - headers = {"Authorization": f"token {os.environ['PUBLISHED_CATALOG_GITEA_TOKEN']}"} + headers = {"Authorization": f"token {target.token}"} with httpx.Client(base_url=target.base_url, headers=headers, timeout=30.0) as client: - commit = publish_catalog_bytes(client, target, raw, "Publish 044 published-envelope snapshot ss-prod-visual v1 (canary v4)") - return {"path": target.path, "commit_sha": str((commit.get("commit") or {}).get("sha") or "")} + commit = publish_catalog_bytes(client, target, raw, "Publish 037 published envelope ss-prod-visual v1 (canary v4)") + return { + "path": target.path, + "commit_sha": str((commit.get("commit") or {}).get("sha") or ""), + "baseline_set_id": envelope["baseline_set_id"], + "baseline_set_version": envelope["baseline_set_version"], + "release_id": envelope["release_id"], + "catalog_digest": envelope["catalog_digest"], + "catalog_revision_id": envelope["catalog_revision"].get("catalog_revision_id"), + } # #endregion ScenarioExecution.LiveCanaryV4.Publish @@ -262,11 +285,17 @@ def main() -> None: result_path = os.environ.get("SS_REPLAY_RESULT", "/tmp/kilo/live_canary_v4_result.json") with open(result_path, "w") as handle: json.dump(report, handle, indent=1, default=str) - # Fail-closed pin proof: the whole packet exists to prove the published-catalog pin end-to-end. + # Fail-closed pin proof: the plan pin must equal the persisted record pin AND carry the + # published envelope identity; a truthiness-only check would pass a divergent generation. assert isinstance(plan_pin, dict) and plan_pin.get("catalog_digest"), "runner_plan pin missing" assert evaluation is not None and evaluation.baseline_pin, "AgentEvaluation baseline_pin not stamped" + record_pin = evaluation.baseline_pin if isinstance(evaluation.baseline_pin, dict) else {} + assert plan_pin == record_pin, f"plan pin != AgentEvaluation pin: {plan_pin} != {record_pin}" + for field in ("baseline_set_id", "baseline_set_version", "release_id", "catalog_digest", "catalog_revision_id"): + assert plan_pin.get(field) == publication.get(field), f"pin {field} diverges from published envelope" if __name__ == "__main__": main() # #endregion ScenarioExecution.LiveCanaryV4.Run +# #endregion ScenarioExecution.LiveCanaryV4 diff --git a/specs/044-dashboard-scenario-execution/prototype/live_mcp_replay.py b/specs/044-dashboard-scenario-execution/prototype/live_mcp_replay.py index adc4fed1d..c592a8019 100644 --- a/specs/044-dashboard-scenario-execution/prototype/live_mcp_replay.py +++ b/specs/044-dashboard-scenario-execution/prototype/live_mcp_replay.py @@ -20,6 +20,7 @@ tests/test_mcp_initial_scenario_e2e.py — only the transport is live (httpx + S # tests (client_flow_http + initial_scenario_e2e), so only the transport is live here. # @REJECTED Hand-authoring the scenario graph was rejected — the derived-capability compile from live # context (MCPX-FR-028) is the acceptance target, and baseline refs stay out (PAT-blocked pin). +import argparse import base64 from hashlib import sha256 import json @@ -39,10 +40,12 @@ sys.path.insert(0, os.path.join(_REPO_ROOT, "backend")) import httpx # noqa: E402 from src.core.database import SessionLocal # noqa: E402 -from src.models.scenario_run import ScenarioStepRun # noqa: E402 +from src.models.scenario_run import ScenarioRun, ScenarioStepRun # noqa: E402 BASE = os.environ.get("SS_REPLAY_BASE", "http://127.0.0.1:8000") ENVIRONMENT_ID = "ss-prod" +BASELINE_SET = "ss-prod-visual" +BASELINE_SET_VERSION = "1" DASHBOARD_ID = 11 DASHBOARD_NAME = "Sales Dashboard" ACTOR = os.environ.get("SS_REPLAY_USER", "admin") @@ -186,7 +189,7 @@ class LiveMcpReplay: # (baseline capability stays off), so the gated start never needs the unpublished catalog. # @REJECTED Hand-building the scenario fixture like the 2026-09-07 run was rejected — T029m demands the # derived-capability compile from live context (MCPX-FR-028), not a client-authored graph. -def run_chain() -> dict: +def run_chain(baseline: bool = False) -> dict: replay = LiveMcpReplay() evidence: dict = {"base": BASE, "environment_id": ENVIRONMENT_ID, "dashboard_id": DASHBOARD_ID} @@ -218,14 +221,14 @@ def run_chain() -> dict: agent_run_id = agent_run["run_id"] evidence["agent_run_id"] = agent_run_id - capabilities = {**context["derived_capabilities"]["capabilities"], "screenshot": True, "baseline": False} + capabilities = {**context["derived_capabilities"]["capabilities"], "screenshot": True, "baseline": baseline} compiled = replay.call("inspect_scenario", {"request": { "agent_run_id": agent_run_id, "objective": {"goal": "t029m live sales smoke", "selected_case_ids": ["B01"], "rationale": "read-only external MCP replay"}, "query_model": context["query_model"], "checklist_catalog_version": CHECKLIST_CATALOG_VERSION, - "baseline_version": "unpinned-live-replay", + "baseline_version": BASELINE_SET_VERSION if baseline else "unpinned-live-replay", "capabilities": capabilities, "parameters": { "filter_values": {"type": "string_list", "required": True, "default": []}, @@ -277,14 +280,16 @@ def run_chain() -> dict: "activation_status": boot.get("activation_status")} start_key = f"t029m-run-{uuid.uuid4().hex[:12]}" - started = replay.call("start_scenario_run", {"request": { + start_request = { "scenario_id": boot["scenario_id"], "revision_id": boot["revision_id"], "environment_id": ENVIRONMENT_ID, "idempotency_key": start_key, - }}) - retry = replay.call("start_scenario_run", {"request": { - "scenario_id": boot["scenario_id"], "revision_id": boot["revision_id"], - "environment_id": ENVIRONMENT_ID, "idempotency_key": start_key, - }}) + } + if baseline: + # Explicit selector: the pin must resolve from the published Gitea envelope, not request bytes. + start_request["baseline_set"] = BASELINE_SET + start_request["baseline_set_version"] = BASELINE_SET_VERSION + started = replay.call("start_scenario_run", {"request": start_request}) + retry = replay.call("start_scenario_run", {"request": start_request}) evidence["start"] = {"status": started.get("status"), "run_id": started.get("run_id"), "retry_status": retry.get("status"), "retry_run_id": retry.get("run_id"), "error": started.get("error")} @@ -320,6 +325,17 @@ def run_chain() -> dict: evidence["terminal"] = {"status": status, "phase": detail.get("phase"), "error_code": detail.get("error_code"), "synthetic_pass_guard": "non_pass_confirmed"} evidence["steps"] = _step_report(run_id) + if baseline: + # T046 baseline-pin proof: the plan pin must come from the published catalog at start time. + with SessionLocal() as db: + plan_pin = (db.get(ScenarioRun, run_id).runner_plan or {}).get("baseline_pin") + evidence["baseline_pin"] = { + "baseline_set_id": (plan_pin or {}).get("baseline_set_id"), + "baseline_set_version": (plan_pin or {}).get("baseline_set_version"), + "catalog_digest": (plan_pin or {}).get("catalog_digest"), + } + if not isinstance(plan_pin, dict) or not plan_pin.get("catalog_digest"): + raise ReplayError("baseline pin was not resolved through the MCP start") return evidence @@ -339,10 +355,14 @@ def _step_report(run_id: str) -> list[dict]: # #region ScenarioExecution.LiveMcpReplay.Main [C:2] [TYPE Function] [SEMANTICS replay,entrypoint,report] -# @BRIEF Entry point: run the chain, print and dump the evidence JSON (fail-closed on ReplayError). +# @BRIEF Entry point: run the chain (optionally with the published baseline selector), dump evidence. def main() -> None: + parser = argparse.ArgumentParser(description="T029m live external MCP replay (optionally baseline-pinned).") + parser.add_argument("--baseline", action="store_true", + help="compile with the baseline capability and start with the published-catalog selector") + args = parser.parse_args() try: - evidence = run_chain() + evidence = run_chain(baseline=args.baseline) evidence["result"] = "ok" except ReplayError as exc: evidence = {"result": "blocked", "error": str(exc)} diff --git a/specs/044-dashboard-scenario-execution/tasks.md b/specs/044-dashboard-scenario-execution/tasks.md index 241c5e235..8ebbdd6e8 100644 --- a/specs/044-dashboard-scenario-execution/tasks.md +++ b/specs/044-dashboard-scenario-execution/tasks.md @@ -243,10 +243,10 @@ Historical [x] rows above retain only their dated local/transport evidence; they Contract: [Production baseline-backed evaluation](contracts/production-chain.md). -- [ ] T043 [P0/P1/P2] Baseline set/version/catalog/release/commit/IDs/digests affect idempotency; moving catalog after admission cannot change plan/result; legacy unpinned result is ineligible. Implement at the existing 044 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-10):** request-hash pinning + fail-closed resolver landed offline (commits `83727aa7`/`bbbd4ccf`, `test_start_run_server_authority.py`); the caller-side published-catalog source is wired. Live pin-from-Gitea trace BLOCKED on a valid Gitea PAT (provided token rejected 401). +- [ ] T043 [P0/P1/P2] Baseline set/version/catalog/release/commit/IDs/digests affect idempotency; moving catalog after admission cannot change plan/result; legacy unpinned result is ineligible. Implement at the existing 044 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11):** request-hash pinning + fail-closed resolver landed offline (`83727aa7`/`bbbd4ccf`); caller-side published-catalog source wired. Live pin-from-Gitea is now PROVEN: REST canary v4 (`ace916a0…`/`adeabe63…`) + MCP `--baseline` chain resolve the published-envelope pin and the walker stamps it into `AgentEvaluation.baseline_pin` (strict equality) — `docs/reports/agentic-runtime-live-canary-v4-baseline-pin-2026-09-11.md`. Residual: the graph-level deterministic comparison PASS and the MCP publish tool are separate rows (050 T045). - [ ] T044 [P0/P1/P2] GET/HEAD prove same ACL/status/headers; MIME/digest/length checked before bytes, cross-owner hidden, expired410, corrupt409, traversal/range/oversize rejected. Implement at the existing 044 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-10):** Slice G GET/HEAD runtime landed and committed (`83727aa7`, `scenario_artifact_content.py`); live storage canary proved durable bytes with digests (canary v1 `597274d3`). Live ACL/status/header canary not yet exercised. - [ ] T045 [P0/P1/P2] Startup/readiness/start-loop and shutdown/drain/cancel/reconcile survive fault injection; unknown effect quarantines capacity; late response cannot win. Implement at the existing 044 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-10):** startup/readiness live-verified (provider loop ready, readiness preflight; browser-probe deadlock fixed `ba2f1f45`); capacity leases claimed/released across the live canaries. Fault-injection drain/cancel/reconcile canary not yet exercised. -- [ ] T046 [P0/P1/P2] End-to-end real browser→capture→durable artifact→deterministic comparison→optional immutable evaluation→policy→result; all required evidence present before PASS. Implement at the existing 044 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-10):** the chain real browser→capture→durable artifact→immutable evaluation→policy→result is PROVEN live (canary v2 `4eebfab3`: `AgentEvaluation` persisted, DecisionPolicy row 11). Still missing: the deterministic comparison step in a live graph and a live baseline pin (Gitea PAT). Note: the canary's typed inconclusive outcome is the honest verdict while prompts remain text-only (images not attached). +- [ ] T046 [P0/P1/P2] End-to-end real browser→capture→durable artifact→deterministic comparison→optional immutable evaluation→policy→result; all required evidence present before PASS. Implement at the existing 044 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11):** the chain real browser→capture→durable artifact→immutable evaluation→policy→result is PROVEN live; a deterministic comparison step and a live baseline pin are now both in live graphs (canary v4: `compare_to_baseline` deterministically passes; pin resolved from the published envelope and stamped into `AgentEvaluation.baseline_pin`, `docs/reports/agentic-runtime-live-canary-v4-baseline-pin-2026-09-11.md`). Residual: a graph-level terminal PASS (the baseline-semantic policy marks comparisons without a bound evaluation as typed `EVALUATION_UNAVAILABLE`; `evaluate-visual` itself passes). Frontend boundary for this package: manual CRUD/editor, human review/approval, monitoring and read-only evidence/evaluation only; all agent interaction is external MCP. No agent chat/prompt/assistant editing/proposal generation/workspace/start/handoff controls. Runtime removal is OPEN, not performed by this spec refresh. Optional approved performance baseline is outside scope. diff --git a/specs/044-dashboard-scenario-execution/traceability.md b/specs/044-dashboard-scenario-execution/traceability.md index b9b30f89f..f9abd1274 100644 --- a/specs/044-dashboard-scenario-execution/traceability.md +++ b/specs/044-dashboard-scenario-execution/traceability.md @@ -93,7 +93,7 @@ surfaces outside the 050 MCP catalog (no spec row existed before this note): | Seam | Contract | Surface | Evidence | |---|---|---|---| | `Api.ScenarioLiveBindings` | `ScenarioExecution.LiveBinding` + `Core.ConfigManager` | admin REST `GET/PUT/PATCH /api/scenario-live-bindings` (`admin:settings`); validates `LiveExecutionBinding.from_snapshot` + `DashboardQueryModel` (env/dashboard/fingerprint) and writes `settings.scenario_live_execution_bindings` atomically | `backend/tests/api/test_scenario_live_bindings_api.py` (7); first live exercise: re-registered `ss-prod-d11-live-001` with the correct principal, then the T029m run adopted it server-side | -| `Tooling.PublishCatalog` | T045/D7 published-catalog source (`PUBLISHED_CATALOG_*`) | CLI `backend/src/scripts/publish_catalog.py` — pre-validates catalog bytes, then Gitea contents PUT (typed auth/conflict/unavailable) | `backend/tests/scripts/test_publish_catalog.py` (7, offline); publication itself blocked on a valid Gitea PAT | +| `Tooling.PublishCatalog` | T045/D7 published-catalog source (`PUBLISHED_CATALOG_*`) | CLI `backend/src/scripts/publish_catalog.py` — validates the resolver-canonical envelope, then Gitea contents create=POST/update=PUT (typed auth/conflict/unavailable) | `backend/tests/scripts/test_publish_catalog.py` (8, offline); LIVE 2026-09-11: create `e41d6ad2`, update-sha `3bb2c63e`, envelope `4d5b7e4c`/`3f782574`, publish-failure canary typed `PUBLISH_AUTH_FAILED`. 050 T045 stays OPEN for the curated MCP `publish_baseline_catalog` (MCPX-FR-030) | Cross-reference: the live external MCP replay that exercised the binding path end-to-end (and exposed the identity-less-compiled-step defect) is `docs/2026-09-11-sales-prod-mcp-replay.md` (050 T029m / diff --git a/specs/050-mcp-interface/tasks.md b/specs/050-mcp-interface/tasks.md index 7adfcf235..3ae5d0c58 100644 --- a/specs/050-mcp-interface/tasks.md +++ b/specs/050-mcp-interface/tasks.md @@ -93,7 +93,7 @@ Historical [x] rows above retain only their dated local/transport evidence; they Contract: [Contract-complete public parity](contracts/modules.md). - [ ] T044 [P0/P1/P2] REST/MCP lifecycle/read/auth errors and disabled automation validation are identical; service principal cannot decide human gate. Implement at the existing 050 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11):** offline parity tests green; live REST-only canary v2 exercised start→gate→approve→dispatch (`4eebfab3`); the live MCP-external replay (T029m CLOSED, `docs/2026-09-11-sales-prod-mcp-replay.md`) exercised the same lifecycle through `/mcp` with the identical typed start/gate/terminal outcomes — remaining: dedicated REST-vs-MCP error-shape parity fixtures for this matrix. -- [x] T045 [P0/P1/P2] consume/publish failures return typed errors/pending state with no legacy fallback; every prerequisite is externally MCP-reachable. Implement at the existing 050 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11): CLOSED.** Consume path typed fail-closed (`bbbd4ccf`); publisher `backend/src/scripts/publish_catalog.py` with pre-validation before any network I/O and typed auth/conflict/unavailable (offline `tests/scripts/test_publish_catalog.py`); LIVE evidence (`docs/reports/agentic-runtime-live-canary-v4-baseline-pin-2026-09-11.md`): create `e41d6ad2` (POST), update-sha `3bb2c63e` (PUT), publish-failure canary with a bogus token → typed `PUBLISH_AUTH_FAILED` (HTTP 401, exit 1, no partial state); every prerequisite externally reachable (T029m CLOSED). Defects found and fixed live: Gitea create=POST/update=PUT contract; `run.sh` wiping deployment keys below the MCP_JWT line on every start (venv-python validation fallback + single-line in-place rewrite). -- [x] T046 [P0/P1/P2] Fresh external-client chain preserves authoritative context and complete baseline pin without raw ORM/REST repair; no frontend agent controls/routes/requests. Implement at the existing 050 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11): CLOSED.** Fresh REST chain (canary v2 `4eebfab3`) and fresh MCP-external chain (T029m) preserve the server-resolved binding and verified authoritative context (`context_authority=verified`); the complete baseline pin is proven live by canary v4 (run `ace916a0…`, `docs/reports/agentic-runtime-live-canary-v4-baseline-pin-2026-09-11.md`): selector-driven resolution from the PUBLISHED Gitea envelope → `runner_plan.baseline_pin` == `AgentEvaluation.baseline_pin` (walker stamping, commit `792bb125`), no raw ORM/REST repair; frontend agent controls remain absent (2026-09-10 evidence). +- [ ] T045 [P0/P1/P2] consume/publish failures return typed errors/pending state with no legacy fallback; every prerequisite is externally MCP-reachable. Implement at the existing 050 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11): OPEN — partially delivered.** CLOSED parts: consume path typed fail-closed (`bbbd4ccf`, live-proven); CLI publisher `backend/src/scripts/publish_catalog.py` with pre-validation and typed auth/conflict/unavailable (offline tests + LIVE create/update/publish-failure canary, `docs/reports/agentic-runtime-live-canary-v4-baseline-pin-2026-09-11.md`). REMAINING (the reason this stays OPEN): MCPX-FR-030 requires a curated MCP `publish_baseline_catalog` operation on the 037 publication-worker contract (explicit authorized intent, `expected_branch_head` CAS, commit/pushed receipt, reconcile) — `backend/src/mcp_server/` registers no publish tool, and the CLI is a plain contents write, so "every prerequisite is externally MCP-reachable" is NOT yet true for publication. +- [x] T046 [P0/P1/P2] Fresh external-client chain preserves authoritative context and complete baseline pin without raw ORM/REST repair; no frontend agent controls/routes/requests. Implement at the existing 050 domain boundary; verify with independent hardcoded fixtures and retain command/evidence references in traceability.md. **Status (2026-09-11): CLOSED.** Context authority: fresh REST chain (canary v2 `4eebfab3`) and fresh MCP chain (T029m) preserved the server-resolved binding and verified context (`context_authority=verified`). Complete baseline pin: (a) the MCP-external chain with `--baseline` (`live_mcp_replay.py --baseline`, no raw ORM seeding) compiled the baseline capability, started with the published-catalog selector, and carried a full `runner_plan.baseline_pin` (set/version/digest from the published envelope); (b) the REST canary v4 (`ace916a0…`, then re-run `adeabe63…`) proved the walker stamps that same pin into the persisted `AgentEvaluation.baseline_pin` (`plan pin == record pin`, strict equality asserted) — `docs/reports/agentic-runtime-live-canary-v4-baseline-pin-2026-09-11.md`. No raw ORM/REST pin repair; frontend agent controls remain absent (2026-09-10 evidence). Frontend boundary for this package: manual CRUD/editor, human review/approval, monitoring and read-only evidence/evaluation only; all agent interaction is external MCP. No agent chat/prompt/assistant editing/proposal generation/workspace/start/handoff controls. Runtime removal is OPEN, not performed by this spec refresh. Optional approved performance baseline is outside scope.