From 2eca5b514ba5e7a93bdcd0d1cbaa8d37745f4086 Mon Sep 17 00:00:00 2001 From: busya Date: Mon, 13 Jul 2026 15:10:07 +0300 Subject: [PATCH] docs(specs): complete speckit packages 036-039 --- .../checklists/requirements.md | 9 + .../contracts/agent-runs.openapi.yaml | 259 ++++++++++++++++++ .../contracts/events.md | 72 +++++ .../contracts/modules.md | 177 ++++++++++++ .../contracts/ux/agent-run-ux.md | 34 +++ .../contracts/ux/alternatives.md | 16 ++ .../contracts/ux/api-ux.md | 29 ++ .../contracts/ux/decisions.md | 15 + .../contracts/ux/design-tokens.md | 17 ++ .../contracts/ux/screen-models.md | 40 +++ .../data-model.md | 116 ++++++++ specs/036-agent-test-stabilization/plan.md | 80 ++++++ .../quickstart.md | 50 ++++ .../036-agent-test-stabilization/research.md | 73 +++++ specs/036-agent-test-stabilization/spec.md | 2 +- specs/036-agent-test-stabilization/tasks.md | 80 ++++++ .../traceability.md | 29 ++ .../ux_reference.md | 2 +- .../checklists/requirements.md | 9 + .../contracts/baseline-catalog.schema.json | 105 +++++++ .../contracts/dashboard-testing.openapi.yaml | 250 +++++++++++++++++ .../contracts/modules.md | 129 +++++++++ .../contracts/ux/api-ux.md | 17 ++ .../contracts/ux/baseline-engine-ux.md | 28 ++ .../contracts/ux/decisions.md | 12 + .../data-model.md | 116 ++++++++ specs/037-superset-baseline-engine/plan.md | 76 +++++ .../quickstart.md | 36 +++ .../037-superset-baseline-engine/research.md | 81 ++++++ specs/037-superset-baseline-engine/spec.md | 2 +- specs/037-superset-baseline-engine/tasks.md | 68 +++++ .../traceability.md | 18 ++ .../ux_reference.md | 2 +- .../checklist-catalog.md | 58 ++++ .../checklists/requirements.md | 9 + .../dashboard-test-scenario.schema.json | 178 ++++++++++++ .../contracts/modules.md | 114 ++++++++ .../contracts/scenario-api.openapi.yaml | 144 ++++++++++ .../contracts/ux/api-ux.md | 13 + .../contracts/ux/decisions.md | 12 + .../contracts/ux/scenario-graph-ux.md | 24 ++ .../data-model.md | 87 ++++++ specs/038-dashboard-scenario-model/plan.md | 71 +++++ .../quickstart.md | 33 +++ .../038-dashboard-scenario-model/research.md | 87 ++++++ specs/038-dashboard-scenario-model/spec.md | 2 +- specs/038-dashboard-scenario-model/tasks.md | 67 +++++ .../traceability.md | 19 ++ .../ux_reference.md | 2 +- .../checklists/requirements.md | 9 + .../contracts/modules.md | 114 ++++++++ .../contracts/ux/alternatives.md | 15 + .../contracts/ux/api-ux.md | 27 ++ .../contracts/ux/dashboard-scenario-ux.md | 33 +++ .../contracts/ux/decisions.md | 15 + .../contracts/ux/design-tokens.md | 18 ++ .../contracts/ux/screen-models.md | 29 ++ specs/039-dashboard-scenario-ui/data-model.md | 56 ++++ specs/039-dashboard-scenario-ui/plan.md | 69 +++++ specs/039-dashboard-scenario-ui/quickstart.md | 41 +++ specs/039-dashboard-scenario-ui/research.md | 68 +++++ specs/039-dashboard-scenario-ui/spec.md | 2 +- specs/039-dashboard-scenario-ui/tasks.md | 74 +++++ .../039-dashboard-scenario-ui/traceability.md | 23 ++ .../039-dashboard-scenario-ui/ux_reference.md | 2 +- 65 files changed, 3556 insertions(+), 8 deletions(-) create mode 100644 specs/036-agent-test-stabilization/contracts/agent-runs.openapi.yaml create mode 100644 specs/036-agent-test-stabilization/contracts/events.md create mode 100644 specs/036-agent-test-stabilization/contracts/modules.md create mode 100644 specs/036-agent-test-stabilization/contracts/ux/agent-run-ux.md create mode 100644 specs/036-agent-test-stabilization/contracts/ux/alternatives.md create mode 100644 specs/036-agent-test-stabilization/contracts/ux/api-ux.md create mode 100644 specs/036-agent-test-stabilization/contracts/ux/decisions.md create mode 100644 specs/036-agent-test-stabilization/contracts/ux/design-tokens.md create mode 100644 specs/036-agent-test-stabilization/contracts/ux/screen-models.md create mode 100644 specs/036-agent-test-stabilization/data-model.md create mode 100644 specs/036-agent-test-stabilization/plan.md create mode 100644 specs/036-agent-test-stabilization/quickstart.md create mode 100644 specs/036-agent-test-stabilization/research.md create mode 100644 specs/036-agent-test-stabilization/tasks.md create mode 100644 specs/036-agent-test-stabilization/traceability.md create mode 100644 specs/037-superset-baseline-engine/contracts/baseline-catalog.schema.json create mode 100644 specs/037-superset-baseline-engine/contracts/dashboard-testing.openapi.yaml create mode 100644 specs/037-superset-baseline-engine/contracts/modules.md create mode 100644 specs/037-superset-baseline-engine/contracts/ux/api-ux.md create mode 100644 specs/037-superset-baseline-engine/contracts/ux/baseline-engine-ux.md create mode 100644 specs/037-superset-baseline-engine/contracts/ux/decisions.md create mode 100644 specs/037-superset-baseline-engine/data-model.md create mode 100644 specs/037-superset-baseline-engine/plan.md create mode 100644 specs/037-superset-baseline-engine/quickstart.md create mode 100644 specs/037-superset-baseline-engine/research.md create mode 100644 specs/037-superset-baseline-engine/tasks.md create mode 100644 specs/037-superset-baseline-engine/traceability.md create mode 100644 specs/038-dashboard-scenario-model/checklist-catalog.md create mode 100644 specs/038-dashboard-scenario-model/contracts/dashboard-test-scenario.schema.json create mode 100644 specs/038-dashboard-scenario-model/contracts/modules.md create mode 100644 specs/038-dashboard-scenario-model/contracts/scenario-api.openapi.yaml create mode 100644 specs/038-dashboard-scenario-model/contracts/ux/api-ux.md create mode 100644 specs/038-dashboard-scenario-model/contracts/ux/decisions.md create mode 100644 specs/038-dashboard-scenario-model/contracts/ux/scenario-graph-ux.md create mode 100644 specs/038-dashboard-scenario-model/data-model.md create mode 100644 specs/038-dashboard-scenario-model/plan.md create mode 100644 specs/038-dashboard-scenario-model/quickstart.md create mode 100644 specs/038-dashboard-scenario-model/research.md create mode 100644 specs/038-dashboard-scenario-model/tasks.md create mode 100644 specs/038-dashboard-scenario-model/traceability.md create mode 100644 specs/039-dashboard-scenario-ui/contracts/modules.md create mode 100644 specs/039-dashboard-scenario-ui/contracts/ux/alternatives.md create mode 100644 specs/039-dashboard-scenario-ui/contracts/ux/api-ux.md create mode 100644 specs/039-dashboard-scenario-ui/contracts/ux/dashboard-scenario-ux.md create mode 100644 specs/039-dashboard-scenario-ui/contracts/ux/decisions.md create mode 100644 specs/039-dashboard-scenario-ui/contracts/ux/design-tokens.md create mode 100644 specs/039-dashboard-scenario-ui/contracts/ux/screen-models.md create mode 100644 specs/039-dashboard-scenario-ui/data-model.md create mode 100644 specs/039-dashboard-scenario-ui/plan.md create mode 100644 specs/039-dashboard-scenario-ui/quickstart.md create mode 100644 specs/039-dashboard-scenario-ui/research.md create mode 100644 specs/039-dashboard-scenario-ui/tasks.md create mode 100644 specs/039-dashboard-scenario-ui/traceability.md diff --git a/specs/036-agent-test-stabilization/checklists/requirements.md b/specs/036-agent-test-stabilization/checklists/requirements.md index a43247482..128c64ee8 100644 --- a/specs/036-agent-test-stabilization/checklists/requirements.md +++ b/specs/036-agent-test-stabilization/checklists/requirements.md @@ -24,3 +24,12 @@ - [x] CHK010 Required outputs are limited to agent runtime stabilization artifacts. - [x] CHK011 Feature explicitly excludes Superset query baseline engine and scenario generation logic. - [x] CHK012 Success criteria are measurable by backend/frontend tests and one live smoke flow. + +## Implementation Package + +- [x] CHK013 Research and implementation plan resolve all design decisions. +- [x] CHK014 Data model, module, event, OpenAPI, and UX contracts are present. +- [x] CHK015 Quickstart defines a safe vertical-slice verification path. +- [x] CHK016 Traceability maps every functional requirement to contracts, tasks, and tests. +- [x] CHK017 Tasks use exact repository paths, dependency order, and test-first sequencing. +- [x] CHK018 Machine-readable contracts and semantic anchors pass validation. diff --git a/specs/036-agent-test-stabilization/contracts/agent-runs.openapi.yaml b/specs/036-agent-test-stabilization/contracts/agent-runs.openapi.yaml new file mode 100644 index 000000000..231115bd1 --- /dev/null +++ b/specs/036-agent-test-stabilization/contracts/agent-runs.openapi.yaml @@ -0,0 +1,259 @@ +openapi: 3.1.0 +info: + title: Agent Run Recovery API + version: 0.1.0 + description: Durable run, event, draft, and approval substrate for dashboard scenario agent flows. +servers: + - url: /api +paths: + /agent/runs: + post: + operationId: createAgentRun + security: [{ bearerAuth: [] }, { serviceAndUserAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/CreateAgentRunRequest' } + responses: + '201': + description: Created + content: { application/json: { schema: { $ref: '#/components/schemas/AgentRunSnapshot' } } } + '403': { $ref: '#/components/responses/Forbidden' } + '422': { $ref: '#/components/responses/Invalid' } + /agent/runs/{runId}: + get: + operationId: getAgentRun + security: [{ bearerAuth: [] }] + parameters: + - $ref: '#/components/parameters/RunId' + responses: + '200': + description: Ownership-scoped authoritative snapshot + content: { application/json: { schema: { $ref: '#/components/schemas/AgentRunSnapshot' } } } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { description: Not found } + /agent/runs/{runId}/events: + post: + operationId: appendAgentRunEvent + security: [{ serviceAndUserAuth: [] }] + parameters: + - $ref: '#/components/parameters/RunId' + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/AppendEventRequest' } + responses: + '201': + description: Appended + content: { application/json: { schema: { $ref: '#/components/schemas/AgentRunEvent' } } } + '409': { description: Sequence, stage, or terminal-state conflict } + '422': { $ref: '#/components/responses/Invalid' } + /agent/runs/{runId}/artifacts: + post: + operationId: registerDraftArtifact + security: [{ serviceAndUserAuth: [] }] + parameters: + - $ref: '#/components/parameters/RunId' + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + required: [metadata, file] + properties: + metadata: + $ref: '#/components/schemas/RegisterDraftMetadata' + file: + type: string + format: binary + responses: + '201': + description: Draft stored outside repository + content: { application/json: { schema: { $ref: '#/components/schemas/DraftArtifactRef' } } } + '413': { description: Draft too large } + '422': { $ref: '#/components/responses/Invalid' } + /agent/runs/{runId}/approval-gates: + post: + operationId: requestApprovalGate + security: [{ serviceAndUserAuth: [] }] + parameters: + - $ref: '#/components/parameters/RunId' + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/ApprovalRequest' } + responses: + '201': + description: Bound pending gate + content: { application/json: { schema: { $ref: '#/components/schemas/ApprovalGateView' } } } + '403': { $ref: '#/components/responses/Forbidden' } + /agent/runs/{runId}/approval-gates/{gateId}/decision: + post: + operationId: decideApprovalGate + security: [{ bearerAuth: [] }] + parameters: + - $ref: '#/components/parameters/RunId' + - name: gateId + in: path + required: true + schema: { type: string, format: uuid } + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [decision] + properties: + decision: { type: string, enum: [confirm, deny] } + reason: { type: [string, 'null'], maxLength: 2000 } + responses: + '200': + description: Immutable decision recorded + content: { application/json: { schema: { $ref: '#/components/schemas/ApprovalGateView' } } } + '409': { description: Gate already decided or expired } + '422': { $ref: '#/components/responses/Invalid' } + /agent/runs/{runId}/approval-gates/{gateId}/consume: + post: + operationId: consumeApprovalGate + description: Internal execution endpoint; recalculates request hash and RBAC before one side effect. + security: [{ serviceAndUserAuth: [] }] + parameters: + - $ref: '#/components/parameters/RunId' + - name: gateId + in: path + required: true + schema: { type: string, format: uuid } + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [operationPayload] + properties: + operationPayload: { type: object, additionalProperties: true } + responses: + '200': { description: Side effect committed and gate consumed } + '403': { $ref: '#/components/responses/Forbidden' } + '409': { description: Hash, actor, status, expiry, or replay conflict } +components: + securitySchemes: + bearerAuth: { type: http, scheme: bearer, bearerFormat: JWT } + serviceAndUserAuth: + type: apiKey + in: header + name: X-Service-Authorization + description: Requires service identity plus Authorization user JWT. + parameters: + RunId: + name: runId + in: path + required: true + schema: { type: string, format: uuid } + responses: + Forbidden: { description: Ownership or permission denied } + Invalid: { description: Validation failed } + schemas: + UIContextV2: + type: object + additionalProperties: false + required: [objectType, objectId, envId, route, contextVersion, intent] + properties: + objectType: { const: dashboard } + objectId: { type: string, pattern: '^[0-9]{1,20}$' } + objectName: { type: [string, 'null'], maxLength: 256 } + envId: { type: string, minLength: 1, maxLength: 128 } + route: { type: string, pattern: '^/dashboards/', maxLength: 512 } + contextVersion: { const: 2 } + intent: { const: build_dashboard_test_scenario } + CreateAgentRunRequest: + type: object + required: [conversationId, context, idempotencyKey] + properties: + conversationId: { type: string, minLength: 1 } + context: { $ref: '#/components/schemas/UIContextV2' } + idempotencyKey: { type: string, minLength: 16, maxLength: 128 } + AppendEventRequest: + type: object + required: [sequence, eventType, stage, status, payloadHash] + properties: + sequence: { type: integer, minimum: 1 } + eventType: + type: string + enum: [run_started, progress, drafts_updated, approval_requested, approval_resolved, terminal] + stage: { type: string, enum: [context, inspect, scenario, parameters, generate, validate, save] } + status: { type: string, enum: [pending, active, completed, blocked, failed, skipped] } + payload: { type: object, additionalProperties: true } + payloadHash: { type: string, pattern: '^[a-f0-9]{64}$' } + RegisterDraftMetadata: + type: object + required: [kind, name, intendedPath, sha256, validationStatus] + properties: + kind: { type: string, enum: [scenario, runner_plan, report_template, baseline_candidate, evidence_manifest, other] } + name: { type: string, maxLength: 255 } + intendedPath: { type: string, maxLength: 1024 } + sha256: { type: string, pattern: '^[a-f0-9]{64}$' } + validationStatus: { type: string, enum: [valid, warning, invalid, pending] } + warnings: { type: array, items: { type: object } } + ApprovalRequest: + type: object + required: [operation, artifactIds, targetPaths] + properties: + operation: { type: string, enum: [repository_write, baseline_approval] } + artifactIds: { type: array, minItems: 1, uniqueItems: true, items: { type: string, format: uuid } } + targetPaths: { type: array, minItems: 1, uniqueItems: true, items: { type: string } } + warnings: { type: array, items: { type: object } } + AgentRunEvent: + allOf: + - $ref: '#/components/schemas/AppendEventRequest' + - type: object + required: [id, runId, occurredAt] + properties: + id: { type: string, format: uuid } + runId: { type: string, format: uuid } + occurredAt: { type: string, format: date-time } + DraftArtifactRef: + type: object + required: [artifactId, kind, name, intendedPath, sha256, validationStatus] + properties: + artifactId: { type: string, format: uuid } + kind: { type: string } + name: { type: string } + intendedPath: { type: string } + sha256: { type: string } + validationStatus: { type: string } + warnings: { type: array, items: { type: object } } + previewUrl: { type: string } + downloadUrl: { type: string } + ApprovalGateView: + type: object + required: [gateId, operation, status, targetPaths, reasonRequired, expiresAt] + properties: + gateId: { type: string, format: uuid } + operation: { type: string } + status: { type: string, enum: [pending, confirmed, denied, consumed, expired] } + targetPaths: { type: array, items: { type: string } } + reasonRequired: { type: boolean } + reason: { type: [string, 'null'] } + expiresAt: { type: string, format: date-time } + AgentRunSnapshot: + type: object + required: [agentRunId, conversationId, status, currentStage, lastSequence, context, events, artifacts] + properties: + agentRunId: { type: string, format: uuid } + conversationId: { type: string } + status: { type: string, enum: [created, running, waiting_input, waiting_approval, completed, failed, cancelled] } + currentStage: { type: string } + lastSequence: { type: integer } + context: { $ref: '#/components/schemas/UIContextV2' } + events: { type: array, items: { $ref: '#/components/schemas/AgentRunEvent' } } + artifacts: { type: array, items: { $ref: '#/components/schemas/DraftArtifactRef' } } + pendingGate: + oneOf: + - $ref: '#/components/schemas/ApprovalGateView' + - type: 'null' diff --git a/specs/036-agent-test-stabilization/contracts/events.md b/specs/036-agent-test-stabilization/contracts/events.md new file mode 100644 index 000000000..f9adea3c3 --- /dev/null +++ b/specs/036-agent-test-stabilization/contracts/events.md @@ -0,0 +1,72 @@ +#region AgentTestStabilization.Events [C:4] [TYPE ADR] [SEMANTICS agent-run,event,gradio,stream] +@BRIEF Typed Gradio metadata protocol for scenario runs; backend persistence is authoritative. +@RELATION DEPENDS_ON -> [AgentTestStabilization.DataModel] +@RELATION CALLED_BY -> [AgentRuns.Gradio.Emit] + +## Common Envelope + +Every new metadata event contains: + +| Field | Type | Rule | +|---|---|---| +| type | string enum | Event discriminator | +| agent_run_id | UUID | Stable run correlation | +| sequence | positive integer | Monotonic per run | +| occurred_at | ISO-8601 UTC | Server time | +| conversation_id | string | Existing chat correlation | + +## agent_run_started + +~~~json +{ + "type": "agent_run_started", + "agent_run_id": "8a0c5aa4-beb6-4c0e-baf5-3fef93a2430a", + "sequence": 1, + "occurred_at": "2026-07-13T10:00:00Z", + "conversation_id": "conv-42", + "intent": "build_dashboard_test_scenario", + "stage": "context", + "status": "active" +} +~~~ + +Must be persisted/emitted before any tool_start for the scenario run. + +## scenario_progress + +Adds stage, status, message_code, completed_steps, total_steps, and optional retry_of_sequence. message_code is localized by frontend; prose is not interpreted as state. + +## draft_artifacts + +Adds artifacts: array of artifact_id, kind, name, intended_path, sha256, validation_status, warnings, preview_url, download_url. No raw bytes or storage paths. + +## agent_run_terminal + +Adds status (completed/failed/cancelled), stage, error_code, recoverable, and optional safe recovery actions. + +## confirm_required Extension + +Existing fields remain. Scenario operations add: + +| Field | Rule | +|---|---| +| agent_run_id, approval_gate_id | Required | +| approval_kind | repository_write or baseline_approval | +| target_paths | Exact normalized path list | +| artifact_ids | Exact draft set | +| warnings | Visible structured warnings | +| request_hash | Short display fingerprint; full hash remains server-side | +| reason_required | True for baseline approval | + +## permission_denied Extension + +May include agent_run_id, operation, required_permission, and alternatives. It never includes approval_gate_id and never enters the confirmation checkpoint. + +## Ordering and Recovery + +- Frontend ignores duplicate or older sequence values. +- A sequence gap marks the projection degraded and triggers snapshot recovery. +- Unknown event type is logged and ignored without changing run FSM. +- Stream loss never changes backend status. + +#endregion AgentTestStabilization.Events diff --git a/specs/036-agent-test-stabilization/contracts/modules.md b/specs/036-agent-test-stabilization/contracts/modules.md new file mode 100644 index 000000000..73a0d725c --- /dev/null +++ b/specs/036-agent-test-stabilization/contracts/modules.md @@ -0,0 +1,177 @@ +#region AgentTestStabilization.Modules [C:5] [TYPE ADR] [SEMANTICS contracts,agent-run,dashboard-testing,hitl] +@BRIEF Implementation contracts for durable agent runs, scenario events, drafts, approvals, and frontend recovery. +@RELATION DEPENDS_ON -> [AgentTestStabilization.DataModel] +@RELATION DEPENDS_ON -> [AgentTestStabilization.Events] +@RATIONALE Durable state and side-effect authorization are split from Gradio streaming so worker restarts cannot erase audit truth. +@REJECTED Persisting authoritative run state only in Gradio or frontend memory — rejected because both are restartable clients. + +## Backend + +# #region AgentRuns.Api [C:4] [TYPE Module] [SEMANTICS agent-run,api,ownership,rbac] +# @defgroup AgentRuns Ownership-scoped REST surface for scenario run recovery and internal event ingestion. +# @LAYER API +# @RELATION DEPENDS_ON -> [AgentRuns.Service] +# @RELATION DEPENDS_ON -> [AgentRuns.Schemas] +# @INVARIANT Browser reads require run ownership or admin permission; internal writes require service identity plus propagated user identity. +# @REJECTED Trusting user_id from request JSON — actor identity must come from validated auth. +# #endregion AgentRuns.Api + +# #region AgentRuns.Service.Create [C:4] [TYPE Function] [SEMANTICS agent-run,create,context] +# @ingroup AgentRuns +# @BRIEF Create one durable run for a validated dashboard-scenario context. +# @PRE UIContext v2 is valid; actor has dashboard:testing EXECUTE. +# @POST Persists CREATED then RUNNING run and returns id before any scenario tool action. +# @SIDE_EFFECT Writes AgentRun and initial event in one transaction. +# @DATA_CONTRACT CreateAgentRunRequest -> AgentRunSnapshot +# @TEST_EDGE duplicate_idempotency_key -> same actor/context returns same run. +# @TEST_EDGE context_actor_mismatch -> 403 and no row. +# @TEST_EDGE invalid_v1_scenario_context -> 422. +# #endregion AgentRuns.Service.Create + +# #region AgentRuns.Service.AppendEvent [C:5] [TYPE Function] [SEMANTICS agent-run,event,sequence,state] +# @ingroup AgentRuns +# @BRIEF Append a typed event and advance run lifecycle under monotonic sequence rules. +# @PRE Run is non-terminal; event validates against event_type payload schema. +# @POST Sequence increases exactly once; status/stage projection matches the appended event. +# @SIDE_EFFECT Inserts AgentRunEvent and updates AgentRun atomically. +# @INVARIANT A duplicate sequence is idempotent only when its canonical hash matches. +# @DATA_CONTRACT AppendEventRequest -> AgentRunEventResponse +# @TEST_EDGE stage_regression -> 409 without mutation. +# @TEST_EDGE same_sequence_same_hash -> existing event returned. +# @TEST_EDGE same_sequence_different_hash -> 409 conflict. +# @RATIONALE Transactional projection prevents the snapshot and event log from disagreeing after a crash. +# @REJECTED Client-side sequence authority — concurrent writers could fork history. +# #endregion AgentRuns.Service.AppendEvent + +# #region AgentRuns.Artifacts.Register [C:5] [TYPE Function] [SEMANTICS agent-run,draft,artifact,storage] +# @ingroup AgentRuns +# @BRIEF Store draft bytes outside the repository and register safe preview metadata. +# @PRE Run is owned and active; path and size pass policy; provided digest matches bytes. +# @POST Draft is retrievable by opaque id; target repository remains unchanged. +# @SIDE_EFFECT Writes draft storage and DraftArtifact record. +# @DATA_CONTRACT RegisterDraftRequest + bytes -> DraftArtifactRef +# @INVARIANT content_ref and filesystem path are never returned to clients. +# @TEST_EDGE traversal_path -> 422 before write. +# @TEST_EDGE digest_mismatch -> draft bytes removed and 422. +# @TEST_EDGE invalid_artifact_warning -> registered as invalid but never saveable. +# @REJECTED Temporary write inside the Git worktree — it dirties the repository before approval. +# #endregion AgentRuns.Artifacts.Register + +# #region AgentRuns.Approvals.Request [C:5] [TYPE Function] [SEMANTICS agent-run,hitl,approval,hash] +# @ingroup AgentRuns +# @BRIEF Create a one-shot approval gate bound to exact operation inputs. +# @PRE Actor has permission to request the operation; all referenced drafts belong to the run. +# @POST PENDING gate stores canonical request_hash and expiry; run enters WAITING_APPROVAL. +# @SIDE_EFFECT Writes ApprovalGate and approval_requested event. +# @DATA_CONTRACT ApprovalRequest -> ApprovalGateView +# @INVARIANT Unauthorized calls return permission_denied semantics and create no gate. +# @TEST_EDGE foreign_artifact -> 403. +# @TEST_EDGE missing_baseline_reason_requirement -> gate created with reason_required=true. +# #endregion AgentRuns.Approvals.Request + +# #region AgentRuns.Approvals.Decide [C:5] [TYPE Function] [SEMANTICS agent-run,hitl,decision,audit] +# @ingroup AgentRuns +# @BRIEF Record immutable confirmation or denial for a pending gate. +# @PRE Gate is pending, unexpired, owned by actor; reason present when required. +# @POST Gate becomes CONFIRMED or DENIED exactly once and decision event is appended. +# @SIDE_EFFECT Writes decision audit; denial returns run to RUNNING or CANCELLED according to operation policy. +# @DATA_CONTRACT ApprovalDecisionRequest -> ApprovalGateView +# @TEST_EDGE blank_required_reason -> 422. +# @TEST_EDGE repeated_decision -> 409. +# @TEST_EDGE expired_gate -> EXPIRED and no confirmation. +# #endregion AgentRuns.Approvals.Decide + +# #region AgentRuns.Approvals.Consume [C:5] [TYPE Function] [SEMANTICS agent-run,hitl,consume,side-effect] +# @ingroup AgentRuns +# @BRIEF Execute the exact approved write and atomically consume its gate. +# @PRE Gate confirmed; fresh RBAC passes; actor and recalculated request_hash match. +# @POST Side effect succeeds once and gate is CONSUMED, or neither is committed. +# @SIDE_EFFECT Writes repository artifacts or approved baseline plus audit event. +# @INVARIANT No retargeting, replay, partial path set, or post-confirm payload mutation. +# @TEST_INVARIANT Approval_Request_Binding -> VERIFIED_BY: payload_change, target_change, replay. +# @TEST_EDGE payload_change -> 409 and no write. +# @TEST_EDGE permission_revoked_after_confirm -> 403 and gate not consumed. +# @TEST_EDGE replay_consumed_gate -> 409 and no second write. +# @RATIONALE Rechecking hash and RBAC at consume time closes the confirmation-to-execution gap. +# @REJECTED Confirming only a human-readable prompt — the executable arguments could change after review. +# #endregion AgentRuns.Approvals.Consume + +## Standalone Agent + +# #region AgentRuns.Tracker [C:4] [TYPE Module] [SEMANTICS agent-run,agent,http,events] +# @defgroup AgentRuns Agent-side typed client for backend run lifecycle APIs. +# @LAYER Integration +# @RELATION DEPENDS_ON -> [AgentRuns.Api] +# @INVARIANT Every call carries service JWT and current user JWT; logs redact both. +# @RATIONALE A small client isolates retry/idempotency behavior from the already-large Gradio handler. +# #endregion AgentRuns.Tracker + +# #region AgentRuns.Context.ValidateV2 [C:3] [TYPE Function] [SEMANTICS agent-run,context,intent,validation] +# @ingroup AgentRuns +# @BRIEF Validate UIContext v1/v2 and scenario-intent cross-field rules. +# @POST Returns normalized context; v1 ordinary chat remains byte-shape compatible. +# @TEST_EDGE v1_without_intent -> accepted. +# @TEST_EDGE v2_scenario_dashboard -> accepted. +# @TEST_EDGE scenario_dataset -> rejected. +# @TEST_EDGE unknown_intent -> rejected. +# #endregion AgentRuns.Context.ValidateV2 + +# #region AgentRuns.ToolPipeline.Scenario [C:4] [TYPE Function] [SEMANTICS agent-run,tools,scenario,no-sql] +# @ingroup AgentRuns +# @BRIEF Restrict scenario-intent tools to dashboard inspection, scenario, artifact, and approval operations. +# @PRE RBAC filtering has already run. +# @POST Tool list excludes superset_execute_sql and every arbitrary-SQL operation. +# @SIDE_EFFECT Emits pipeline_result audit metadata. +# @INVARIANT No direct SQL tool is reachable for build_dashboard_test_scenario. +# @TEST_INVARIANT No_Direct_SQL -> VERIFIED_BY: scenario_tool_list, replayed_sql_call. +# @TEST_EDGE hallucinated_sql_tool_call -> invocation guard rejects. +# @REJECTED Relying only on the system prompt to avoid SQL — tool availability is the enforceable boundary. +# #endregion AgentRuns.ToolPipeline.Scenario + +# #region AgentRuns.Gradio.Emit [C:4] [TYPE Function] [SEMANTICS agent-run,gradio,stream,event] +# @ingroup AgentRuns +# @BRIEF Persist then emit typed run metadata through the existing Gradio stream. +# @PRE AgentRun exists; event accepted by backend. +# @POST Emitted metadata carries authoritative run_id and sequence. +# @SIDE_EFFECT Backend event write followed by Gradio yield. +# @INVARIANT Persistence precedes emission; a dropped stream remains recoverable. +# @TEST_EDGE backend_event_failure -> emit recoverable error, never unpersisted progress. +# #endregion AgentRuns.Gradio.Emit + +## Frontend + +// #region AgentRuns.Model [C:5] [TYPE Model] [SEMANTICS agent-run,model,recovery,draft,approval] +// @defgroup AgentRuns Frontend run projection composed by AgentChat.Model. +// @STATE absent | starting | running | waiting_input | waiting_approval | completed | failed | cancelled | disconnected +// @ACTION applyMetadata(meta) — accept typed event when sequence is newer. +// @ACTION recover(runId) — fetch authoritative snapshot after reload/drop. +// @ACTION decideGate(gateId, decision, reason?) — submit HITL decision through existing resume path. +// @INVARIANT Stage state derives from structured metadata/snapshot only, never assistant prose. +// @INVARIANT Drafts are keyed by artifact id and cannot cross run boundaries. +// @RELATION DEPENDS_ON -> [AgentRuns.Api] +// @RELATION BINDS_TO -> [AgentRuns.RunPanel] +// @RATIONALE A composed submodel prevents further uncontrolled growth of AgentChat.Model. +// @REJECTED Duplicate inline run atoms in components — creates divergent recovery state. +// #endregion AgentRuns.Model + + + + + + + + + + + + + + + + + + + + + +#endregion AgentTestStabilization.Modules diff --git a/specs/036-agent-test-stabilization/contracts/ux/agent-run-ux.md b/specs/036-agent-test-stabilization/contracts/ux/agent-run-ux.md new file mode 100644 index 000000000..81ab3b49d --- /dev/null +++ b/specs/036-agent-test-stabilization/contracts/ux/agent-run-ux.md @@ -0,0 +1,34 @@ +#region AgentTestStabilization.RunUx [C:4] [TYPE ADR] [SEMANTICS ux,agent-run,progress,draft,recovery] +@BRIEF Detailed FSM and recovery contract for the run strip and draft preview. +@RELATION BINDS_TO -> [AgentRuns.Model] + +## Run Panel FSM + +| State | Visible contract | Actions | +|---|---|---| +| starting | Context, connecting indicator, no fake run id | Cancel chat | +| running | Run id, active/completed stages, last update | Continue chat | +| waiting_input | Required parameter summary | Provide parameters | +| waiting_approval | Exact operation, target paths, warnings | Confirm/deny | +| disconnected | Last known stage marked stale | Recover snapshot | +| completed | All completed/skipped stages and drafts | Preview/download | +| failed | Error code, failed stage, recovery choices | Retry safe analysis or restart | +| cancelled | Explicit cancellation and no side effect claim | Start new run | + +## Draft States + +- pending: metadata accepted, validation running. +- valid: preview/download and save request allowed. +- warning: preview required; save gate must repeat warnings. +- invalid: preview allowed, save request disabled. +- persisted: target path and persisted timestamp visible. + +## UX Tests + +1. Stream drop at validate → disconnected → snapshot returns waiting_approval with same drafts. +2. Foreign run event → ignored and logged. +3. Invalid artifact → save disabled with associated reason. +4. Denied save → no persisted marker; cancellation message remains visible. +5. Permission denied → no confirm button and focusable alternatives. + +#endregion AgentTestStabilization.RunUx diff --git a/specs/036-agent-test-stabilization/contracts/ux/alternatives.md b/specs/036-agent-test-stabilization/contracts/ux/alternatives.md new file mode 100644 index 000000000..70353a8f8 --- /dev/null +++ b/specs/036-agent-test-stabilization/contracts/ux/alternatives.md @@ -0,0 +1,16 @@ +#region AgentTestStabilization.Alternatives [C:3] [TYPE ADR] [SEMANTICS ux,alternatives,agent-run] +@BRIEF Rejected UX/runtime compositions for recoverable dashboard-testing runs. + +## Considered Alternatives + +1. **Chat transcript as run state** — rejected because localized prose cannot provide durable sequence, recovery, or audit semantics. +2. **Frontend-only run tracker** — rejected because reload, disconnect, or Gradio restart would lose authoritative state. +3. **A second confirmation modal for scenario writes** — rejected because the existing confirmation card already owns HITL presentation and must bind to the backend gate. +4. **Persist drafts directly in the target repository** — rejected because preview and download must remain side-effect free until approval is consumed. +5. **Hide ordinary dashboard AI behind the scenario action** — rejected because general chat and scenario generation are distinct business intents and both remain available. + +## Selected Composition + +Backend AgentRun is authoritative; the standalone agent persists structured events before streaming; AgentRunModel projects snapshots; existing ConfirmationCard renders bound gates. + +#endregion AgentTestStabilization.Alternatives diff --git a/specs/036-agent-test-stabilization/contracts/ux/api-ux.md b/specs/036-agent-test-stabilization/contracts/ux/api-ux.md new file mode 100644 index 000000000..65c424e17 --- /dev/null +++ b/specs/036-agent-test-stabilization/contracts/ux/api-ux.md @@ -0,0 +1,29 @@ +#region AgentTestStabilization.ApiUx [C:3] [TYPE ADR] [SEMANTICS ux,api,agent-run,recovery] +@BRIEF UX mapping for Gradio metadata and Agent Run REST recovery. +@RELATION DEPENDS_ON -> [AgentTestStabilization.Events] +@RELATION DEPENDS_ON -> [AgentRuns.Api] + +## Interaction Map + +| Trigger | API/event | Immediate UI | Failure recovery | +|---|---|---|---| +| Scenario first send | agent_run_started | Show run id and context stage | If absent by 60s, actionable timeout | +| Progress | scenario_progress | Update named stage | Sequence gap triggers snapshot GET | +| Draft registered | draft_artifacts | Replace draft inventory by id | Keep prior inventory and show recover | +| Reload/drop | GET /api/agent/runs/{id} | Restore authoritative projection | 403 ownership; 404 expired/not found | +| Save request | extended confirm_required | Show exact paths/warnings | Deny is terminal for operation, not run | +| Unauthorized request | permission_denied | Dismiss-only access card | No confirm control | + +## Status Handling + +- 401: session expired; stop privileged action and request login. +- 403: show required permission; never retry automatically. +- 409: projection/gate conflict; recover snapshot before another action. +- 413/422: show field/artifact validation details. +- 5xx: preserve run id and last snapshot; offer retry GET, not duplicate POST. + +## Accessibility + +Progress uses an ordered list with aria-current=step. New stage announcements are polite. Confirmation focus moves into the card and returns to the invoking control after deny/consume. + +#endregion AgentTestStabilization.ApiUx diff --git a/specs/036-agent-test-stabilization/contracts/ux/decisions.md b/specs/036-agent-test-stabilization/contracts/ux/decisions.md new file mode 100644 index 000000000..06194b6e9 --- /dev/null +++ b/specs/036-agent-test-stabilization/contracts/ux/decisions.md @@ -0,0 +1,15 @@ +#region AgentTestStabilization.UxDecisions [C:3] [TYPE ADR] [SEMANTICS ux,decisions,agent-run] +@BRIEF Final UX decisions for adding scenario-run state to the existing agent page. + +## Decisions + +1. Compose AgentRuns.Model into AgentChat.Model; do not create a second chat client. +2. Keep the run panel in the agent workspace above draft/confirmation content, not in a global drawer. +3. Display the full run id with copy affordance; stage state never depends on prose. +4. Recover by snapshot after a stream gap; never silently restart the run. +5. Draft download is always available only through opaque authenticated URLs and is side-effect free. +6. Warnings are repeated in the approval card so the user reviews the exact write risk. +7. Permission denied is a separate dismiss-only state, consistent with 035. +8. Baseline reason is a required labeled field with inline validation before confirm. + +#endregion AgentTestStabilization.UxDecisions diff --git a/specs/036-agent-test-stabilization/contracts/ux/design-tokens.md b/specs/036-agent-test-stabilization/contracts/ux/design-tokens.md new file mode 100644 index 000000000..83062f61b --- /dev/null +++ b/specs/036-agent-test-stabilization/contracts/ux/design-tokens.md @@ -0,0 +1,17 @@ +#region AgentTestStabilization.DesignTokens [C:2] [TYPE ADR] [SEMANTICS ux,design-tokens,agent-run] +@BRIEF Semantic token and shared-component mapping for run progress, drafts, and recovery. + +| Purpose | Required tokens/components | +|---|---| +| Panel/card | bg-surface-card, border-border, text-text | +| Active/info stage | bg-primary-light, text-primary, Icon | +| Completed | bg-success-light, text-success | +| Warning/waiting | bg-warning-light, text-warning | +| Failed/invalid | bg-destructive-light, border-destructive-ring, text-destructive | +| Stale/disconnected | bg-surface-muted, text-text-muted | +| Actions | Button from $lib/ui; primary/secondary/destructive variants | +| Loading | Skeleton or Spinner from $lib/ui | + +Raw color families and page-level raw buttons are forbidden. Focus rings must use shared component behavior and remain visible in warning/destructive surfaces. + +#endregion AgentTestStabilization.DesignTokens diff --git a/specs/036-agent-test-stabilization/contracts/ux/screen-models.md b/specs/036-agent-test-stabilization/contracts/ux/screen-models.md new file mode 100644 index 000000000..ec46b2a4a --- /dev/null +++ b/specs/036-agent-test-stabilization/contracts/ux/screen-models.md @@ -0,0 +1,40 @@ +#region AgentTestStabilization.ScreenModels [C:4] [TYPE ADR] [SEMANTICS ux,screen-models,agent-run] +@BRIEF Screen-model inventory and invariants for recoverable scenario runs inside the existing agent page. +@RELATION DEPENDS_ON -> [AgentRuns.Model] + +## Model Inventory + +| Model | Ownership | State | +|---|---|---| +| AgentChat.Model | Existing chat, connection, messages, generic HITL | Composes AgentRuns.Model; does not duplicate run atoms | +| AgentRuns.Model | Scenario-run projection and recovery | run id/status, stages, sequence, drafts, pending gate, recovery error | + +## AgentRuns.Model Actions + +- startFromContext(context): begins starting state; run id arrives from agent_run_started. +- applyMetadata(event): validates run id and monotonic sequence before mutation. +- recover(runId): loads AgentRunSnapshot and atomically replaces projection. +- previewArtifact(id): obtains safe preview; does not mutate repository. +- requestSave(ids): asks backend/agent for a bound gate; does not write. +- decideGate(decision, reason): validates reason locally, then submits authoritative decision. +- reset(): clears only run projection when starting a new conversation/context. + +## Invariants + +1. Run id cannot change after start without reset. +2. Snapshot replacement is accepted only for the same run id. +3. UI stage derives from event/snapshot fields, never chat text. +4. Sequence gaps trigger degraded/recovery, not speculative stage updates. +5. Invalid drafts cannot enter save request. +6. Baseline approval cannot confirm with blank reason. +7. Ordinary chat with UIContext v1 has no AgentRuns.Model projection. + +## L1 Tests + +- v1 ordinary chat stays absent. +- v2 started event initializes one run. +- duplicate, stale, foreign-run, and sequence-gap events. +- recovered snapshot restores drafts and pending gate. +- deny clears pending gate without consuming artifacts. + +#endregion AgentTestStabilization.ScreenModels diff --git a/specs/036-agent-test-stabilization/data-model.md b/specs/036-agent-test-stabilization/data-model.md new file mode 100644 index 000000000..cfd74d25d --- /dev/null +++ b/specs/036-agent-test-stabilization/data-model.md @@ -0,0 +1,116 @@ +#region AgentTestStabilization.DataModel [C:4] [TYPE ADR] [SEMANTICS data-model,agent-run,event,draft,approval] +@BRIEF Canonical entities, validation rules, lifecycle transitions, and ownership rules for feature 036. +@RELATION DEPENDS_ON -> [AgentTestStabilization.Spec] +@RELATION DEPENDS_ON -> [AgentTestStabilization.Research] + +## DashboardScenarioUIContext + +| Field | Type | Rule | +|---|---|---| +| objectType | literal dashboard | Required for scenario intent | +| objectId | numeric string | Required, 1–20 digits | +| objectName | string/null | Max 256 chars | +| envId | string | Required, max 128 chars | +| route | string | Must begin /dashboards/, max 512 chars | +| contextVersion | 1 or 2 | v1 ordinary chat; v2 scenario extension | +| intent | literal/null | build_dashboard_test_scenario only in v2 | + +Compatibility: v1 payloads without intent remain valid. A scenario intent with v1 is invalid; an unknown intent is invalid. + +## AgentRun + +| Field | Type | Notes | +|---|---|---| +| id | UUID | Stable public agent_run_id | +| conversation_id | UUID/string | Existing conversation correlation | +| user_id | string | Owner; immutable | +| intent | enum | Initial value: dashboard scenario build | +| dashboard_id, environment_id | string | Immutable context correlation | +| context_snapshot | JSON | Validated UIContext v2 | +| status | enum | See FSM | +| current_stage | enum | Highest accepted scenario stage | +| last_sequence | int | Monotonic per run | +| error_code, error_detail | nullable string | Sanitized terminal/degraded information | +| created_at, updated_at, finished_at | timestamp | UTC | + +### AgentRun FSM + +~~~text +CREATED → RUNNING ↔ WAITING_INPUT + ↔ WAITING_APPROVAL + → COMPLETED + → FAILED + → CANCELLED +~~~ + +Terminal states are immutable. Reconnection reads state; it never restarts execution implicitly. + +## AgentRunEvent + +| Field | Rule | +|---|---| +| run_id | Existing owned run | +| sequence | Positive, unique per run, strictly increasing | +| event_type | run_started, progress, drafts_updated, approval_requested, approval_resolved, terminal | +| stage | context, inspect, scenario, parameters, generate, validate, or save | +| status | pending, active, completed, blocked, failed, or skipped | +| payload | Typed, redacted JSON; max 64 KiB | +| occurred_at | UTC timestamp | + +Duplicate (run_id, sequence) is idempotent only if canonical payload hashes match; otherwise it is a conflict. + +## DraftArtifact + +| Field | Rule | +|---|---| +| id | UUID | +| run_id | Parent AgentRun | +| kind | scenario, runner_plan, report_template, baseline_candidate, evidence_manifest, other | +| name | Display name, max 255 | +| intended_path | Relative POSIX path; no absolute path, parent traversal, NUL, or symlink traversal | +| content_ref | Opaque backend storage key; never exposed as filesystem path | +| sha256 | Lowercase 64-character digest | +| validation_status | valid, warning, invalid, pending | +| warnings | Bounded list of structured code/message objects | +| persisted_at | Null until approved write succeeds | + +Draft downloads are read-only and do not change repository state. + +## ApprovalGate + +| Field | Rule | +|---|---| +| id | UUID | +| run_id | Parent run | +| operation | repository_write or baseline_approval | +| request_hash | SHA-256 of canonical operation, targets, artifact hashes, and baseline payload | +| target_paths | Normalized relative paths | +| risk_level | guarded or dangerous | +| required_permission | dashboard:testing WRITE or APPROVE | +| status | pending, confirmed, denied, consumed, expired | +| reason_required | true for baseline approval | +| reason | Required non-blank when applicable, max 2000 | +| actor_id, decided_at, expires_at | Immutable decision audit | + +### Approval FSM + +~~~text +PENDING → CONFIRMED → CONSUMED + → DENIED + → EXPIRED +~~~ + +Execution requires CONFIRMED status, matching actor, non-expired gate, unchanged request_hash, and a fresh RBAC check. Consumption is atomic with the durable side effect. + +## Snapshot DTO + +AgentRunSnapshot contains run fields, ordered stages, recent events, all draft references, and the current pending gate. It never contains raw draft bytes, JWTs, internal storage paths, or another user's run. + +## Retention + +- Active and waiting runs are retained. +- Terminal run metadata and audit: minimum 30 days. +- Unpersisted draft bytes: configurable, default 7 days after terminal state. +- Approval decisions: retained with the associated run audit and never silently rewritten. + +#endregion AgentTestStabilization.DataModel diff --git a/specs/036-agent-test-stabilization/plan.md b/specs/036-agent-test-stabilization/plan.md new file mode 100644 index 000000000..51239538b --- /dev/null +++ b/specs/036-agent-test-stabilization/plan.md @@ -0,0 +1,80 @@ +# Implementation Plan: Agent Test Stabilization + +**Branch**: 036-agent-test-stabilization | **Date**: 2026-07-13 | **Spec**: [spec.md](./spec.md) +**Input**: [research.md](./research.md), [data-model.md](./data-model.md), [contracts/](./contracts/) + +## Summary + +Extend the existing Gradio/LangGraph agent with a durable backend-owned AgentRun lifecycle. Dashboard scenario intent is carried in UIContext v2 without changing the positional Gradio contract. Structured progress and draft metadata drive the UI, while repository writes and baseline approvals use one-shot, payload-bound HITL gates. + +## Technical Context + +**Language/Version**: Python 3.11+ agent package; Python 3.13+ backend; TypeScript/Svelte 5 frontend +**Dependencies**: Gradio, LangGraph/LangChain, httpx; FastAPI 0.126, Pydantic 2, SQLAlchemy; Svelte 5.56, Vitest 4 +**Storage**: PostgreSQL for run/event/gate records; configured backend draft-storage root for draft bytes +**Testing**: pytest in agent/ and backend/; Vitest L1/L2; Playwright smoke +**Performance Goals**: run id before first tool; snapshot GET p95 under 200ms; progress render under 100ms +**Constraints**: no Gradio positional break; no repository mutation before approval; dual identity auth; ordinary chat compatible +**Scale**: 10 concurrent chats, 100 events and 50 draft references per run, 30-day terminal-run retention + +## Constitution Check + +| Principle | Gate | +|---|---| +| Semantic contracts | C3+ modules/functions receive hierarchical anchors and test edges | +| Decision memory | Durable owner, UIContext v2, event protocol, draft storage, and bound gate decisions are recorded | +| External orchestrator | Agent and backend remain outside Superset | +| Module discipline | New run service is decomposed into repository, event, artifact, and approval modules | +| RBAC | READ/EXECUTE/WRITE/APPROVE checks occur in backend; agent filtering is defense in depth | +| Svelte 5 | AgentChat model extension and new panels are runes/model-first | +| TDD | Contract tests precede C3+ implementation; replay and SQL exclusions have tests | +| Attention | Contract IDs share agent-run or dashboard-testing semantics and stay bounded | + +**Gate result**: PASS. No constitution exception is required. + +## Project Structure + +~~~text +agent/src/ss_tools/agent/ +├── _context.py +├── _run_tracker.py # NEW backend run API client +├── _tool_filter.py +└── app.py + +backend/src/ +├── api/routes/agent_runs.py # NEW REST surface +├── models/agent_run.py +├── schemas/agent_run.py +└── services/agent_runs/ + ├── service.py + ├── repository.py + ├── artifacts.py + └── approvals.py + +frontend/src/lib/ +├── models/AgentChatTypes.ts +├── models/AgentChatModel.svelte.ts +├── models/AgentChat.StreamProcessor.svelte.ts +└── components/agent/ + ├── AgentRunPanel.svelte + └── DraftArtifactList.svelte +~~~ + +## Delivery Phases + +1. **Contract and migration foundation**: DTOs, DB migration, repository, permissions. +2. **Run API**: ownership-scoped create/snapshot/event/draft/gate endpoints. +3. **Agent integration**: UIContext v2, tracker, scenario allow-list, events. +4. **Frontend model/UI**: typed FSM, reconnection snapshot, draft and approval rendering. +5. **Verification**: unit/contract/integration tests, ordinary-chat regression, live smoke. + +## Cross-Spec Boundary + +- 036 does not inspect dashboards or compare metrics. +- 037 registers draft baseline candidates and requests gates through 036. +- 038 registers draft scenario artifacts through 036. +- 039 consumes 036 event/snapshot DTOs and must not create a second run state machine. + +## Complexity Tracking + +No planned file needs an exception. If AgentChatModel.svelte.ts crosses its decomposition gate, extract AgentRunModel.svelte.ts as a composed submodel instead of growing the existing model. diff --git a/specs/036-agent-test-stabilization/quickstart.md b/specs/036-agent-test-stabilization/quickstart.md new file mode 100644 index 000000000..5e372448c --- /dev/null +++ b/specs/036-agent-test-stabilization/quickstart.md @@ -0,0 +1,50 @@ +# Quickstart: Agent Test Stabilization + +**Purpose**: Implement and verify 036 independently before 037–039. + +## Prerequisites + +- Backend and agent virtual environments installed. +- PostgreSQL and configured auth/app databases available. +- Frontend dependencies installed. +- Test user has dashboard:testing READ and EXECUTE; separate approver has APPROVE. + +## Recommended Test Order + +~~~bash +cd agent +python -m pytest tests/test_agent/test_agent_context.py tests/test_agent/test_agent_run_tracker.py tests/test_agent/test_scenario_tool_filter.py -v + +cd ../backend +python -m pytest tests/services/agent_runs tests/api/test_agent_runs.py -v + +cd ../frontend +npm run test -- --run src/lib/models/__tests__/AgentRunModel.test.ts +npm run test -- --run src/lib/components/agent/__tests__/AgentRunPanel.ux.test.ts + +npx playwright test e2e/tests/agent-scenario-run.e2e.js +~~~ + +## Manual Smoke + +1. Start backend, standalone agent, and frontend with the repository run scripts. +2. Open a dashboard with env_id selected. +3. Navigate to /agent with UIContext v2 and intent=build_dashboard_test_scenario. +4. Confirm that a run id appears before the first tool card. +5. Simulate or run progress through context and inspect, then reload the page. +6. Recover the same run snapshot and verify no draft crosses conversation/run boundaries. +7. Register one warning draft; preview and download it; verify Git worktree is unchanged. +8. Request save, deny, and verify no target exists. +9. Request again, change the target after confirm, and verify consume returns 409. +10. Confirm the exact request and verify a single write plus consumed gate audit. +11. Repeat with a viewer and verify permission_denied has no confirm button. +12. Open ordinary v1 chat and verify existing 035 behavior. + +## Exit Gates + +- agent_run_started precedes tool_start in captured stream. +- Snapshot recovery survives Gradio restart. +- Scenario tool list contains no arbitrary SQL operation. +- Payload/path mutation and replay tests pass. +- Existing 033/035 scoped suites pass. +- Backend lint, frontend lint/build, and semantic anchor audit pass. diff --git a/specs/036-agent-test-stabilization/research.md b/specs/036-agent-test-stabilization/research.md new file mode 100644 index 000000000..18321d44b --- /dev/null +++ b/specs/036-agent-test-stabilization/research.md @@ -0,0 +1,73 @@ +#region AgentTestStabilization.Research [C:4] [TYPE ADR] [SEMANTICS research,agent,dashboard-testing,run,hitl] +@BRIEF Phase 0 decisions for recoverable dashboard-scenario agent runs, structured events, draft artifacts, and bound approvals. +@RELATION DEPENDS_ON -> [AgentTestStabilization.Spec] +@RELATION DEPENDS_ON -> [AgentChat.GradioApp.Handler] +@RELATION DEPENDS_ON -> [AgentChat.Model] +@RATIONALE The existing 033/035 chat path already streams and confirms tools, but it does not provide a durable business-run aggregate or recoverable draft inventory. +@REJECTED Treating conversation history or LangGraph checkpoints as the AgentRun system of record — rejected because neither exposes an owned, queryable lifecycle for progress, drafts, and approval decisions. + +## 1. Existing System Fact Check + +| Concern | Existing implementation | Planning consequence | +|---|---|---| +| Gradio submit | AgentChatModel.svelte.ts submits eight values; serialized uicontext is last | Do not add or reorder positional inputs | +| Context validation | agent/src/ss_tools/agent/_context.py accepts context version 1 and dashboard/dataset/migration types | Add backward-compatible v2 intent validation | +| Stream processing | AgentChat.StreamProcessor.svelte.ts dispatches metadata by type | Add typed metadata variants; never parse assistant prose | +| HITL | _confirmation.py emits confirm_required and LangGraph resumes confirm/deny | Extend the envelope; preserve ordinary tool confirmation | +| Recovery | Conversation messages persist; active scenario progress/drafts do not | Add durable backend AgentRun aggregate | +| Dashboard entry | DashboardHeader.svelte already builds an ordinary /agent dashboard link | 039 adds a separate business action with explicit intent | +| SQL tool | Agent registry contains superset_execute_sql | Scenario intent pipeline must exclude it | + +## 2. Durable Run Ownership + +- **Decision**: FastAPI/backend owns AgentRun, AgentRunEvent, DraftArtifact, and ApprovalGate; the standalone agent writes through authenticated internal REST calls. +- **Rationale**: Backend already owns users, conversations, RBAC, repository resolution, and audit persistence. Gradio workers may restart and cannot be the durable authority. +- **Alternative rejected**: In-memory run registry in the agent — loses state on restart and cannot safely enforce ownership. +- **Alternative rejected**: Reuse generic TaskManager as the sole store — AgentRun includes waiting-for-user and draft/approval semantics, while a task is execution monitoring. A later implementation may link task_id, but must not collapse the aggregates. + +## 3. UIContext Compatibility + +- **Decision**: v1 remains valid for ordinary chat. v2 adds intent, with the only initial value build_dashboard_test_scenario. +- **Invariant**: v2 scenario intent requires objectType=dashboard, numeric objectId, non-empty envId, and a dashboard route. +- **Wire rule**: uicontext_str stays the final Gradio positional argument. No agent_run_id positional input is added; the backend emits agent_run_started. +- **Security**: Context remains informational metadata and is validated before prompt injection. Unknown fields are rejected. + +## 4. Structured Event Protocol + +- **Decision**: Add agent_run_started, scenario_progress, draft_artifacts, and agent_run_terminal metadata types. +- **Stage order**: context → inspect → scenario → parameters → generate → validate → save. +- **Monotonicity**: A stage may repeat with a higher sequence, but cannot regress except when the event explicitly declares retry_of_sequence. +- **Recovery**: Frontend obtains the authoritative snapshot with GET /api/agent/runs/{run_id} after reload or stream loss. +- **Timeout**: Existing 60-second first-activity guard remains; agent_run_started counts as first activity and must be emitted before any tool action. + +## 5. Draft Artifact Safety + +- **Decision**: Draft content is stored outside the target repository under backend-managed draft storage; frontend receives opaque artifact_id, preview metadata, and safe download URLs. +- **Invariant**: intended_path is relative, normalized, contains no parent traversal, and is not written until a bound approval executes. +- **Alternative rejected**: Writing generated files to the repository and calling them drafts — that already mutates Git state and defeats preview-before-write. + +## 6. Bound HITL Approval + +- **Decision**: An ApprovalGate binds operation, actor, run, target paths, and canonical payload SHA-256. Confirm/deny records an immutable decision. Execution recalculates the hash. +- **Baseline approval**: Requires a non-blank reason and permission dashboard:testing APPROVE. +- **Repository save**: Requires permission dashboard:testing WRITE. +- **Unauthorized action**: Emits permission_denied directly; no gate and no confirm control. +- **Resurrection ban**: A confirmed gate cannot be replayed, retargeted, or used after expiry. + +## 7. Source Touch Map + +| Area | Planned files | +|---|---| +| Agent context/events | agent/src/ss_tools/agent/_context.py, app.py, new _run_tracker.py | +| Agent tool filtering | agent/src/ss_tools/agent/_tool_filter.py, tools.py | +| Durable run API | new backend/src/api/routes/agent_runs.py, backend/src/services/agent_runs/ | +| Persistence | new models/migration under backend/src/models/ and backend/alembic/versions/ | +| Frontend model | AgentChatTypes.ts, AgentChatModel.svelte.ts, AgentChat.StreamProcessor.svelte.ts | +| Frontend UI | new frontend/src/lib/components/agent/AgentRunPanel.svelte and draft list | +| Tests | agent pytest, backend API/service pytest, frontend L1/L2 vitest, one Playwright smoke | + +## 8. Resolved Scope + +036 supplies the execution and approval substrate only. Superset query truth is 037, scenario graph semantics are 038, and the full business workspace is 039. + +#endregion AgentTestStabilization.Research diff --git a/specs/036-agent-test-stabilization/spec.md b/specs/036-agent-test-stabilization/spec.md index 8a32819c3..99ae6b630 100644 --- a/specs/036-agent-test-stabilization/spec.md +++ b/specs/036-agent-test-stabilization/spec.md @@ -12,7 +12,7 @@ @SEMANTICS: spec, requirements, feature, agent, gradio, langgraph, scenario, artifact, hitl, progress, dashboard-testing **Feature Branch**: `036-agent-test-stabilization` -**Created**: 2026-07-07 | **Status**: Draft +**Created**: 2026-07-07 | **Status**: Ready for Implementation **Input**: "Stabilize the existing Gradio LangGraph agent runtime so it can support long running dashboard test scenario generation with stable dashboard UIContext, explicit scenario intent, structured progress events, draft artifact preview, recoverable agent run identifiers, and HITL confirmation for repository writes or baseline approvals." ## User Scenarios diff --git a/specs/036-agent-test-stabilization/tasks.md b/specs/036-agent-test-stabilization/tasks.md new file mode 100644 index 000000000..c2a7ad4f7 --- /dev/null +++ b/specs/036-agent-test-stabilization/tasks.md @@ -0,0 +1,80 @@ +#region AgentTestStabilization.Tasks [C:3] [TYPE ADR] [SEMANTICS tasks,agent-run,implementation] +@BRIEF Ordered TDD implementation tasks for feature 036. + +**Input**: all documents in specs/036-agent-test-stabilization/ +**Prerequisites**: spec, research, plan, data model, module/event/OpenAPI contracts + +## Phase 1 — Contract Fixtures and Setup + +- [ ] T001 Create canonical UIContext v1/v2 and invalid-intent fixtures under specs/036-agent-test-stabilization/fixtures/context/. +- [ ] T002 [P] Create canonical event/snapshot/gate fixtures under specs/036-agent-test-stabilization/fixtures/events/. +- [ ] T003 Materialize fixtures into agent/tests/fixtures/, backend/tests/fixtures/agent_runs/, and frontend/src/lib/models/__fixtures__/agent-runs/. +- [ ] T004 [P] Add dashboard:testing READ/EXECUTE/WRITE/APPROVE permission seeds and role mapping in backend/src/scripts/seed_permissions.py. + +## Phase 2 — Foundational Persistence + +- [ ] T005 Write failing model/repository tests in backend/tests/services/agent_runs/test_repository.py for ownership, terminal immutability, and sequence uniqueness. +- [ ] T006 Write failing RBAC/API tests in backend/tests/api/test_agent_runs.py, including permission_denied and foreign-run access. +- [ ] T007 Add AgentRun, AgentRunEvent, DraftArtifact, and ApprovalGate ORM models in backend/src/models/agent_run.py plus a migration under backend/alembic/versions/. +- [ ] T008 Add Pydantic DTOs in backend/src/schemas/agent_run.py matching contracts/agent-runs.openapi.yaml. +- [ ] T009 Implement repository transaction boundaries in backend/src/services/agent_runs/repository.py. + +**Checkpoint**: Migration upgrades/downgrades; repository tests pass; no agent/frontend changes yet. + +## Phase 3 — US1 Dashboard Context Run Start + +- [ ] T010 [US1] Write failing v1/v2 cross-field tests in agent/tests/test_agent/test_agent_context_v2.py. +- [ ] T011 [US1] Extend agent/src/ss_tools/agent/_context.py: v1 ordinary compatibility, v2 scenario requirements, extra-field rejection. +- [ ] T012 [US1] Write failing scenario allow-list and invocation-guard tests in agent/tests/test_agent/test_scenario_tool_filter.py, explicitly covering superset_execute_sql. +- [ ] T013 [US1] Extend agent/src/ss_tools/agent/_tool_filter.py so RBAC runs first and scenario intent excludes every arbitrary-SQL tool. +- [ ] T014 [US1] Write failing create/idempotency tests for AgentRuns.Service.Create. +- [ ] T015 [US1] Implement backend/src/services/agent_runs/service.py create/snapshot flows and backend/src/api/routes/agent_runs.py. +- [ ] T016 [US1] Register the new router in backend/src/api/routes/__init__.py and backend/src/app.py. +- [ ] T017 [US1] Add typed UIContext v2 to frontend/src/lib/models/AgentChatTypes.ts without changing the Gradio argument order. +- [ ] T018 [US1] Extend Dashboard-context parsing in frontend/src/lib/models/AgentChatModel.svelte.ts and keep ordinary v1 chat absent from run mode. + +**Checkpoint**: Scenario start returns a durable run id before tools; ordinary chat regressions pass. + +## Phase 4 — US2 Recoverable Long Runs + +- [ ] T019 [US2] Write failing sequence, stage, terminal, and idempotency tests in backend/tests/services/agent_runs/test_events.py. +- [ ] T020 [US2] Implement append/project transaction in backend/src/services/agent_runs/service.py. +- [ ] T021 [US2] Add agent/src/ss_tools/agent/_run_tracker.py with dual-auth, idempotency keys, redaction, and persisted-before-yield behavior. +- [ ] T022 [US2] Write failing L1 recovery tests in frontend/src/lib/models/__tests__/AgentRunModel.test.ts. +- [ ] T023 [US2] Create frontend/src/lib/models/AgentRunModel.svelte.ts and compose it from AgentChatModel.svelte.ts. +- [ ] T024 [US2] Extend AgentChat.StreamProcessor.svelte.ts for started/progress/drafts/terminal events and gap recovery. +- [ ] T025 [US2] Restore run id from route/session-safe state on frontend/src/routes/agent/+page.svelte, then GET authoritative snapshot. + +**Checkpoint**: Reload and simulated Gradio restart restore the same run/status/drafts. + +## Phase 5 — US3 Structured Progress and Drafts + +- [ ] T026 [US3] Write failing artifact path, digest, ownership, and cleanup tests in backend/tests/services/agent_runs/test_artifacts.py. +- [ ] T027 [US3] Implement backend/src/services/agent_runs/artifacts.py with out-of-repository storage and opaque download/preview. +- [ ] T028 [US3] Emit agent_run_started before tool_start and persist all scenario progress in agent/src/ss_tools/agent/app.py. +- [ ] T029 [US3] Write failing L2 tests in frontend/src/lib/components/agent/__tests__/AgentRunPanel.ux.test.ts and DraftArtifactList.ux.test.ts. +- [ ] T030 [US3] Implement frontend/src/lib/components/agent/AgentRunPanel.svelte and frontend/src/lib/components/agent/DraftArtifactList.svelte using $lib/ui and semantic tokens. +- [ ] T031 [US3] Wire the panels into frontend/src/lib/components/agent/AgentChat.svelte without parsing prose. + +## Phase 6 — US4 Bound HITL Gates + +- [ ] T032 [US4] Write failing request/decision/consume tests in backend/tests/services/agent_runs/test_approvals.py for reason, expiry, hash mutation, RBAC revocation, and replay. +- [ ] T033 [US4] Implement backend/src/services/agent_runs/approvals.py and REST gate endpoints in backend/src/api/routes/agent_runs.py with atomic consume. +- [ ] T034 [US4] Extend agent/src/ss_tools/agent/_confirmation.py, frontend/src/lib/models/AgentChatTypes.ts, and frontend/src/lib/components/assistant/ConfirmationCard.svelte for gate id, exact targets, warnings, and required reason. +- [ ] T035 [US4] Add denial and permission-denied frontend tests; confirm no confirm control for unauthorized actors. + +## Phase 7 — Integration and Quality Gates + +- [ ] T036 Add agent/tests/test_agent/test_agent_run_tracker.py covering backend loss and no unpersisted event emission. +- [ ] T037 Add frontend/e2e/tests/agent-scenario-run.e2e.js for context → run id → progress → draft → deny/recover. +- [ ] T038 Run all existing 033/035 agent, context, confirmation, retry, timeout, and frontend model/component suites. +- [ ] T039 Run quickstart.md including Gradio restart and Git worktree unchanged checks. +- [ ] T040 Run backend/agent ruff, backend/agent pytest, frontend lint/test/build. +- [ ] T041 Audit ATTN_1–4, exact anchor pairs, unresolved relations, and direct-SQL exclusion. + +## Dependencies + +T001–T009 → US1 → US2 → US3 → US4 → integration. +Within a phase, test tasks precede implementation. 037 must not begin before the US4 consume contract passes. + +#endregion AgentTestStabilization.Tasks diff --git a/specs/036-agent-test-stabilization/traceability.md b/specs/036-agent-test-stabilization/traceability.md new file mode 100644 index 000000000..7f208ee3f --- /dev/null +++ b/specs/036-agent-test-stabilization/traceability.md @@ -0,0 +1,29 @@ +#region AgentTestStabilization.Traceability [C:3] [TYPE ADR] [SEMANTICS traceability,agent-run,requirements] +@BRIEF Requirements-to-contract-to-task-to-test matrix for feature 036. + +| Requirement | Contract | Tasks | Verification | +|---|---|---|---| +| AGSTAB-FR-001 | AgentRuns.Context.ValidateV2 | T008–T011 | agent context v1/v2 tests | +| AGSTAB-FR-002 | AgentRuns.Service.Create, AgentRuns.Model | T014–T018, T026–T028 | create/snapshot/reload tests | +| AGSTAB-FR-003 | AgentRuns.Service.AppendEvent, AgentRuns.Gradio.Emit | T019–T021, T029 | sequence/stage/stream tests | +| AGSTAB-FR-004 | AgentRuns.Artifacts.Register, AgentRuns.DraftList | T022–T024, T031 | path/digest/draft UX tests | +| AGSTAB-FR-005 | AgentRuns.Approvals.Request/Decide/Consume | T015, T022, T032–T034 | bound gate and reason tests | +| AGSTAB-FR-006 | AgentRuns.Api, scenario invocation guard | T006, T012, T034 | RBAC and permission_denied tests | +| AGSTAB-FR-007 | UIContext compatibility and regression phase | T009, T038 | existing 033/035 suites | +| AGSTAB-FR-008 | End-to-end contracts | T035–T037 | Playwright/live smoke | + +## Story Coverage + +| Story | Independent checkpoint | +|---|---| +| US1 | v2 context produces a run id; v1 chat remains unchanged | +| US2 | reload/restart recovers authoritative stage and drafts | +| US3 | structured events alone render progress/drafts | +| US4 | deny, mutation, permission, expiry, and replay cannot write | + +## Upstream/Downstream + +- Depends on 033 chat streaming and 035 context/guardrails. +- Blocks 037 baseline approvals, 038 draft scenario artifacts, and 039 workspace recovery. + +#endregion AgentTestStabilization.Traceability diff --git a/specs/036-agent-test-stabilization/ux_reference.md b/specs/036-agent-test-stabilization/ux_reference.md index 3393054b9..b67885b02 100644 --- a/specs/036-agent-test-stabilization/ux_reference.md +++ b/specs/036-agent-test-stabilization/ux_reference.md @@ -2,7 +2,7 @@ @BRIEF UX reference for stabilized agent execution before dashboard test scenario generation. **Feature Branch**: `036-agent-test-stabilization` -**Created**: 2026-07-07 | **Status**: Draft +**Created**: 2026-07-07 | **Status**: Ready for Implementation ## 1. User Persona & Context diff --git a/specs/037-superset-baseline-engine/checklists/requirements.md b/specs/037-superset-baseline-engine/checklists/requirements.md index 63cc2a10d..9601cd26c 100644 --- a/specs/037-superset-baseline-engine/checklists/requirements.md +++ b/specs/037-superset-baseline-engine/checklists/requirements.md @@ -23,3 +23,12 @@ - [x] CHK009 Query model, normalized filters, query context, result, baseline, and comparison entities are defined. - [x] CHK010 Success criteria can be tested with deterministic fixtures. - [x] CHK011 Scope excludes scenario graph UI and generated Playwright/XLSX execution details. + +## Implementation Package + +- [x] CHK012 Research and implementation plan resolve all design decisions. +- [x] CHK013 Data model, module, OpenAPI, JSON Schema, and UX contracts are present. +- [x] CHK014 Quickstart defines deterministic Superset fixture verification. +- [x] CHK015 Traceability maps every functional requirement to contracts, tasks, and tests. +- [x] CHK016 Tasks use exact repository paths, dependency order, and test-first sequencing. +- [x] CHK017 Machine-readable contracts, external references, and semantic anchors pass validation. diff --git a/specs/037-superset-baseline-engine/contracts/baseline-catalog.schema.json b/specs/037-superset-baseline-engine/contracts/baseline-catalog.schema.json new file mode 100644 index 000000000..fc29dd918 --- /dev/null +++ b/specs/037-superset-baseline-engine/contracts/baseline-catalog.schema.json @@ -0,0 +1,105 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://superset-tools.local/schemas/dashboard-baseline-catalog-v1.json", + "title": "Dashboard Baseline Catalog", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "dashboard", "entries"], + "properties": { + "schema_version": { "const": 1 }, + "dashboard": { + "type": "object", + "additionalProperties": false, + "required": ["id"], + "properties": { + "id": { "type": "integer", "minimum": 1 }, + "slug": { "type": ["string", "null"] }, + "title": { "type": ["string", "null"] } + } + }, + "entries": { + "type": "array", + "items": { "$ref": "#/$defs/entry" } + } + }, + "$defs": { + "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "filters": { + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "filters", "filters_hash"], + "properties": { + "schema_version": { "const": 1 }, + "filters": { "type": "array", "items": { "type": "object" } }, + "filters_hash": { "$ref": "#/$defs/sha256" } + } + }, + "value": { + "type": "object", + "additionalProperties": true, + "required": ["kind", "canonical"], + "properties": { + "kind": { + "enum": ["null", "boolean", "integer", "decimal", "string", "date", "datetime", "percent", "table"] + }, + "canonical": {} + } + }, + "policy": { + "type": "object", + "required": ["type"], + "properties": { + "type": { "enum": ["exact", "absolute_tolerance", "relative_tolerance", "range", "row_set"] } + } + }, + "entry": { + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", "baseline_id", "dashboard_id", "result_key", + "normalized_filters", "expected", "policy", "status", + "fingerprints", "provenance", "approval", "created_at" + ], + "properties": { + "schema_version": { "const": 1 }, + "baseline_id": { "type": "string", "format": "uuid" }, + "dashboard_id": { "type": "integer", "minimum": 1 }, + "chart_id": { "type": ["integer", "null"] }, + "dataset_id": { "type": ["integer", "null"] }, + "result_key": { "type": "string", "minLength": 1 }, + "label": { "type": "string" }, + "normalized_filters": { "$ref": "#/$defs/filters" }, + "expected": { "$ref": "#/$defs/value" }, + "policy": { "$ref": "#/$defs/policy" }, + "status": { "enum": ["approved", "superseded", "retired"] }, + "fingerprints": { + "type": "object", + "additionalProperties": false, + "required": ["query", "dataset", "filter"], + "properties": { + "query": { "$ref": "#/$defs/sha256" }, + "dataset": { "$ref": "#/$defs/sha256" }, + "filter": { "$ref": "#/$defs/sha256" } + } + }, + "provenance": { "type": "object" }, + "approval": { + "type": "object", + "required": ["actor_id", "reason", "approved_at", "approval_gate_id"], + "properties": { + "actor_id": { "type": "string" }, + "reason": { "type": "string", "minLength": 1, "maxLength": 2000 }, + "approved_at": { "type": "string", "format": "date-time" }, + "approval_gate_id": { "type": "string", "format": "uuid" } + } + }, + "created_at": { "type": "string", "format": "date-time" }, + "updated_at": { "type": ["string", "null"], "format": "date-time" } + }, + "oneOf": [ + { "required": ["chart_id"], "properties": { "chart_id": { "type": "integer" } } }, + { "required": ["dataset_id"], "properties": { "dataset_id": { "type": "integer" } } } + ] + } + } +} diff --git a/specs/037-superset-baseline-engine/contracts/dashboard-testing.openapi.yaml b/specs/037-superset-baseline-engine/contracts/dashboard-testing.openapi.yaml new file mode 100644 index 000000000..d0e4da334 --- /dev/null +++ b/specs/037-superset-baseline-engine/contracts/dashboard-testing.openapi.yaml @@ -0,0 +1,250 @@ +openapi: 3.1.0 +info: + title: Dashboard Testing Baseline API + version: 0.1.0 + description: Superset-native query inspection, execution, normalization, comparison, and candidate lifecycle. +servers: + - url: /api +paths: + /dashboard-testing/query-model: + get: + operationId: inspectDashboardQueryModel + security: [{ bearerAuth: [] }] + parameters: + - { name: environment_id, in: query, required: true, schema: { type: string } } + - { name: dashboard_id, in: query, required: true, schema: { type: integer } } + responses: + '200': + description: Deterministic dashboard query model + content: { application/json: { schema: { $ref: '#/components/schemas/DashboardQueryModel' } } } + '403': { description: Permission denied } + '404': { description: Dashboard not found } + /dashboard-testing/filters/normalize: + post: + operationId: normalizeDashboardFilters + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: { application/json: { schema: { $ref: '#/components/schemas/NormalizeFiltersRequest' } } } + responses: + '200': + description: Canonical filter identity + content: { application/json: { schema: { $ref: '#/components/schemas/NormalizedFilterContext' } } } + '422': { description: Invalid target, operator, type, scope, or value } + /dashboard-testing/queries/execute: + post: + operationId: executeDashboardQuery + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: { application/json: { schema: { $ref: '#/components/schemas/ExecuteQueryRequest' } } } + responses: + '200': + description: Canonical Superset result + content: { application/json: { schema: { $ref: '#/components/schemas/NormalizedValue' } } } + '403': { description: Permission denied by app or Superset } + '422': { description: Request/filter/query validation failed } + '504': { description: Superset timeout } + /dashboard-testing/comparisons: + post: + operationId: compareDashboardResult + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: { application/json: { schema: { $ref: '#/components/schemas/ComparisonRequest' } } } + responses: + '200': + description: Pass/fail/inconclusive/missing/stale result + content: { application/json: { schema: { $ref: '#/components/schemas/ComparisonResult' } } } + /dashboard-testing/baselines: + get: + operationId: listDashboardBaselines + security: [{ bearerAuth: [] }] + parameters: + - { name: repository_key, in: query, required: true, schema: { type: string } } + - { name: dashboard_key, in: query, required: true, schema: { type: string } } + responses: + '200': + description: Validated reviewable baseline catalog + content: + application/json: + schema: + type: object + required: [schema_version, entries] + properties: + schema_version: { type: integer } + entries: { type: array, items: { $ref: '#/components/schemas/BaselineEntry' } } + warnings: { type: array, items: { $ref: '#/components/schemas/Warning' } } + /dashboard-testing/baseline-candidates: + post: + operationId: createBaselineCandidate + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: { application/json: { schema: { $ref: '#/components/schemas/CandidateRequest' } } } + responses: + '201': + description: Draft candidate registered through feature 036 + content: { application/json: { schema: { $ref: '#/components/schemas/BaselineCandidate' } } } + '422': { description: Invalid or insufficient provenance } + /dashboard-testing/baseline-candidates/{candidateId}/approval-gate: + post: + operationId: requestBaselineApproval + security: [{ bearerAuth: [] }] + parameters: + - { name: candidateId, in: path, required: true, schema: { type: string, format: uuid } } + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [agent_run_id] + properties: + agent_run_id: { type: string, format: uuid } + responses: + '201': + description: Bound 036 approval gate + content: { application/json: { schema: { $ref: '../../036-agent-test-stabilization/contracts/agent-runs.openapi.yaml#/components/schemas/ApprovalGateView' } } } + '403': { description: Permission denied; no gate created } +components: + securitySchemes: + bearerAuth: { type: http, scheme: bearer, bearerFormat: JWT } + schemas: + Warning: + type: object + required: [code, message] + properties: + code: { type: string } + message: { type: string } + resource: { type: [string, 'null'] } + DashboardQueryModel: + type: object + required: [schema_version, environment_id, dashboard, charts, datasets, native_filters, capabilities, warnings, query_model_fingerprint] + properties: + schema_version: { const: 1 } + environment_id: { type: string } + dashboard: { type: object, required: [id, title], properties: { id: { type: integer }, title: { type: string }, slug: { type: [string, 'null'] } } } + charts: { type: array, items: { type: object } } + datasets: { type: array, items: { type: object } } + native_filters: { type: array, items: { type: object } } + capabilities: { type: object, additionalProperties: { type: boolean } } + warnings: { type: array, items: { $ref: '#/components/schemas/Warning' } } + query_model_fingerprint: { type: string, pattern: '^[a-f0-9]{64}$' } + FilterInput: + type: object + additionalProperties: false + required: [filter_id, value] + properties: + filter_id: { type: string } + value: {} + NormalizeFiltersRequest: + type: object + additionalProperties: false + required: [environment_id, dashboard_id, chart_id, query_model_fingerprint, filters] + properties: + environment_id: { type: string } + dashboard_id: { type: integer } + chart_id: { type: integer } + query_model_fingerprint: { type: string } + filters: { type: array, items: { $ref: '#/components/schemas/FilterInput' } } + NormalizedFilterContext: + type: object + required: [schema_version, filters, filters_hash] + properties: + schema_version: { const: 1 } + filters: { type: array, items: { type: object } } + filters_hash: { type: string, pattern: '^[a-f0-9]{64}$' } + ExecuteQueryRequest: + type: object + additionalProperties: false + required: [environment_id, dashboard_id, chart_id, result_key, filters] + properties: + environment_id: { type: string } + dashboard_id: { type: integer } + chart_id: { type: integer } + result_key: { type: string } + filters: { $ref: '#/components/schemas/NormalizedFilterContext' } + SourceRef: + type: object + required: [environment_id, dashboard_id, result_key, query_hash] + properties: + environment_id: { type: string } + dashboard_id: { type: integer } + chart_id: { type: [integer, 'null'] } + dataset_id: { type: [integer, 'null'] } + result_key: { type: string } + query_hash: { type: string } + NormalizedValue: + type: object + required: [kind, canonical, source, warnings] + properties: + kind: { type: string, enum: ['null', boolean, integer, decimal, string, date, datetime, percent, table] } + raw: {} + canonical: {} + display: {} + format: { type: [object, 'null'] } + source: { $ref: '#/components/schemas/SourceRef' } + warnings: { type: array, items: { $ref: '#/components/schemas/Warning' } } + ComparisonPolicy: + type: object + required: [type] + properties: + type: { type: string, enum: [exact, absolute_tolerance, relative_tolerance, range, row_set] } + additionalProperties: true + BaselineEntry: + type: object + required: [schema_version, baseline_id, dashboard_id, result_key, normalized_filters, expected, policy, status, fingerprints, provenance, approval] + properties: + schema_version: { const: 1 } + baseline_id: { type: string, format: uuid } + dashboard_id: { type: integer } + chart_id: { type: [integer, 'null'] } + dataset_id: { type: [integer, 'null'] } + result_key: { type: string } + label: { type: string } + normalized_filters: { $ref: '#/components/schemas/NormalizedFilterContext' } + expected: { $ref: '#/components/schemas/NormalizedValue' } + policy: { $ref: '#/components/schemas/ComparisonPolicy' } + status: { type: string, enum: [approved, superseded, retired] } + fingerprints: { type: object } + provenance: { type: object } + approval: { type: object } + ComparisonRequest: + type: object + additionalProperties: false + required: [actual, baseline] + properties: + actual: { $ref: '#/components/schemas/NormalizedValue' } + baseline: { $ref: '#/components/schemas/BaselineEntry' } + ComparisonResult: + type: object + required: [status, actual, policy, warnings] + properties: + status: { type: string, enum: [pass, fail, inconclusive, missing_baseline, stale_baseline, permission_denied, source_error] } + actual: { $ref: '#/components/schemas/NormalizedValue' } + expected: {} + policy: { $ref: '#/components/schemas/ComparisonPolicy' } + diff: {} + stale_dimensions: { type: array, items: { type: string } } + warnings: { type: array, items: { $ref: '#/components/schemas/Warning' } } + CandidateRequest: + type: object + additionalProperties: false + required: [agent_run_id, repository_key, dashboard_key, proposed_entry, observations] + properties: + agent_run_id: { type: string, format: uuid } + repository_key: { type: string } + dashboard_key: { type: string } + proposed_entry: { $ref: '#/components/schemas/BaselineEntry' } + observations: { type: array, minItems: 1, items: { $ref: '#/components/schemas/NormalizedValue' } } + BaselineCandidate: + type: object + required: [candidate_id, status, artifact_id, proposed_entry, validation_status, discrepancies] + properties: + candidate_id: { type: string, format: uuid } + status: { const: draft } + artifact_id: { type: string, format: uuid } + proposed_entry: { $ref: '#/components/schemas/BaselineEntry' } + validation_status: { type: string, enum: [valid, warning, invalid] } + discrepancies: { type: array, items: { type: object } } diff --git a/specs/037-superset-baseline-engine/contracts/modules.md b/specs/037-superset-baseline-engine/contracts/modules.md new file mode 100644 index 000000000..8ee4d472a --- /dev/null +++ b/specs/037-superset-baseline-engine/contracts/modules.md @@ -0,0 +1,129 @@ +#region SupersetBaselineEngine.Modules [C:5] [TYPE ADR] [SEMANTICS contracts,baseline,superset,chart-data] +@BRIEF C3+ contracts for query inspection, filter mapping, Superset-native execution, normalization, comparison, and baseline lifecycle. +@RELATION DEPENDS_ON -> [SupersetBaselineEngine.DataModel] +@RELATION DEPENDS_ON -> [AgentRuns.Approvals.Consume] +@RATIONALE The engine is a backend domain; agent tools remain thin authenticated clients. +@REJECTED Direct SQL or agent-supplied raw query context — rejected because saved dashboard semantics must remain authoritative. + +# #region BaselineEngine.Api [C:4] [TYPE Module] [SEMANTICS baseline,api,rbac] +# @defgroup BaselineEngine REST routes for inspection, execution, comparison, candidates, and approval requests. +# @LAYER API +# @RELATION DEPENDS_ON -> [BaselineEngine.QueryModel] +# @RELATION DEPENDS_ON -> [BaselineEngine.QueryExecutor] +# @RELATION DEPENDS_ON -> [BaselineEngine.Comparison] +# @RELATION DEPENDS_ON -> [BaselineEngine.Catalog] +# @INVARIANT No request schema exposes sql, raw endpoint, or raw query_context. +# #endregion BaselineEngine.Api + +# #region BaselineEngine.QueryModel.Inspect [C:5] [TYPE Function] [SEMANTICS baseline,inspection,dashboard,metadata] +# @ingroup BaselineEngine +# @BRIEF Build a deterministic query model from authoritative Superset dashboard, chart, and dataset metadata. +# @PRE Environment and dashboard are readable by actor. +# @POST Returns stable sorted model, per-resource warnings, capabilities, and fingerprint; missing metadata is never invented. +# @SIDE_EFFECT Async GET calls through SupersetClient. +# @DATA_CONTRACT InspectRequest -> DashboardQueryModel +# @RELATION DEPENDS_ON -> [SupersetClient] +# @TEST_EDGE inaccessible_chart -> warning plus executable=false. +# @TEST_EDGE scoped_filter -> only target charts list filter id. +# @TEST_EDGE metadata_order_changes -> byte-stable canonical model. +# #endregion BaselineEngine.QueryModel.Inspect + +# #region BaselineEngine.Filters.Normalize [C:5] [TYPE Function] [SEMANTICS baseline,filter,canonical,scope] +# @ingroup BaselineEngine +# @BRIEF Validate dashboard filter values against metadata/scope and produce canonical typed filter identity. +# @PRE Query model is authoritative and fingerprint-valid. +# @POST Filters are typed, sorted, scoped, and hashed; invalid targets/operators produce structured validation errors. +# @SIDE_EFFECT None. +# @DATA_CONTRACT FilterInput[] + DashboardQueryModel -> NormalizedFilterContext +# @INVARIANT Locale formatting never enters filters_hash. +# @TEST_EDGE locale_decimal -> canonical decimal string. +# @TEST_EDGE filter_outside_chart_scope -> 422. +# @TEST_EDGE missing_target_value -> recoverable FILTER_VALUE_UNAVAILABLE. +# #endregion BaselineEngine.Filters.Normalize + +# #region SupersetClient.ChartData.Execute [C:5] [TYPE Function] [SEMANTICS baseline,superset,chart-data,async] +# @ingroup BaselineEngine +# @BRIEF Execute backend-built chart-data query context through Superset POST /api/v1/chart/data. +# @PRE Saved chart/dataset metadata and normalized filters validated; result limits configured. +# @POST Returns version-adapted raw result with source metadata or typed upstream error. +# @SIDE_EFFECT Async Superset REST call. +# @DATA_CONTRACT AuthoritativeChartMetadata + NormalizedFilterContext -> SupersetRawResult +# @INVARIANT Caller cannot inject SQL, endpoint, datasource, adhoc expression, or unscoped filter. +# @TEST_INVARIANT No_Direct_SQL -> VERIFIED_BY: request_schema, malicious_extra_fields, adapter_payload. +# @TEST_EDGE raw_sql_field -> Pydantic extra-forbid 422 before call. +# @TEST_EDGE superset_403_404_422_timeout_5xx -> error taxonomy preserved. +# @RATIONALE A dedicated mixin centralizes Superset 4.1.2 chart-data differences and existing TLS/auth behavior. +# @REJECTED Using existing dataset preview builder for chart truth — it substitutes count/default columns and is not chart-fidelity. +# #endregion SupersetClient.ChartData.Execute + +# #region BaselineEngine.Result.Normalize [C:5] [TYPE Function] [SEMANTICS baseline,result,normalization,decimal] +# @ingroup BaselineEngine +# @BRIEF Convert scalar, big-number, temporal, percent, and table results into canonical typed values. +# @PRE Raw result was returned by the chart-data adapter with bounded row count. +# @POST Equivalent locale/display variants normalize identically; ambiguity returns inconclusive reason. +# @SIDE_EFFECT None. +# @DATA_CONTRACT SupersetRawResult + ResultDescriptor -> NormalizedValue +# @INVARIANT Numeric canonicalization uses Decimal/string, never binary float equality. +# @TEST_EDGE localized_number -> canonical decimal. +# @TEST_EDGE timezone_datetime -> UTC ISO-8601. +# @TEST_EDGE unsupported_nested_value -> inconclusive, not pass. +# #endregion BaselineEngine.Result.Normalize + +# #region BaselineEngine.Comparison.Compare [C:5] [TYPE Function] [SEMANTICS baseline,comparison,tolerance,diff] +# @ingroup BaselineEngine +# @BRIEF Apply a validated comparison policy to actual and approved expected canonical values. +# @PRE Value kinds and policy are compatible; baseline is approved and fingerprint checked. +# @POST Returns pass/fail/inconclusive/stale with deterministic diff and no mutation. +# @SIDE_EFFECT None. +# @DATA_CONTRACT NormalizedValue + BaselineEntry -> ComparisonResult +# @INVARIANT Missing/stale/unsupported baselines cannot return pass. +# @TEST_EDGE relative_expected_zero -> explicit absolute fallback or inconclusive. +# @TEST_EDGE table_duplicate_keys -> inconclusive unless multiset policy. +# @TEST_EDGE stale_query_fingerprint -> stale_baseline. +# #endregion BaselineEngine.Comparison.Compare + +# #region BaselineEngine.Catalog.Load [C:4] [TYPE Function] [SEMANTICS baseline,catalog,yaml,validation] +# @ingroup BaselineEngine +# @BRIEF Load and validate reviewable baseline YAML from a resolved repository. +# @PRE GitService resolves repository and safe dashboard key; read permission passes. +# @POST Returns sorted valid entries plus catalog warnings; unsafe YAML constructs are rejected. +# @SIDE_EFFECT Bounded file read through file executor. +# @DATA_CONTRACT Repository + DashboardKey -> BaselineCatalog +# @TEST_EDGE invalid_schema -> CATALOG_INVALID. +# @TEST_EDGE duplicate_baseline_identity -> reject catalog. +# @TEST_EDGE symlink_escape -> reject before read. +# #endregion BaselineEngine.Catalog.Load + +# #region BaselineEngine.Candidate.Create [C:5] [TYPE Function] [SEMANTICS baseline,candidate,discovery,draft] +# @ingroup BaselineEngine +# @BRIEF Create a draft candidate from current normalized observations and provenance. +# @PRE No matching approved baseline or explicit refresh requested; source observations validate. +# @POST Candidate remains draft, is registered as 036 DraftArtifact, and never changes approved catalog. +# @SIDE_EFFECT Stores draft through AgentRuns.Artifacts.Register. +# @DATA_CONTRACT DiscoveryRequest + Observation[] -> BaselineCandidate +# @INVARIANT Disagreeing sources produce warning/invalid candidate according to policy. +# @TEST_EDGE api_xlsx_disagree -> warning-gated candidate. +# @TEST_EDGE raw_expected_without_provenance -> rejected. +# #endregion BaselineEngine.Candidate.Create + +# #region BaselineEngine.Candidate.Approve [C:5] [TYPE Function] [SEMANTICS baseline,candidate,approval,repository] +# @ingroup BaselineEngine +# @BRIEF Approve exact candidate through 036 and atomically write the updated baseline catalog. +# @PRE Bound gate confirmed, reason non-blank, APPROVE permission fresh, fingerprints/request hash unchanged. +# @POST Catalog contains one approved immutable entry; gate consumed once; candidate audit links commit/worktree result. +# @SIDE_EFFECT Atomic YAML write in Git worktree and approval audit. +# @RELATION DEPENDS_ON -> [AgentRuns.Approvals.Consume] +# @INVARIANT Engine itself cannot mark a candidate approved without consumed gate metadata. +# @TEST_EDGE stale_after_confirmation -> 409 no catalog write. +# @TEST_EDGE replay -> 409 no duplicate entry. +# @REJECTED Direct approve endpoint without gate binding — approval must cover exact value and fingerprints. +# #endregion BaselineEngine.Candidate.Approve + +# #region AgentChat.Tools.DashboardTesting [C:4] [TYPE Module] [SEMANTICS baseline,agent,tools,no-sql] +# @defgroup BaselineEngine Thin agent tools for inspect, execute, compare, discover, and request approval. +# @RELATION DEPENDS_ON -> [BaselineEngine.Api] +# @INVARIANT Tools never accept SQL or local repository paths and are included only after RBAC/intent filtering. +# @REJECTED Duplicating normalization or catalog logic in the standalone agent. +# #endregion AgentChat.Tools.DashboardTesting + +#endregion SupersetBaselineEngine.Modules diff --git a/specs/037-superset-baseline-engine/contracts/ux/api-ux.md b/specs/037-superset-baseline-engine/contracts/ux/api-ux.md new file mode 100644 index 000000000..8c80c42a4 --- /dev/null +++ b/specs/037-superset-baseline-engine/contracts/ux/api-ux.md @@ -0,0 +1,17 @@ +#region SupersetBaselineEngine.ApiUx [C:3] [TYPE ADR] [SEMANTICS ux,api,baseline,comparison] +@BRIEF User-visible mapping of inspection, execution, comparison, and candidate API outcomes. + +| Outcome | UI state | Recovery | +|---|---|---| +| Query model complete | Counts/capabilities and source environment | Continue | +| Partial metadata | Warning with affected chart/dataset | Exclude/manual checkpoint | +| Query running | Source and target chart visible, aria-busy | Cancel view only; backend timeout governs | +| pass/fail | Actual, expected, diff, policy, provenance | Inspect evidence | +| missing baseline | Candidate action, never pass | Discover candidate | +| stale baseline | Fingerprint dimensions and warning | Rediscover; do not overwrite | +| inconclusive | Raw bounded value and reason | Adjust mapping/manual checkpoint | +| 403/404/422/timeout/5xx | Distinct error code and resource | Switch env, request access, correct filters, retry | + +Every result card includes “Superset API; no direct SQL” when execution was chart-data based. + +#endregion SupersetBaselineEngine.ApiUx diff --git a/specs/037-superset-baseline-engine/contracts/ux/baseline-engine-ux.md b/specs/037-superset-baseline-engine/contracts/ux/baseline-engine-ux.md new file mode 100644 index 000000000..0eebb5654 --- /dev/null +++ b/specs/037-superset-baseline-engine/contracts/ux/baseline-engine-ux.md @@ -0,0 +1,28 @@ +#region SupersetBaselineEngine.ResultUx [C:4] [TYPE ADR] [SEMANTICS ux,baseline,result,candidate] +@BRIEF Display contract for query source, canonical comparison, staleness, and candidate review. + +## Result Card + +- Identity: dashboard, chart/dataset, result label/key, environment. +- Filters: normalized human-readable list plus short hash. +- Values: actual, expected, diff, tolerance policy. +- Status: pass, fail, inconclusive, missing, stale, or source error. +- Provenance: observed time, Superset source, baseline approval actor/reason/time. + +## Candidate Card + +- Candidate never uses approved styling. +- Shows all source observations and discrepancy warnings. +- Approval control is delegated to the 036 confirmation card. +- Reason is mandatory; exact value/filter/fingerprint target is reviewable. +- Keep draft and discard are distinct from approve. + +## UX Tests + +1. Missing baseline cannot render green/pass. +2. Stale result names changed fingerprint dimensions. +3. Locale-equivalent values show no false diff. +4. API/XLSX discrepancy repeats warning in approval gate. +5. Direct-SQL copy never appears as an offered route. + +#endregion SupersetBaselineEngine.ResultUx diff --git a/specs/037-superset-baseline-engine/contracts/ux/decisions.md b/specs/037-superset-baseline-engine/contracts/ux/decisions.md new file mode 100644 index 000000000..d5b986cc1 --- /dev/null +++ b/specs/037-superset-baseline-engine/contracts/ux/decisions.md @@ -0,0 +1,12 @@ +#region SupersetBaselineEngine.UxDecisions [C:3] [TYPE ADR] [SEMANTICS ux,decisions,baseline] +@BRIEF Final UX decisions for baseline engine outputs consumed by 039. + +1. Always show source identity and “no direct SQL” statement. +2. Never collapse stale, missing, source_error, or inconclusive into fail/pass. +3. Display tolerance policy alongside the result, not only in details. +4. Approved and candidate values use distinct status labels and tones. +5. Fingerprint change is explained by query/dataset/filter category. +6. Candidate approval reuses 036 gate and repeats all discrepancies. +7. Large table diffs show bounded summary plus downloadable evidence, not unbounded DOM rows. + +#endregion SupersetBaselineEngine.UxDecisions diff --git a/specs/037-superset-baseline-engine/data-model.md b/specs/037-superset-baseline-engine/data-model.md new file mode 100644 index 000000000..08fe9ca56 --- /dev/null +++ b/specs/037-superset-baseline-engine/data-model.md @@ -0,0 +1,116 @@ +#region SupersetBaselineEngine.DataModel [C:5] [TYPE ADR] [SEMANTICS data-model,superset,baseline,filter,comparison] +@BRIEF Canonical query, filter, value, baseline, fingerprint, and comparison entities for feature 037. +@RELATION DEPENDS_ON -> [SupersetBaselineEngine.Research] + +## DashboardQueryModel + +- schema_version, environment_id, dashboard_id/title/slug; +- charts: ChartQueryModel[]; +- datasets: DatasetQueryModel[]; +- native_filters: NativeFilterModel[]; +- capabilities: chart_data, dataset_query, xlsx_export; +- warnings: structured source/resource/code/detail; +- query_model_fingerprint. + +ChartQueryModel includes chart id/uuid/title/viz_type, dataset id, metric/result descriptors, group-bys, saved query inputs, applicable_filter_ids, excluded_filter_ids, and execution capability. DatasetQueryModel includes id/uuid/name, columns with semantic types, metrics, and access state. + +## NormalizedFilterContext + +~~~json +{ + "schema_version": 1, + "filters": [ + { + "filter_id": "NATIVE_FILTER-date", + "dataset_id": 77, + "column": "business_date", + "operator": "TEMPORAL_RANGE", + "value": {"from": "2026-05-29", "to": "2026-05-29", "inclusive": true}, + "target_chart_ids": [128] + } + ], + "filters_hash": "sha256" +} +~~~ + +Canonical order is dataset_id, column, operator, canonical value, filter_id. Duplicate semantic filters are rejected unless their scopes are disjoint. + +## SupersetQueryRequest + +Contains environment_id, dashboard_id, chart_id or dataset_id, result_key, and NormalizedFilterContext. It cannot contain SQL, raw endpoint, raw query_context, or arbitrary expression fields. + +## NormalizedValue + +| Field | Meaning | +|---|---| +| kind | null, boolean, integer, decimal, string, date, datetime, percent, table | +| raw | Redacted bounded source value | +| canonical | JSON-safe canonical value; decimal is string | +| display | Optional Superset display value | +| format | Optional format metadata | +| source | environment/dashboard/chart/dataset/result/query hash | +| warnings | normalization warnings | + +Table canonical value contains ordered columns and rows. Each cell is a scalar NormalizedValue payload without recursive source duplication. + +## BaselineEntry + +Required fields: + +- schema_version, baseline_id, dashboard_id; +- chart_id or dataset_id, result_key, label; +- normalized_filters and filters_hash; +- expected NormalizedValue; +- ComparisonPolicy; +- status: approved, superseded, retired; +- query/dataset/filter fingerprints; +- provenance and approval; +- created_at, updated_at. + +Approved files contain no observed runtime history and no secrets. + +## ComparisonPolicy + +Discriminated union: + +- exact; +- absolute_tolerance(amount decimal string); +- relative_tolerance(ratio decimal string, zero_absolute_fallback optional); +- range(min/max and inclusive flags); +- row_set(keys, order_sensitive, allow_extra_rows, per_column policies). + +Policy/value type compatibility is validated when loading catalog and before comparison. + +## BaselineCandidate + +Draft-only derivative containing candidate_id, proposed BaselineEntry, observation set, discrepancies, validation_status, producing agent_run_id, and DraftArtifact id. It has no approved status until 036 consumes an approval gate. + +## FingerprintSet + +All are lowercase SHA-256 over versioned canonical JSON: + +- query_fingerprint; +- dataset_fingerprint; +- filter_fingerprint; +- source_response_hash; +- catalog_entry_hash. + +## ComparisonResult + +Contains status, actual, expected, policy, diff, stale dimensions, warnings, source_error, baseline_id/candidate_id, and evidence refs. Inconclusive always carries a machine code and user-safe explanation. + +## Catalog File + +Path: dashboard_tests/{dashboard_key}/baselines.yaml + +~~~yaml +schema_version: 1 +dashboard: + id: 42 + slug: fi-0080 +entries: [] +~~~ + +Entries sort by chart/dataset identity, result_key, filters_hash, baseline_id. Writer uses atomic temp-file replace inside the resolved repository and only through a consumed 036 gate. + +#endregion SupersetBaselineEngine.DataModel diff --git a/specs/037-superset-baseline-engine/plan.md b/specs/037-superset-baseline-engine/plan.md new file mode 100644 index 000000000..9254ee5af --- /dev/null +++ b/specs/037-superset-baseline-engine/plan.md @@ -0,0 +1,76 @@ +# Implementation Plan: Superset Baseline Engine + +**Branch**: 037-superset-baseline-engine | **Date**: 2026-07-13 | **Spec**: [spec.md](./spec.md) + +## Summary + +Add a backend dashboard-testing domain that inspects saved Superset dashboard/chart/dataset metadata, converts validated native filters to authoritative chart-data query context, normalizes results, and compares them to reviewable approved baselines. Candidate approval and repository persistence reuse 036 bound HITL gates. + +## Technical Context + +**Language/Version**: Python 3.13+ +**Dependencies**: FastAPI 0.126, Pydantic 2, existing async SupersetClient/httpx, ruamel/PyYAML-compatible safe YAML layer already used by repository tooling +**Storage**: Approved YAML in resolved Git worktree; drafts via 036; no new runtime-value table required for MVP +**Testing**: pytest unit/contract; Superset 4.1.2 Testcontainers integration +**Performance Goals**: fixture inspection deterministic; single chart execution p95 under 5s excluding Superset timeout; scalar compare under 10ms +**Constraints**: no arbitrary SQL fields; decimal-safe normalization; async I/O; RBAC and 036 approval binding +**Scale**: up to 100 charts, 20 filters, 10,000 normalized table rows per bounded result + +## Constitution Check + +| Principle | Result | +|---|---| +| Contracts/decision memory | PASS — C4/C5 query and approval boundaries fully contracted | +| External orchestrator | PASS — only Superset REST API is called | +| Module discipline | PASS — inspection, filters, execution, normalization, catalog, comparison separated | +| RBAC | PASS — READ/EXECUTE/APPROVE split; approval rechecked by 036 | +| TDD | PASS — fixtures and rejected SQL path tests precede implementation | +| Async backend | PASS — existing async client; no requests or blocking file I/O on event loop | +| Attention | PASS — BaselineEngine.* hierarchy and shared baseline semantics | + +## Project Structure + +~~~text +backend/src/ +├── api/routes/dashboard_testing.py +├── schemas/dashboard_testing.py +├── services/dashboard_testing/ +│ ├── query_model.py +│ ├── filters.py +│ ├── query_executor.py +│ ├── superset_adapter.py +│ ├── normalization.py +│ ├── fingerprints.py +│ ├── baseline_catalog.py +│ └── comparison.py +└── core/superset_client/ + └── _chart_data.py + +backend/tests/ +├── services/dashboard_testing/ +├── api/test_dashboard_testing.py +└── integration/test_dashboard_testing_superset.py +~~~ + +## Delivery Phases + +1. Canonical DTOs and fixtures. +2. Dashboard query-model inspection and filter normalization. +3. Chart-data adapter/execution with no-SQL guard. +4. Value normalization and comparison policies. +5. YAML catalog, fingerprints, draft candidate, 036 approval integration. +6. API, agent tools, integration and regression gates. + +## Public Boundary + +The API is defined in contracts/dashboard-testing.openapi.yaml. Agent tools call only this backend boundary; they do not call Superset or repository paths directly. + +## Cross-Spec Boundary + +- Requires 036 run/draft/gate substrate. +- Exposes query model, comparison, and candidate DTOs consumed by 038/039. +- Does not build scenario graphs or render the workspace. + +## Complexity Tracking + +No exception planned. Table normalization must enforce row and payload limits rather than loading unbounded Superset results. diff --git a/specs/037-superset-baseline-engine/quickstart.md b/specs/037-superset-baseline-engine/quickstart.md new file mode 100644 index 000000000..a7a5f1bda --- /dev/null +++ b/specs/037-superset-baseline-engine/quickstart.md @@ -0,0 +1,36 @@ +# Quickstart: Superset Baseline Engine + +## Prerequisite + +Complete 036 through approval consume tests. Use checked-in fixtures first; Docker is required only for Superset 4.1.2 integration. + +## Test Order + +~~~bash +cd backend +python -m pytest tests/services/dashboard_testing -v +python -m pytest tests/api/test_dashboard_testing.py -v +python -m pytest tests/integration/test_dashboard_testing_superset.py --run-integration -v +~~~ + +## Independent Smoke + +1. Inspect a fixture dashboard and snapshot the deterministic DashboardQueryModel. +2. Normalize the same date/decimal/list filters in two locale representations; hashes must match. +3. Execute a saved scalar chart via POST /api/v1/chart/data and verify source ids/hash. +4. Execute a table chart with row limit and canonical columns/rows. +5. Attempt requests containing sql, raw query_context, adhoc expression, and endpoint; all must fail before Superset call. +6. Compare exact, absolute, relative, range, and row-set fixtures. +7. Change query/dataset/filter fingerprints independently; each must return stale_baseline. +8. Create a draft candidate and verify approved catalog is unchanged. +9. Approve through a 036 gate with reason; verify one atomic YAML update. +10. Replay or mutate the request; verify 409 and no second write. + +## Exit Gates + +- Deterministic fixture snapshots. +- Decimal/date/percent equivalence without false diffs. +- 403/404/422/timeout/5xx taxonomy preserved. +- No direct SQL request surface or agent tool reachability. +- Baseline schema validates and catalog writer is deterministic. +- Unit/API/integration, ruff, and semantic audits pass. diff --git a/specs/037-superset-baseline-engine/research.md b/specs/037-superset-baseline-engine/research.md new file mode 100644 index 000000000..14cc37760 --- /dev/null +++ b/specs/037-superset-baseline-engine/research.md @@ -0,0 +1,81 @@ +#region SupersetBaselineEngine.Research [C:4] [TYPE ADR] [SEMANTICS research,superset,baseline,chart-data,normalization] +@BRIEF Phase 0 decisions for authoritative dashboard query inspection, Superset-native execution, canonical values, and baseline lifecycle. +@RELATION DEPENDS_ON -> [SupersetBaselineEngine.Spec] +@RELATION DEPENDS_ON -> [AgentTestStabilization.Modules] +@RATIONALE Existing SupersetClient already resolves dashboards, charts, datasets, filters, TLS, and auth, so the feature extends that boundary instead of creating a second HTTP stack. +@REJECTED SQL Lab and arbitrary SQL as baseline truth — rejected by feature scope and because it bypasses dashboard query semantics. + +## 1. Existing Capability Audit + +- SupersetClient already exposes dashboard detail, dashboard charts/datasets, chart detail, dataset detail, native filter state parsing, and async HTTP. +- POST /api/v1/chart/data is present in the checked-in Superset OpenAPI and is already used by translation preview code. +- Existing build_dataset_preview_query_context is a reduced preview builder, not a chart-fidelity engine; it must not be reused as though it reproduces a saved chart. +- Integration infrastructure pins Apache Superset 4.1.2 through ADR-0012. +- The standalone agent currently has SQL tools. Feature 036 must remove them from scenario intent reachability. + +## 2. Query Model Inspection + +- **Decision**: Build DashboardQueryModel from authoritative dashboard, chart, and dataset metadata returned by the selected environment. +- **Filter source**: Parse native_filter_configuration and scope/excluded chart rules from dashboard metadata; keep unresolved/unsupported filter nodes as warnings. +- **Chart source**: For every chart, retain id, dataset reference, viz type, metrics/group-bys, saved form/query context, result capabilities, and metadata fingerprint inputs. +- **Partial access**: Return per-resource warnings and capability=false; never invent chart/dataset fields. +- **Alternative rejected**: Reconstruct query models from rendered DOM — fragile and loses API-level provenance. + +## 3. Superset-Native Execution + +- **Decision**: Public request identifies environment, dashboard, chart/dataset result key, and normalized filters. Backend loads authoritative metadata and builds POST /api/v1/chart/data payload. +- **Invariant**: Request DTO has no sql, query text, adhoc SQL expression, or free-form endpoint field. +- **Validation**: Each filter target must exist on the dataset and be in the dashboard filter scope for the chart. +- **Version adapter**: Keep Superset 4.1.2 response parsing in a narrow adapter so a later Superset version does not leak through domain DTOs. +- **Alternative rejected**: Accept raw query_context from the agent — it can smuggle unsupported expressions and bypass saved-chart constraints. + +## 4. Canonical Filter Identity + +- Sort filters by target dataset/column/operator and canonical value. +- Preserve semantic types: date, datetime, decimal, integer, boolean, string, enum/list, null. +- Use inclusive/exclusive range structure instead of localized strings. +- Compute filters_hash from schema version plus canonical JSON, UTF-8, sorted keys, no insignificant whitespace. +- Scope is part of the query model fingerprint, not of a single chart filter hash. + +## 5. Result Normalization + +- Decimal values use decimal strings, never binary float serialization. +- Dates/datetimes use ISO-8601; timezone-aware datetimes normalize to UTC. +- Percent stores canonical ratio plus display value/format metadata when Superset supplies it. +- Scalar and big-number results identify metric/result key and label. +- Table results use ordered column descriptors and canonical rows; row-set comparison requires explicit key columns or uses multiset semantics. +- Unsupported/ambiguous values produce inconclusive with reason, never false pass. + +## 6. Baseline Artifact and Provenance + +- **Decision**: Approved catalog is reviewable YAML at dashboard_tests/{dashboard_key}/baselines.yaml inside the resolved Git repository. +- **Draft candidates**: Stored as 036 DraftArtifact outside the repository until approved. +- **Runtime observations/evidence**: Stored outside the approved catalog and linked by run/evidence id. +- **Provenance**: Environment, dashboard/chart/dataset ids, result key, normalized filters hash, observed timestamp, Superset version, source response hash, actor/run, and fingerprints. +- **Alternative rejected**: Database-only approved expectations — invisible in review and disconnected from repository history. + +## 7. Staleness + +Three fingerprints are recorded: + +1. query_fingerprint: saved chart/query semantics and result selector; +2. dataset_fingerprint: dataset id/uuid, columns, metric definitions, relevant metadata; +3. filter_fingerprint: dashboard native filter definitions and scope. + +Any mismatch makes an approved entry STALE; it is never silently updated. Discovery creates a new candidate. + +## 8. Comparison Policies + +- exact: canonical equality; +- absolute_tolerance: abs(actual-expected) <= amount; +- relative_tolerance: abs(actual-expected) <= abs(expected) * ratio, with zero rule; +- range: inclusive/exclusive min/max; +- row_set: keyed ordered/unordered policy with missing/extra/changed rows. + +Statuses: pass, fail, inconclusive, missing_baseline, stale_baseline, permission_denied, source_error. + +## 9. Error Taxonomy + +Preserve 401/session, 403 permission, 404 resource, 422 query/filter validation, timeout, 5xx upstream, empty result, normalization unsupported, and catalog invalid as distinct codes. + +#endregion SupersetBaselineEngine.Research diff --git a/specs/037-superset-baseline-engine/spec.md b/specs/037-superset-baseline-engine/spec.md index 88f7497af..8a2e83e11 100644 --- a/specs/037-superset-baseline-engine/spec.md +++ b/specs/037-superset-baseline-engine/spec.md @@ -11,7 +11,7 @@ @SEMANTICS: spec, requirements, feature, superset, baseline, chart-data, dataset, filters, normalization, dashboard-testing **Feature Branch**: `037-superset-baseline-engine` -**Created**: 2026-07-07 | **Status**: Draft +**Created**: 2026-07-07 | **Status**: Ready for Implementation **Input**: "Create a Superset-native query and baseline engine for dashboard testing. The system must inspect dashboard query models, map dashboard native filters into Superset chart or dataset query context, execute Superset-side chart or dataset queries without direct SQL, normalize returned metric and table values, compare them with approved baseline catalog entries, and create draft baseline candidates requiring human approval." ## User Scenarios diff --git a/specs/037-superset-baseline-engine/tasks.md b/specs/037-superset-baseline-engine/tasks.md new file mode 100644 index 000000000..d41a128d7 --- /dev/null +++ b/specs/037-superset-baseline-engine/tasks.md @@ -0,0 +1,68 @@ +#region SupersetBaselineEngine.Tasks [C:3] [TYPE ADR] [SEMANTICS tasks,baseline,implementation] +@BRIEF Ordered TDD backlog for Superset-native baseline engine. + +## Phase 1 — Fixtures and DTO Foundation + +- [ ] T001 Create canonical dashboard/chart/dataset/native-filter Superset fixtures under specs/037-superset-baseline-engine/fixtures/superset/. +- [ ] T002 [P] Create scalar, percent, date, table, empty, malformed, and locale result fixtures under specs/037-superset-baseline-engine/fixtures/results/. +- [ ] T003 [P] Create valid/invalid/stale baseline catalog fixtures under specs/037-superset-baseline-engine/fixtures/baselines/. +- [ ] T004 Materialize fixtures into backend/tests/fixtures/dashboard_testing/. +- [ ] T005 Implement extra-forbid DTOs from contracts/dashboard-testing.openapi.yaml in backend/src/schemas/dashboard_testing.py. + +## Phase 2 — US1 Inspect Dashboard Query Model + +- [ ] T006 [US1] Write failing deterministic inspection tests in backend/tests/services/dashboard_testing/test_query_model.py. +- [ ] T007 [US1] Write failing filter scope/type/hash tests in backend/tests/services/dashboard_testing/test_filters.py. +- [ ] T008 [US1] Implement backend/src/services/dashboard_testing/query_model.py using authoritative SupersetClient metadata. +- [ ] T009 [US1] Implement backend/src/services/dashboard_testing/filters.py with canonical typed values and deterministic hashes. +- [ ] T010 [US1] Add fingerprint helpers in backend/src/services/dashboard_testing/fingerprints.py and cover metadata order invariance. + +**Checkpoint**: Fixture dashboards yield byte-stable models and correct chart/filter scopes. + +## Phase 3 — US2 Superset-Native Execution + +- [ ] T011 [US2] Write failing no-SQL schema and payload tests in backend/tests/services/dashboard_testing/test_query_executor.py. +- [ ] T012 [US2] Add backend/src/core/superset_client/_chart_data.py adapter for saved-chart POST /api/v1/chart/data. +- [ ] T013 [US2] Implement backend/src/services/dashboard_testing/query_executor.py: reload authoritative metadata, scope filters, bound limits, typed errors. +- [ ] T014 [US2] Add agent tools inspect_dashboard_query_model and execute_dashboard_result in agent/src/ss_tools/agent/tools.py as thin backend clients. +- [ ] T015 [US2] Verify scenario intent tool pipeline includes these tools and excludes superset_execute_sql. + +**Checkpoint**: Scalar and table fixtures execute through chart-data only; injected SQL/raw context cannot reach Superset. + +## Phase 4 — US3 Normalize and Compare + +- [ ] T016 [US3] Write failing normalization tests in backend/tests/services/dashboard_testing/test_normalization.py. +- [ ] T017 [US3] Implement normalization.py with Decimal strings, ISO temporal values, percent metadata, and bounded tables. +- [ ] T018 [US3] Write failing comparison policy tests in backend/tests/services/dashboard_testing/test_comparison.py. +- [ ] T019 [US3] Implement exact/absolute/relative/range/row-set comparison in backend/src/services/dashboard_testing/comparison.py. +- [ ] T020 [US3] Add empty/unsupported/duplicate-row-key cases that must return inconclusive to backend/tests/services/dashboard_testing/test_comparison.py. +- [ ] T021 [US3] Add independent query/dataset/filter staleness tests in backend/tests/services/dashboard_testing/test_staleness.py. +- [ ] T022 [US3] Add deterministic diff/evidence reference output tests in backend/tests/services/dashboard_testing/test_comparison.py. + +## Phase 5 — US4 Baseline Candidate Lifecycle + +- [ ] T023 [US4] Write failing JSON-schema/YAML/path tests in backend/tests/services/dashboard_testing/test_baseline_catalog.py. +- [ ] T024 [US4] Implement safe catalog loader and deterministic writer in backend/src/services/dashboard_testing/baseline_catalog.py. +- [ ] T025 [US4] Resolve repository only through GitService and reject traversal/symlink escape. +- [ ] T026 [US4] Validate catalogs against contracts/baseline-catalog.schema.json before use/write. +- [ ] T027 [US4] Write failing candidate provenance/discrepancy tests in backend/tests/services/dashboard_testing/test_candidates.py. +- [ ] T028 [US4] Implement candidate creation in backend/src/services/dashboard_testing/candidates.py as 036 DraftArtifact; never update approved YAML. +- [ ] T029 [US4] Implement request-baseline-approval endpoint in backend/src/api/routes/dashboard_testing.py using 036 request hash and reason_required. +- [ ] T030 [US4] Implement approval consume callback in backend/src/services/dashboard_testing/candidates.py that atomically writes YAML and records gate id/reason. +- [ ] T031 [US4] Cover stale-after-confirm, RBAC revoked, payload mutation, and replay in backend/tests/services/dashboard_testing/test_candidates.py. +- [ ] T032 [US4] Add agent tools discover_baseline_candidate and request_baseline_approval without direct approve capability. + +## Phase 6 — API and Integration + +- [ ] T033 Add backend/src/api/routes/dashboard_testing.py matching OpenAPI and register router. +- [ ] T034 Write API contract/RBAC tests in backend/tests/api/test_dashboard_testing.py. +- [ ] T035 Add Superset 4.1.2 Testcontainers test in backend/tests/integration/test_dashboard_testing_superset.py. +- [ ] T036 Add compatibility tests for existing translation preview chart-data usage in backend/tests/core/superset_client/test_chart_data.py. +- [ ] T037 Run quickstart, backend full relevant tests, ruff, and OpenAPI/schema validation. +- [ ] T038 Audit C3+ contracts, direct-SQL ban, async boundaries, ATTN_1–4, and unresolved relations. + +## Dependencies + +T001–T005 → US1 → US2 → US3; US4 depends on US3 and completed 036. API integration follows all domain contracts. + +#endregion SupersetBaselineEngine.Tasks diff --git a/specs/037-superset-baseline-engine/traceability.md b/specs/037-superset-baseline-engine/traceability.md new file mode 100644 index 000000000..38db7ad65 --- /dev/null +++ b/specs/037-superset-baseline-engine/traceability.md @@ -0,0 +1,18 @@ +#region SupersetBaselineEngine.Traceability [C:3] [TYPE ADR] [SEMANTICS traceability,baseline,requirements] +@BRIEF Requirement-to-contract-to-task-to-test matrix for feature 037. + +| Requirement | Contract | Tasks | Test | +|---|---|---|---| +| AGBASE-FR-001 | BaselineEngine.QueryModel.Inspect | T006–T010 | query-model snapshots/scopes | +| AGBASE-FR-002, AGBASE-FR-009 | SupersetClient.ChartData.Execute | T011–T015 | malicious fields and Testcontainers | +| AGBASE-FR-003 | BaselineEngine.Filters.Normalize | T007–T010 | type/scope/hash cases | +| AGBASE-FR-004 | BaselineEngine.Result.Normalize | T016–T019 | scalar/table/date/decimal/percent | +| AGBASE-FR-005, AGBASE-FR-006 | BaselineEngine.Catalog.Load | T023–T027 | schema/path/deterministic YAML | +| AGBASE-FR-007 | Candidate.Create/Approve | T028–T032 | draft and bound approval | +| AGBASE-FR-008 | BaselineEngine.Comparison.Compare | T020–T022 | all policies/statuses | + +## Dependencies + +Upstream: 036 draft/gate contracts. Downstream: 038 baseline refs and 039 baseline impact panel. + +#endregion SupersetBaselineEngine.Traceability diff --git a/specs/037-superset-baseline-engine/ux_reference.md b/specs/037-superset-baseline-engine/ux_reference.md index 1c07f130f..fbbdba849 100644 --- a/specs/037-superset-baseline-engine/ux_reference.md +++ b/specs/037-superset-baseline-engine/ux_reference.md @@ -2,7 +2,7 @@ @BRIEF UX reference for Superset-native baseline execution and comparison results. **Feature Branch**: `037-superset-baseline-engine` -**Created**: 2026-07-07 | **Status**: Draft +**Created**: 2026-07-07 | **Status**: Ready for Implementation ## 1. User Persona & Context diff --git a/specs/038-dashboard-scenario-model/checklist-catalog.md b/specs/038-dashboard-scenario-model/checklist-catalog.md new file mode 100644 index 000000000..1511eb3b0 --- /dev/null +++ b/specs/038-dashboard-scenario-model/checklist-catalog.md @@ -0,0 +1,58 @@ +#region DashboardScenarioModel.ChecklistCatalog [C:4] [TYPE ADR] [SEMANTICS scenario,checklist,catalog,pdf] +@BRIEF Normalized reusable catalog of all 19 cases from research/Чеклист 29.05 (1).pdf. +@RELATION DEPENDS_ON -> [DashboardScenarioModel.DataModel] +@RATIONALE Separating reusable intent from FI-0080 data and historic outcomes prevents the source PDF from becoming hidden implementation logic. +@REJECTED Copying historic pass/fail text into expected assertions — it is prior evidence, not future truth. + +**Catalog version**: 1 +**Source date/title**: 29.05.2026, “Чеклист 29.05” +**Rule**: every case is classified; none is silently omitted. + +## Basic Cases + +| ID | Reusable goal | Required capabilities | Parameters/evidence | Preferred mapping | +|---|---|---|---|---| +| B01 | Main dashboard filters change report data consistently | native_filters, browser | filter values; before/after state; API evidence when available | browser apply + superset_api observe + assertion | +| B02 | Text-search filters display and apply entered value | text_filter, browser | text values; visible chip/input and resulting rows | browser + screenshot/assertion | +| B03 | In-table column filter returns matching rows/count | table_filter, browser | column/value; row/count evidence | browser + table assertion | +| B04 | Pagination/page-size changes update rows correctly | pagination, browser | target page/size; row identity evidence | browser + assertion | +| B05 | Single-row comment/reason/status persists after save | row_edit, browser | row key, comment, reason, status | browser interaction + refresh evidence; human if write test data unavailable | +| B06 | Bulk edit applies values to all selected rows | bulk_edit, browser | at least two row keys and values | browser + post-save evidence | +| B07 | Blank bulk fields do not overwrite existing values | bulk_edit, persistence_refresh | rows with existing values | browser + before/after assertion | +| B08 | Bulk edit works for more than 100 records | bulk_edit, browser | safe dataset/filter producing >100 rows | browser + count/evidence; manual if safe test data absent | +| B09 | Saved comment remains visible after page refresh | row_edit, persistence_refresh | row key and expected submitted text | browser refresh + assertion/screenshot | + +## Complex Cases + +| ID | Reusable goal | Required capabilities | Parameters/evidence | Preferred mapping | +|---|---|---|---|---| +| C01 | Combined dashboard and table filters remain consistent during edit | native_filters, table_filter, row_edit | filter set, row key, edit values | browser chain + evidence | +| C02 | “Needs filling” time-window behavior changes after 1–7 and over 8 days | time_rollover, native_filters | controlled dates/test data | scheduled/manual checkpoint unless safe clock fixture exists | +| C03 | Rolling comments remain distinct per business date | time_rollover, row_edit | two dates, same row key, two comments | browser + Superset API evidence when exposed | +| C04 | XLSX export completes and yields readable workbook | xlsx_export, browser | export action; file metadata | browser download + xlsx parse | +| C05 | XLSX export reflects dashboard-level filters | xlsx_export, native_filters | filter set; API/UI/XLSX values | browser + superset_api + xlsx + assertion | +| C06 | XLSX export reflects table-level filters | xlsx_export, table_filter | table filter; visible and workbook rows | browser + xlsx row-set assertion | +| C07 | Related dashboard/tab displays expected persisted comments | cross_dashboard, browser | target dashboard/tab and row key | browser navigation + evidence; needs_context if relation absent | + +## Technical Cases + +| ID | Reusable goal | Required capabilities | Safe mapping | Forbidden mapping | +|---|---|---|---|---| +| T01 | Verify persisted comment fields and author/timestamp semantics | dataset_field_read or human | Superset saved dataset/chart result; otherwise human checkpoint | SQL Lab or generated SELECT | +| T02 | Verify rolling records share row identifier but retain distinct dates/values | dataset_field_read or human | Superset dataset result and row-set assertion; otherwise human | SQL text | +| T03 | Verify blank bulk edit did not create invalid empty/zero combinations | dataset_field_read or human | Superset dataset result with structural assertion; otherwise human | SQL text | + +## Historic Source Notes + +The PDF records historic outcomes such as “Успешно пройдено”, known remarks for B05/B07, and failures for C02/T03. These notes may inform warnings or regression rationale but are never copied as baseline values or current expected outcomes. + +## Mapping Rules + +1. Missing required capability → human_checkpoint when safe manual verification exists, otherwise unsupported. +2. Missing parameter, selector, relationship, or test data → needs_context. +3. Mutating cases B05–B09/C01/C03 require explicit safe environment/test-data context. +4. T01–T03 must satisfy the 037 no-direct-SQL invariant. +5. C02 must not simulate elapsed time against production records without an approved test fixture. +6. Each mapping records selected template, rationale, and resulting step ids. + +#endregion DashboardScenarioModel.ChecklistCatalog diff --git a/specs/038-dashboard-scenario-model/checklists/requirements.md b/specs/038-dashboard-scenario-model/checklists/requirements.md index eef1839f1..111e9f989 100644 --- a/specs/038-dashboard-scenario-model/checklists/requirements.md +++ b/specs/038-dashboard-scenario-model/checklists/requirements.md @@ -23,3 +23,12 @@ - [x] CHK009 Key entities define scenario, steps, refs, parameters, checklist cases, mappings, and validation result. - [x] CHK010 Success criteria are measurable with deterministic fixtures and snapshots. - [x] CHK011 Scope excludes UI implementation and Superset query engine internals. + +## Implementation Package + +- [x] CHK012 Research and implementation plan resolve all design decisions. +- [x] CHK013 Data model, checklist catalog, module, API, JSON Schema, and UX contracts are present. +- [x] CHK014 All 19 source-checklist cases have an explicit automation classification. +- [x] CHK015 Traceability maps every functional requirement to contracts, tasks, and tests. +- [x] CHK016 Tasks use exact repository paths, dependency order, and test-first sequencing. +- [x] CHK017 Machine-readable contracts, external references, and semantic anchors pass validation. diff --git a/specs/038-dashboard-scenario-model/contracts/dashboard-test-scenario.schema.json b/specs/038-dashboard-scenario-model/contracts/dashboard-test-scenario.schema.json new file mode 100644 index 000000000..f420bc1e0 --- /dev/null +++ b/specs/038-dashboard-scenario-model/contracts/dashboard-test-scenario.schema.json @@ -0,0 +1,178 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://superset-tools.local/schemas/dashboard-test-scenario-v1.json", + "title": "DashboardTestScenario", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", "compiler_version", "template_version", "scenario_id", + "revision_hash", "dashboard_context", "objective", "input_fingerprints", + "parameters", "phases", "steps", "outputs", "artifact_plan", + "checklist_coverage", "warnings", "blockers", "risk_summary" + ], + "properties": { + "schema_version": { "const": 1 }, + "compiler_version": { "type": "string" }, + "template_version": { "type": "string" }, + "scenario_id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,127}$" }, + "revision_hash": { "$ref": "#/$defs/sha256" }, + "parent_revision_hash": { "oneOf": [{ "$ref": "#/$defs/sha256" }, { "type": "null" }] }, + "dashboard_context": { + "type": "object", + "additionalProperties": false, + "required": ["environment_id", "dashboard_id", "dashboard_name"], + "properties": { + "environment_id": { "type": "string" }, + "dashboard_id": { "type": "integer", "minimum": 1 }, + "dashboard_name": { "type": "string" }, + "route": { "type": "string" } + } + }, + "objective": { + "type": "object", + "additionalProperties": false, + "required": ["goal", "selected_case_ids"], + "properties": { + "goal": { "type": "string", "minLength": 1, "maxLength": 2000 }, + "selected_case_ids": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/$defs/checklistId" } + }, + "rationale": { "type": ["string", "null"], "maxLength": 4000 } + } + }, + "input_fingerprints": { + "type": "object", + "additionalProperties": false, + "required": ["query_model", "checklist_catalog", "baseline_summary", "parameters"], + "properties": { + "query_model": { "$ref": "#/$defs/sha256" }, + "checklist_catalog": { "$ref": "#/$defs/sha256" }, + "baseline_summary": { "$ref": "#/$defs/sha256" }, + "parameters": { "$ref": "#/$defs/sha256" } + } + }, + "parameters": { "type": "array", "items": { "$ref": "#/$defs/parameter" } }, + "phases": { + "type": "array", + "items": { "enum": ["setup", "interact", "observe", "assert", "evidence", "report"] } + }, + "steps": { "type": "array", "minItems": 1, "maxItems": 100, "items": { "$ref": "#/$defs/step" } }, + "outputs": { "type": "array", "items": { "$ref": "#/$defs/ref" } }, + "artifact_plan": { "type": "array", "items": { "$ref": "#/$defs/artifactPlan" } }, + "checklist_coverage": { "type": "array", "items": { "$ref": "#/$defs/coverage" } }, + "warnings": { "type": "array", "items": { "$ref": "#/$defs/finding" } }, + "blockers": { "type": "array", "items": { "$ref": "#/$defs/finding" } }, + "risk_summary": { "type": "object", "additionalProperties": true } + }, + "$defs": { + "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "checklistId": { "type": "string", "pattern": "^(B0[1-9]|C0[1-7]|T0[1-3])$" }, + "parameter": { + "type": "object", + "additionalProperties": false, + "required": ["name", "label", "type", "required", "source", "status", "affected_step_ids"], + "properties": { + "name": { "type": "string", "pattern": "^[a-z][a-z0-9_]{1,63}$" }, + "label": { "type": "string" }, + "type": { + "enum": ["string", "integer", "decimal", "boolean", "date", "datetime", "enum", "string_list", "baseline_choice", "selector_hint"] + }, + "required": { "type": "boolean" }, + "default": {}, + "source": { "type": ["string", "null"] }, + "validation": { "type": ["object", "null"] }, + "value": {}, + "status": { "enum": ["unresolved", "resolved", "invalid"] }, + "affected_step_ids": { "type": "array", "uniqueItems": true, "items": { "type": "string" } } + } + }, + "ref": { + "type": "object", + "additionalProperties": false, + "required": ["name", "kind"], + "properties": { + "name": { + "type": "string", + "pattern": "^(context|param|step|baseline|candidate|artifact)\\.[A-Za-z0-9_.-]+$" + }, + "kind": { "enum": ["context", "parameter", "step_output", "baseline", "candidate", "artifact"] }, + "value_type": { "type": ["string", "null"] } + } + }, + "expected": { + "type": "object", + "additionalProperties": false, + "required": ["kind"], + "properties": { + "kind": { "enum": ["structural", "baseline_ref", "candidate_ref", "human_evidence"] }, + "ref": { "type": ["string", "null"] }, + "predicate": { "type": ["string", "null"] }, + "description": { "type": ["string", "null"], "maxLength": 2000 } + } + }, + "step": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", "phase", "title", "tool", "action", "inputs", "outputs", + "expected", "depends_on", "automation_status", "checklist_case_ids", "risk" + ], + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,127}$" }, + "phase": { "enum": ["setup", "interact", "observe", "assert", "evidence", "report"] }, + "title": { "type": "string", "maxLength": 300 }, + "description": { "type": ["string", "null"], "maxLength": 2000 }, + "tool": { "enum": ["browser", "superset_api", "xlsx", "assertion", "screenshot", "report", "artifact", "human"] }, + "action": { "type": "string", "pattern": "^[a-z][a-z0-9_]{1,63}$" }, + "inputs": { "type": "array", "items": { "$ref": "#/$defs/ref" } }, + "outputs": { "type": "array", "items": { "$ref": "#/$defs/ref" } }, + "expected": { "$ref": "#/$defs/expected" }, + "depends_on": { "type": "array", "uniqueItems": true, "items": { "type": "string" } }, + "automation_status": { + "enum": ["ready", "needs_context", "needs_selector", "needs_baseline", "manual", "unsupported", "blocked"] + }, + "checklist_case_ids": { "type": "array", "uniqueItems": true, "items": { "$ref": "#/$defs/checklistId" } }, + "risk": { "enum": ["read", "browser_interaction", "draft_write", "human"] } + } + }, + "artifactPlan": { + "type": "object", + "additionalProperties": false, + "required": ["artifact_key", "kind", "relative_path_template", "template_id", "input_refs", "required"], + "properties": { + "artifact_key": { "type": "string" }, + "kind": { "enum": ["scenario", "runner_plan", "report_template", "evidence_manifest", "browser_module", "xlsx_module", "baseline_candidate"] }, + "relative_path_template": { "type": "string", "pattern": "^(?!/)(?!.*\\.\\.).+$" }, + "template_id": { "type": "string" }, + "input_refs": { "type": "array", "items": { "type": "string" } }, + "required": { "type": "boolean" }, + "generation_blockers": { "type": "array", "items": { "type": "string" } } + } + }, + "coverage": { + "type": "object", + "required": ["case_id", "classification", "rationale", "step_ids"], + "properties": { + "case_id": { "$ref": "#/$defs/checklistId" }, + "classification": { "enum": ["automated", "human_checkpoint", "unsupported", "needs_context"] }, + "rationale": { "type": "string" }, + "step_ids": { "type": "array", "items": { "type": "string" } } + } + }, + "finding": { + "type": "object", + "required": ["code", "severity", "message"], + "properties": { + "code": { "type": "string" }, + "severity": { "enum": ["info", "warning", "error", "blocker"] }, + "message": { "type": "string" }, + "json_pointer": { "type": ["string", "null"] }, + "step_id": { "type": ["string", "null"] }, + "case_id": { "oneOf": [{ "$ref": "#/$defs/checklistId" }, { "type": "null" }] }, + "recovery_options": { "type": "array", "items": { "type": "string" } } + } + } + } +} diff --git a/specs/038-dashboard-scenario-model/contracts/modules.md b/specs/038-dashboard-scenario-model/contracts/modules.md new file mode 100644 index 000000000..fd9961f79 --- /dev/null +++ b/specs/038-dashboard-scenario-model/contracts/modules.md @@ -0,0 +1,114 @@ +#region DashboardScenarioModel.Modules [C:5] [TYPE ADR] [SEMANTICS contracts,scenario,graph,validator,compiler] +@BRIEF C3+ contracts for checklist mapping, deterministic graph compilation, validation, resolution, serialization, and safe draft packs. +@RELATION DEPENDS_ON -> [DashboardScenarioModel.DataModel] +@RELATION DEPENDS_ON -> [DashboardScenarioModel.ChecklistCatalog] +@RELATION DEPENDS_ON -> [SupersetBaselineEngine.Modules] +@RELATION DEPENDS_ON -> [AgentRuns.Artifacts.Register] + +# #region ScenarioGraph.Api [C:4] [TYPE Module] [SEMANTICS scenario,api,compile,validate] +# @defgroup ScenarioGraph REST surface for compile, validate, resolve, and draft-pack operations. +# @LAYER API +# @RELATION DEPENDS_ON -> [ScenarioGraph.Compiler] +# @RELATION DEPENDS_ON -> [ScenarioGraph.Validator] +# @RELATION DEPENDS_ON -> [ScenarioGraph.Resolver] +# @RELATION DEPENDS_ON -> [ScenarioGraph.PackCompiler] +# @INVARIANT Request schemas forbid executable code, SQL, raw baseline values, and local paths. +# #endregion ScenarioGraph.Api + +# #region ScenarioGraph.Catalog.Load [C:4] [TYPE Function] [SEMANTICS scenario,checklist,catalog,version] +# @ingroup ScenarioGraph +# @BRIEF Load the versioned 19-case declarative catalog and validate ids/capability/template references. +# @PRE Bundled catalog version is supported. +# @POST Returns exactly B01–B09, C01–C07, T01–T03 in stable order. +# @SIDE_EFFECT Bounded package-resource read. +# @DATA_CONTRACT CatalogResource -> ChecklistCase[19] +# @INVARIANT Historic PDF outcomes are source notes, not expected values. +# @TEST_EDGE missing_case -> startup/catalog validation failure. +# @TEST_EDGE sql_template_for_technical_case -> rejected. +# #endregion ScenarioGraph.Catalog.Load + +# #region ScenarioGraph.CapabilityMapper.Map [C:5] [TYPE Function] [SEMANTICS scenario,capability,mapping,coverage] +# @ingroup ScenarioGraph +# @BRIEF Classify every checklist case for a dashboard and select one allowed step template. +# @PRE Query model, baseline summary, and capability registry validate. +# @POST Every catalog case is automated, human_checkpoint, unsupported, or needs_context with rationale. +# @SIDE_EFFECT None. +# @DATA_CONTRACT ChecklistCase[] + DashboardCapabilities -> CapabilityMapping[] +# @INVARIANT No case is dropped and no tool is selected outside its registered capabilities. +# @TEST_EDGE xlsx_unavailable -> C04–C06 manual/unsupported with rationale. +# @TEST_EDGE technical_without_dataset_fields -> human checkpoint, no SQL. +# #endregion ScenarioGraph.CapabilityMapper.Map + +# #region ScenarioGraph.Compiler.Compile [C:5] [TYPE Function] [SEMANTICS scenario,compiler,deterministic,dag] +# @ingroup ScenarioGraph +# @BRIEF Compile canonical inputs and mappings into a stable dashboard-specific DAG. +# @PRE Intent, query model, catalog, baseline summary, and parameters have valid fingerprints. +# @POST Same canonical inputs/compiler version yield byte-identical graph and stable ids/order. +# @SIDE_EFFECT None. +# @DATA_CONTRACT CompileScenarioRequest -> DashboardTestScenario +# @INVARIANT Steps consume only context/parameter/baseline/earlier-step refs. +# @TEST_INVARIANT Deterministic_Graph -> VERIFIED_BY: repeated_compile, shuffled_input_order. +# @TEST_EDGE missing_selector -> NEEDS_SELECTOR step and save blocker. +# @TEST_EDGE missing_baseline -> NEEDS_BASELINE; no embedded numeric truth. +# @RATIONALE Rule/template compilation makes the agent a planner/explainer, not an executable-code generator. +# @REJECTED LLM-generated ids/dependencies/code — non-deterministic and unsafe. +# #endregion ScenarioGraph.Compiler.Compile + +# #region ScenarioGraph.Validator.Validate [C:5] [TYPE Function] [SEMANTICS scenario,validator,graph,safety] +# @ingroup ScenarioGraph +# @BRIEF Return complete deterministic findings for schema, DAG, refs, parameters, baselines, tools, safety, and coverage. +# @PRE Candidate graph parses against supported schema version. +# @POST Valid is true only with zero errors/blockers; findings are stably ordered and actionable. +# @SIDE_EFFECT None. +# @DATA_CONTRACT DashboardTestScenario -> ScenarioValidationResult +# @INVARIANT Cycles, missing/duplicate refs, unregistered tools, SQL, raw metric truth, and path traversal block compilation. +# @TEST_EDGE cycle -> error contains cycle path. +# @TEST_EDGE duplicate_output -> both producer ids reported. +# @TEST_EDGE raw_metric_expected -> forbidden baseline literal error. +# @TEST_EDGE unreachable_step -> warning/error according to required coverage. +# #endregion ScenarioGraph.Validator.Validate + +# #region ScenarioGraph.Resolver.Resolve [C:4] [TYPE Function] [SEMANTICS scenario,resolve,parameter,revision] +# @ingroup ScenarioGraph +# @BRIEF Apply typed parameter/selector/manual resolutions and emit an immutable scenario revision. +# @PRE Base revision hash matches; changes target declared unresolved items. +# @POST Unrelated step ids/order remain unchanged; new parent/revision hashes link revisions. +# @SIDE_EFFECT None. +# @DATA_CONTRACT ResolveScenarioRequest + BaseScenario -> DashboardTestScenario +# @TEST_EDGE stale_base_revision -> 409. +# @TEST_EDGE invalid_parameter_type -> 422. +# @TEST_EDGE unrelated_graph_change -> invariant failure. +# #endregion ScenarioGraph.Resolver.Resolve + +# #region ScenarioGraph.Serializer.Canonical [C:4] [TYPE Function] [SEMANTICS scenario,serialize,json,yaml] +# @ingroup ScenarioGraph +# @BRIEF Serialize graph to canonical JSON/YAML and compute revision hash. +# @POST Key/order/decimal/date/newline rules are stable across runs; JSON and YAML represent equal domain data. +# @SIDE_EFFECT None. +# @DATA_CONTRACT DashboardTestScenario -> CanonicalBytes + SHA256 +# @TEST_EDGE shuffled_dicts -> identical bytes. +# @TEST_EDGE timestamp_display_field -> excluded from revision identity. +# #endregion ScenarioGraph.Serializer.Canonical + +# #region ScenarioGraph.PackCompiler.Generate [C:5] [TYPE Function] [SEMANTICS scenario,artifact,template,draft] +# @ingroup ScenarioGraph +# @BRIEF Generate a preview-only or save-eligible draft pack using registered versioned templates. +# @PRE Scenario validation result available; template ids registered; target paths safe. +# @POST Outputs match ArtifactPlan, contain no LLM executable bodies, and are registered as 036 drafts. +# @SIDE_EFFECT Renders bounded templates and calls AgentRuns.Artifacts.Register. +# @DATA_CONTRACT DashboardTestScenario + ValidationResult -> DraftPack +# @INVARIANT Errors/unresolved required inputs make pack preview_only; direct code/path input is impossible. +# @TEST_INVARIANT No_LLM_To_Code -> VERIFIED_BY: injected_code_field, template_registry_only. +# @TEST_EDGE unknown_template -> blocked. +# @TEST_EDGE path_traversal -> blocked before artifact registration. +# @RATIONALE Versioned templates make generated behavior reviewable and reproducible. +# @REJECTED Generate arbitrary Playwright/Python code then scan it — scanners cannot prove semantic safety. +# #endregion ScenarioGraph.PackCompiler.Generate + +# #region AgentChat.Tools.ScenarioGraph [C:4] [TYPE Module] [SEMANTICS scenario,agent,tools,compiler] +# @defgroup ScenarioGraph Thin agent tools that submit bounded intent and display compiler/validator results. +# @RELATION DEPENDS_ON -> [ScenarioGraph.Api] +# @INVARIANT Agent cannot submit executable code, custom tool categories, raw expected metrics, or artifact paths. +# #endregion AgentChat.Tools.ScenarioGraph + +#endregion DashboardScenarioModel.Modules diff --git a/specs/038-dashboard-scenario-model/contracts/scenario-api.openapi.yaml b/specs/038-dashboard-scenario-model/contracts/scenario-api.openapi.yaml new file mode 100644 index 000000000..9249332ac --- /dev/null +++ b/specs/038-dashboard-scenario-model/contracts/scenario-api.openapi.yaml @@ -0,0 +1,144 @@ +openapi: 3.1.0 +info: + title: Dashboard Scenario Compiler API + version: 0.1.0 +servers: + - url: /api/dashboard-testing +paths: + /scenarios/compile: + post: + operationId: compileDashboardScenario + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/CompileRequest' } + responses: + '200': + description: Deterministic scenario plus validation + content: { application/json: { schema: { $ref: '#/components/schemas/ScenarioResponse' } } } + '422': { description: Invalid canonical inputs } + /scenarios/validate: + post: + operationId: validateDashboardScenario + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: + $ref: './dashboard-test-scenario.schema.json' + responses: + '200': + description: Complete validation findings + content: { application/json: { schema: { $ref: '#/components/schemas/ValidationResult' } } } + /scenarios/{scenarioId}/resolve: + post: + operationId: resolveDashboardScenario + security: [{ bearerAuth: [] }] + parameters: + - { name: scenarioId, in: path, required: true, schema: { type: string } } + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/ResolveRequest' } + responses: + '200': + description: New immutable revision + content: { application/json: { schema: { $ref: '#/components/schemas/ScenarioResponse' } } } + '409': { description: Stale base revision } + '422': { description: Invalid resolution } + /scenarios/{scenarioId}/draft-pack: + post: + operationId: compileScenarioDraftPack + security: [{ bearerAuth: [] }] + parameters: + - { name: scenarioId, in: path, required: true, schema: { type: string } } + requestBody: + required: true + content: + application/json: + schema: + type: object + additionalProperties: false + required: [agent_run_id, revision_hash] + properties: + agent_run_id: { type: string, format: uuid } + revision_hash: { type: string, pattern: '^[a-f0-9]{64}$' } + responses: + '201': + description: Draft pack registered via feature 036 + content: { application/json: { schema: { $ref: '#/components/schemas/DraftPack' } } } + '409': { description: Revision changed } + '422': { description: Template, validation, or path failure } +components: + securitySchemes: + bearerAuth: { type: http, scheme: bearer, bearerFormat: JWT } + schemas: + CompileRequest: + type: object + additionalProperties: false + required: [agent_run_id, objective, query_model, checklist_catalog_version, baseline_summary, capabilities, parameters] + properties: + agent_run_id: { type: string, format: uuid } + objective: + type: object + required: [goal, selected_case_ids] + properties: + goal: { type: string, maxLength: 2000 } + selected_case_ids: { type: array, items: { type: string } } + rationale: { type: [string, 'null'], maxLength: 4000 } + query_model: { type: object } + checklist_catalog_version: { const: 1 } + baseline_summary: { type: object } + capabilities: { type: object } + parameters: { type: object } + ValidationResult: + type: object + required: [valid, errors, warnings, blockers, coverage, topological_order, graph_hash] + properties: + valid: { type: boolean } + errors: { type: array, items: { type: object } } + warnings: { type: array, items: { type: object } } + blockers: { type: array, items: { type: object } } + coverage: { type: array, items: { type: object } } + topological_order: { type: array, items: { type: string } } + unresolved_parameters: { type: array, items: { type: string } } + unresolved_selectors: { type: array, items: { type: string } } + unresolved_baselines: { type: array, items: { type: string } } + graph_hash: { type: string } + ScenarioResponse: + type: object + required: [scenario, validation] + properties: + scenario: { $ref: './dashboard-test-scenario.schema.json' } + validation: { $ref: '#/components/schemas/ValidationResult' } + ResolveRequest: + type: object + additionalProperties: false + required: [base_revision_hash, changes] + properties: + base_revision_hash: { type: string, pattern: '^[a-f0-9]{64}$' } + changes: + type: array + items: + type: object + required: [kind, target, value] + properties: + kind: { enum: [parameter, selector, manual_conversion, remove_step] } + target: { type: string } + value: {} + reason: { type: [string, 'null'] } + DraftPack: + type: object + required: [scenario_revision_hash, template_version, status, manifest, artifacts, validation_summary, warnings] + properties: + scenario_revision_hash: { type: string } + template_version: { type: string } + status: { enum: [preview_only, save_eligible] } + manifest: { type: object } + artifacts: { type: array, items: { type: object } } + validation_summary: { $ref: '#/components/schemas/ValidationResult' } + warnings: { type: array, items: { type: object } } diff --git a/specs/038-dashboard-scenario-model/contracts/ux/api-ux.md b/specs/038-dashboard-scenario-model/contracts/ux/api-ux.md new file mode 100644 index 000000000..456b830cc --- /dev/null +++ b/specs/038-dashboard-scenario-model/contracts/ux/api-ux.md @@ -0,0 +1,13 @@ +#region DashboardScenarioModel.ApiUx [C:3] [TYPE ADR] [SEMANTICS ux,api,scenario,validation] +@BRIEF UI mapping for compile, validate, resolve, and draft-pack endpoints. + +| Operation | Optimistic UI | Authoritative result | +|---|---|---| +| compile | inspect/scenario progress active | Graph, coverage and findings replace placeholder | +| validate | validation pending | errors/warnings/blockers grouped by step/case | +| resolve | affected controls pending only | New revision updates impacted statuses, retains other ids | +| draft-pack | generate/validate progress | Manifest and 036 DraftArtifact refs | + +409 stale revision triggers snapshot/recompile guidance; it never merges silently. 422 findings map to fields/steps. 403 renders permission denial without an approval gate. + +#endregion DashboardScenarioModel.ApiUx diff --git a/specs/038-dashboard-scenario-model/contracts/ux/decisions.md b/specs/038-dashboard-scenario-model/contracts/ux/decisions.md new file mode 100644 index 000000000..7732efed1 --- /dev/null +++ b/specs/038-dashboard-scenario-model/contracts/ux/decisions.md @@ -0,0 +1,12 @@ +#region DashboardScenarioModel.UxDecisions [C:3] [TYPE ADR] [SEMANTICS ux,decisions,scenario] +@BRIEF Final display and interaction decisions for scenario graph consumers. + +1. Show a business flow first; tool category is secondary evidence. +2. Always provide a linear accessible step table beside/under the visual DAG. +3. Keep all 19 checklist cases visible in coverage, including unsupported/manual. +4. Resolution controls operate on declared parameters/selectors, not arbitrary graph editing. +5. Revision hash changes are visible; stale edits are rejected, not auto-merged. +6. Preview-only and save-eligible are explicit pack states. +7. Technical PDF cases never show SQL as an offered execution path. + +#endregion DashboardScenarioModel.UxDecisions diff --git a/specs/038-dashboard-scenario-model/contracts/ux/scenario-graph-ux.md b/specs/038-dashboard-scenario-model/contracts/ux/scenario-graph-ux.md new file mode 100644 index 000000000..51170fe0e --- /dev/null +++ b/specs/038-dashboard-scenario-model/contracts/ux/scenario-graph-ux.md @@ -0,0 +1,24 @@ +#region DashboardScenarioModel.GraphUx [C:4] [TYPE ADR] [SEMANTICS ux,scenario,graph,coverage] +@BRIEF Presentation contract for scenario summary, DAG, steps, parameters, coverage, warnings, and blockers. + +## Views + +- Summary: objective, step count, tool categories, revision, blockers/warnings. +- Phase graph: stable topological phase lanes; keyboard-accessible step list is the semantic fallback. +- Step table: title, tool, expected kind, automation status, dependencies, checklist refs. +- Coverage: all B01–B09/C01–C07/T01–T03 classifications with rationale. +- Resolution: parameters/selectors/manual conversion only for declared unresolved targets. + +## Status Semantics + +ready, needs_context, needs_selector, needs_baseline, manual, unsupported, blocked are never reduced to color alone. Unsupported/manual cases remain visible in coverage. + +## Tests + +1. Fifteen-plus steps render without dependency information loss. +2. Cycle fixture is not rendered as executable graph; cycle path is shown. +3. Technical cases show Superset API/human, never SQL. +4. Parameter resolution updates affected steps only. +5. Preview-only pack explains all save blockers. + +#endregion DashboardScenarioModel.GraphUx diff --git a/specs/038-dashboard-scenario-model/data-model.md b/specs/038-dashboard-scenario-model/data-model.md new file mode 100644 index 000000000..ffbc59077 --- /dev/null +++ b/specs/038-dashboard-scenario-model/data-model.md @@ -0,0 +1,87 @@ +#region DashboardScenarioModel.DataModel [C:5] [TYPE ADR] [SEMANTICS data-model,scenario,graph,step,validation] +@BRIEF Canonical scenario graph, step, parameter, ref, capability, validation, and draft-pack models. +@RELATION DEPENDS_ON -> [DashboardScenarioModel.Research] + +## DashboardTestScenario + +Required: + +- schema_version, compiler_version, template_version; +- scenario_id and revision_hash; +- parent_revision_hash nullable; +- dashboard_context and objective; +- input_fingerprints: query model, checklist catalog, baseline summary, parameters; +- parameters, phases, steps; +- outputs and artifact_plan; +- checklist_coverage; +- warnings, blockers, risk_summary. + +Canonical serialization excludes timestamps and display-only volatile fields from revision_hash. + +## ScenarioParameter + +Fields: name, label, type, required, default, source, validation, value, status, affected_step_ids. Supported types: string, integer, decimal, boolean, date, datetime, enum, string_list, baseline_choice, selector_hint. + +## ScenarioStep + +| Field | Rule | +|---|---| +| id | Stable kebab id, unique | +| phase | setup/interact/observe/assert/evidence/report | +| title/description | Bounded display text | +| tool | browser, superset_api, xlsx, assertion, screenshot, report, artifact, human | +| action | Must be allowed for tool by registry | +| inputs | Typed refs only | +| outputs | Unique ScenarioRef declarations | +| expected | Structural expectation or baseline ref, never raw numeric truth | +| depends_on | Existing step ids; DAG | +| automation_status | ready, needs_context, needs_selector, needs_baseline, manual, unsupported, blocked | +| checklist_case_ids | Known catalog ids | +| risk | read, browser_interaction, draft_write, human | + +## ScenarioRef + +Namespaces: + +- context.dashboard.*, context.query_model.*; +- param.{name}; +- step.{step_id}.{output}; +- baseline.{baseline_id} or candidate.{candidate_id}; +- artifact.{artifact_key}. + +Every consumed step ref has exactly one producer. Parameter/context/baseline refs are externally resolved roots. + +## ChecklistCase and CapabilityMapping + +ChecklistCase holds id, section, goal, reusable expected semantics, required/optional capabilities, parameters, tool templates, evidence, fallback, and source page. Historic PDF outcome is stored only as source_note. + +CapabilityMapping holds case_id, classification, matched/missing capabilities, selected template, rationale, and resulting step ids. + +## ScenarioValidationResult + +Fields: valid, errors, warnings, blockers, coverage, topological_order, unresolved_parameters, unresolved_selectors, unresolved_baselines, graph_hash. Finding contains code, severity, message, json_pointer, step_id/case_id, and recovery options. + +## ArtifactPlan and DraftPack + +ArtifactPlan entries declare artifact_key, kind, relative_path_template, template_id, input_refs, required, and generation blockers. + +DraftPack contains scenario revision hash, template version, manifest, DraftArtifact refs from 036, validation summary, and warnings. It has status preview_only or save_eligible. Any validation error, NEEDS_SELECTOR, forbidden action, or unresolved required parameter makes it preview_only. + +## Deterministic Identity + +- scenario_id = dashboard key plus normalized objective slug; +- step id = phase rank + case id + action slug + ordinal; +- revision_hash = SHA-256 of canonical scenario inputs and output graph; +- output refs and artifacts are sorted by canonical graph order. + +## Forbidden Content + +The schema/validator rejects: + +- sql, query text, shell command, executable code body; +- raw numeric expected value for a metric assertion; +- absolute or parent-traversing artifact paths; +- unregistered tool/action; +- implicit dependency or duplicate producer. + +#endregion DashboardScenarioModel.DataModel diff --git a/specs/038-dashboard-scenario-model/plan.md b/specs/038-dashboard-scenario-model/plan.md new file mode 100644 index 000000000..ad94beb07 --- /dev/null +++ b/specs/038-dashboard-scenario-model/plan.md @@ -0,0 +1,71 @@ +# Implementation Plan: Dashboard Scenario Model + +**Branch**: 038-dashboard-scenario-model | **Date**: 2026-07-13 | **Spec**: [spec.md](./spec.md) + +## Summary + +Implement a backend Pydantic ScenarioGraph domain and deterministic compiler that maps the 19 normalized PDF checklist cases plus dashboard/baseline capabilities into a reviewable DAG. Validate all refs, cycles, tools, selectors, parameters, and baseline rules before compiling a draft pack through versioned safe templates and registering it with 036. + +## Technical Context + +**Language/Version**: Python 3.13+ +**Dependencies**: Pydantic 2, FastAPI, deterministic JSON/YAML serialization, existing 036/037 services +**Storage**: Immutable request/graph/draft hashes; draft bytes via 036; no new DB table required for MVP beyond AgentRun event/artifact links +**Testing**: pytest property/unit/contract, JSON Schema validation, golden snapshots +**Performance Goals**: validate 100-step graph under 100ms; resolve parameters under 200ms; byte-stable snapshots +**Constraints**: DAG only; no raw baseline numbers; no SQL; no executable LLM output; all checklist cases classified +**Scale**: 19 source cases, up to 100 steps, 50 parameters, 200 refs + +## Constitution Check + +| Principle | Result | +|---|---| +| Contracts/decision memory | PASS — compiler, validator, serializer and pack generator are contracted | +| Module discipline | PASS — models, catalog, mapping, compiler, validator, serializer, templates separated | +| RBAC/side effects | PASS — graph build is pure/read; draft registration and save use 036 | +| TDD | PASS — invalid fixture matrix and rejected direct-code/SQL paths first | +| Attention | PASS — ScenarioGraph.* IDs and shared scenario semantics | + +## Project Structure + +~~~text +backend/src/ +├── api/routes/dashboard_scenarios.py +├── schemas/dashboard_scenario.py +└── services/dashboard_testing/scenario/ + ├── models.py + ├── checklist_catalog.py + ├── capability_mapper.py + ├── compiler.py + ├── validator.py + ├── serializer.py + ├── pack_compiler.py + └── templates/ + +backend/tests/services/dashboard_testing/scenario/ +agent/src/ss_tools/agent/tools.py +~~~ + +## Delivery Phases + +1. Machine schema, Pydantic models, normalized checklist catalog and fixtures. +2. Capability mapping and deterministic graph compiler. +3. Full validator and canonical serializer. +4. Parameter/selector/manual resolution with immutable revisions. +5. Safe template-based draft-pack compiler and 036 registration. +6. REST/agent tools and regression gates. + +## API and Schema + +- contracts/dashboard-test-scenario.schema.json is the canonical graph interchange contract. +- contracts/scenario-api.openapi.yaml defines compile, validate, resolve, and draft-pack endpoints. + +## Cross-Spec Boundary + +- Reads 037 query models/baseline summaries; never calls Superset directly. +- Writes drafts and progress through 036. +- Supplies scenario, validation, coverage, parameter, and artifact manifest DTOs to 039. + +## Complexity Tracking + +No exception planned. Checklist data remains declarative and versioned; do not turn the 19 cases into one large conditional compiler function. diff --git a/specs/038-dashboard-scenario-model/quickstart.md b/specs/038-dashboard-scenario-model/quickstart.md new file mode 100644 index 000000000..6c724fb6d --- /dev/null +++ b/specs/038-dashboard-scenario-model/quickstart.md @@ -0,0 +1,33 @@ +# Quickstart: Dashboard Scenario Model + +## Prerequisites + +036 draft registration and 037 query-model/baseline summary contracts must pass. + +## Test Order + +~~~bash +cd backend +python -m pytest tests/services/dashboard_testing/scenario -v +python -m pytest tests/api/test_dashboard_scenarios.py -v +~~~ + +## Independent Validation + +1. Load the catalog and assert exactly 19 unique ids. +2. Map a full-capability dashboard; every case has one classification. +3. Map a no-XLSX/no-dataset-field dashboard; C04–C06 and T01–T03 retain rationale with no SQL. +4. Compile the same canonical inputs repeatedly and with shuffled input maps; bytes/hashes must match. +5. Validate invalid fixtures: cycle, missing ref, duplicate output, raw metric expected, stale baseline, unknown selector/tool, unresolved required parameter. +6. Resolve one parameter; unrelated step ids/order must remain unchanged and parent hash must link. +7. Compile a valid graph to draft pack through registered templates. +8. Attempt executable-code, shell, SQL, custom path, and unknown-template injection; all must block before draft registration. + +## Exit Gates + +- JSON Schema and Pydantic round-trip. +- 100% of 19 cases classified in fixtures; at least 80% meet spec classification criterion. +- Byte-stable JSON/YAML snapshots. +- No raw baseline numbers in executable assertions. +- No LLM executable body or SQL surface. +- Unit/API, ruff, schema, and semantic audits pass. diff --git a/specs/038-dashboard-scenario-model/research.md b/specs/038-dashboard-scenario-model/research.md new file mode 100644 index 000000000..cca5078cb --- /dev/null +++ b/specs/038-dashboard-scenario-model/research.md @@ -0,0 +1,87 @@ +#region DashboardScenarioModel.Research [C:5] [TYPE ADR] [SEMANTICS research,scenario,graph,checklist,determinism] +@BRIEF Phase 0 decisions for deterministic scenario compilation, validation, checklist mapping, and safe draft-pack generation. +@RELATION DEPENDS_ON -> [DashboardScenarioModel.Spec] +@RELATION DEPENDS_ON -> [SupersetBaselineEngine.DataModel] +@RATIONALE The agent may explain and propose intent, but a deterministic compiler/validator must own the graph and generated pack. +@REJECTED Direct LLM-to-code or LLM-owned graph identifiers — rejected because refs, safety, and repeatability cannot be guaranteed. + +## 1. Source Checklist Audit + +The source research/Чеклист 29.05 (1).pdf is 21 pages and contains 19 cases: + +- basic B01–B09; +- complex C01–C07; +- technical T01–T03. + +The PDF mixes reusable behavior, FI-0080-specific data, historic pass/fail notes, screenshots, and direct SQL instructions. Historic result text is evidence, not a reusable expected result. SQL instructions are not executable in this feature and are mapped to Superset API verification or human checkpoints. + +## 2. Deterministic Compiler Boundary + +- **Decision**: Agent produces a bounded ScenarioIntentDraft (objective, selected case ids, user rationale). Backend compiler combines it with DashboardQueryModel, checklist catalog, baseline summaries, capability model, and parameters. +- **Determinism**: Same canonical inputs and compiler/template versions produce byte-stable graph and draft manifest. +- **IDs**: Stable ids derive from phase rank, checklist case id, action, and collision-safe ordinal; never random UUIDs inside the canonical graph. +- **Ordering**: Topological order with stable phase/tool/action tie-breakers. +- **Alternative rejected**: temperature=0 as the only determinism mechanism — model/provider behavior is not a serialization contract. + +## 3. Capability Mapping + +Capability tags include native_filters, text_filter, table_filter, pagination, row_edit, bulk_edit, persistence_refresh, time_rollover, xlsx_export, cross_dashboard, superset_metric, dataset_field_read, screenshot, repository_write. + +Each ChecklistCase has: + +- required and optional capabilities; +- parameter requirements; +- candidate tool-chain templates; +- automation policy; +- expected evidence; +- fallback classification. + +Classification is one of automated, human_checkpoint, unsupported, needs_context. No case is silently dropped. + +## 4. Technical PDF Cases Without SQL + +T01–T03 refer to SQL Lab/database fields. Mapping rule: + +1. if an authoritative saved dataset/chart result exposes the fields, use superset_api; +2. otherwise create a human checkpoint describing the required evidence; +3. never emit SQL text, SQL tool, or an expected value invented from the PDF. + +This preserves coverage intent without violating 037. + +## 5. Graph Model + +- DAG phases: setup, interact, observe, assert, evidence, report. +- Every step declares tool, action, inputs, outputs, expected result, dependencies, automation status, checklist refs, and risk. +- Inputs reference parameters, context, baseline refs, or earlier output refs. +- Assertions require baseline_id/candidate_id or structural expectation; raw numeric truth is forbidden. +- Unknown selectors become NEEDS_SELECTOR; missing values become NEEDS_CONTEXT; stale/missing baseline becomes NEEDS_BASELINE. + +## 6. Validator + +Validation is pure and returns all findings: + +- schema/type/enum violations; +- cycles and dependency order; +- missing/duplicate refs; +- parameter type/resolution; +- tool/action compatibility; +- selector and baseline requirements; +- unsupported/dangerous actions; +- checklist coverage and unreachable steps; +- direct SQL/embedded baseline literals. + +Errors block draft-pack compilation. Warnings may allow preview but are repeated in 036 save gate. + +## 7. Safe Draft-Pack Compiler + +- **Decision**: Compile a valid graph through versioned repository-owned templates. +- **MVP outputs**: scenario.yaml, runner.plan.json, report_template.md, evidence_manifest.json, and optional template-based browser/XLSX assertion modules. +- **Invariant**: LLM text may populate bounded descriptions only; it cannot provide executable code bodies, paths, imports, or shell commands. +- **Drafts**: Registered via 036 outside target repository. +- **Alternative rejected**: Agent writes scripts and later validator scans them — unsafe constructs are already materialized and scanning is incomplete. + +## 8. Resolution Semantics + +Supplying parameters produces a new immutable scenario revision with parent_revision_hash. Unrelated steps retain ids and serialization. Manual conversion and selector hints are explicit resolution operations with audit reason. + +#endregion DashboardScenarioModel.Research diff --git a/specs/038-dashboard-scenario-model/spec.md b/specs/038-dashboard-scenario-model/spec.md index abce006cf..1d2fefbca 100644 --- a/specs/038-dashboard-scenario-model/spec.md +++ b/specs/038-dashboard-scenario-model/spec.md @@ -12,7 +12,7 @@ @SEMANTICS: spec, requirements, feature, scenario, graph, dashboard-testing, checklist, validation, artifacts **Feature Branch**: `038-dashboard-scenario-model` -**Created**: 2026-07-07 | **Status**: Draft +**Created**: 2026-07-07 | **Status**: Ready for Implementation **Input**: "Define the dashboard test scenario model used by agents to represent unique dashboard test flows as a validated ScenarioGraph. The model must express ordered and dependent steps across browser automation, Superset query execution, XLSX parsing, assertions, screenshots, reports, human checkpoints, baseline references, parameters, warnings, and missing context markers without exposing users to low-level tool selection." ## User Scenarios diff --git a/specs/038-dashboard-scenario-model/tasks.md b/specs/038-dashboard-scenario-model/tasks.md new file mode 100644 index 000000000..186c3d470 --- /dev/null +++ b/specs/038-dashboard-scenario-model/tasks.md @@ -0,0 +1,67 @@ +#region DashboardScenarioModel.Tasks [C:3] [TYPE ADR] [SEMANTICS tasks,scenario,implementation] +@BRIEF Ordered TDD backlog for deterministic scenario graph and safe draft-pack compilation. + +## Phase 1 — Catalog, Schemas, Fixtures + +- [ ] T001 Transcribe checklist-catalog.md into a versioned declarative resource in backend/src/services/dashboard_testing/scenario/catalog_v1.yaml. +- [ ] T002 Write catalog completeness tests for B01–B09, C01–C07, T01–T03 in backend/tests/services/dashboard_testing/scenario/test_catalog.py. +- [ ] T003 Create valid 18-step scenario plus invalid cycle/missing-ref/duplicate-output/raw-baseline/SQL fixtures under specs/038-dashboard-scenario-model/fixtures/. +- [ ] T004 Materialize fixtures to backend/tests/fixtures/dashboard_scenarios/. +- [ ] T005 Implement Pydantic models in backend/src/services/dashboard_testing/scenario/models.py matching dashboard-test-scenario.schema.json. + +## Phase 2 — US1 Compile Scenario Graph + +- [ ] T006 [US1] Write failing capability mapping tests in backend/tests/services/dashboard_testing/scenario/test_capability_mapper.py. +- [ ] T007 [US1] Implement backend/src/services/dashboard_testing/scenario/checklist_catalog.py validation and backend/src/services/dashboard_testing/scenario/capability_mapper.py with complete classification. +- [ ] T008 [US1] Cover unavailable XLSX, missing selector/test data, unsafe mutation context, cross-dashboard absence, and T01–T03 no-SQL fallbacks. +- [ ] T009 [US1] Write failing deterministic compiler tests in backend/tests/services/dashboard_testing/scenario/test_compiler.py. +- [ ] T010 [US1] Implement registered tool/action and step-template catalogs under backend/src/services/dashboard_testing/scenario/templates/. +- [ ] T011 [US1] Implement backend/src/services/dashboard_testing/scenario/compiler.py with stable ids, phase order, refs, coverage, and fingerprints. +- [ ] T012 [US1] Prove repeated compile and shuffled input order produce identical graph bytes. + +**Checkpoint**: Valid fixture compiles to stable graph and classifies all 19 cases. + +## Phase 3 — US2 Validate Safety and Completeness + +- [ ] T013 [US2] Write failing full invalid-fixture matrix in backend/tests/services/dashboard_testing/scenario/test_validator.py. +- [ ] T014 [US2] Implement schema, ref producer/consumer, duplicate, dependency, and cycle checks in backend/src/services/dashboard_testing/scenario/validator.py. +- [ ] T015 [US2] Implement parameter, selector, baseline, tool/action, path, SQL/code, raw-expected, and coverage checks. +- [ ] T016 [US2] Return deterministic all-findings output with JSON pointers and recovery options. +- [ ] T017 [US2] Add property tests generating small DAG/cycle/ref variations without mirroring validator logic in backend/tests/services/dashboard_testing/scenario/test_validator_properties.py. + +## Phase 4 — US3 Checklist Coverage and Serialization + +- [ ] T018 [US3] Write JSON/YAML golden tests in backend/tests/services/dashboard_testing/scenario/test_serializer.py. +- [ ] T019 [US3] Implement backend/src/services/dashboard_testing/scenario/serializer.py and revision hash exclusions. +- [ ] T020 [US3] Validate JSON Schema and Pydantic round-trip for all golden fixtures. +- [ ] T021 [US3] Add catalog-version and compiler-version fingerprints to scenario inputs. + +## Phase 5 — US4 Parameters and Human Checkpoints + +- [ ] T022 [US4] Write failing typed resolution/stale revision tests in backend/tests/services/dashboard_testing/scenario/test_resolver.py. +- [ ] T023 [US4] Implement backend/src/services/dashboard_testing/scenario/resolver.py for parameter, selector, manual conversion, and remove-step operations. +- [ ] T024 [US4] Enforce immutable revisions and unchanged unrelated step ids/order. +- [ ] T025 [US4] Cover safe-environment/test-data requirements for mutating PDF cases. + +## Phase 6 — Safe Draft Pack + +- [ ] T026 Write failing template registry, preview-only, path, SQL, shell, and injected-code tests in backend/tests/services/dashboard_testing/scenario/test_pack_compiler.py. +- [ ] T027 Create versioned templates for scenario.yaml, runner.plan.json, report_template.md, evidence_manifest.json, and bounded browser/XLSX modules under backend/src/services/dashboard_testing/scenario/pack_templates/v1/. +- [ ] T028 Implement backend/src/services/dashboard_testing/scenario/pack_compiler.py; accept only registered template ids and structured inputs. +- [ ] T029 Register outputs through 036 AgentRuns.Artifacts.Register and emit generate/validate progress. +- [ ] T030 Ensure invalid/unresolved graphs produce preview_only with repeated blockers. + +## Phase 7 — API, Agent, Quality + +- [ ] T031 Add backend/src/api/routes/dashboard_scenarios.py matching scenario-api.openapi.yaml and register router. +- [ ] T032 Write RBAC/contract/revision tests in backend/tests/api/test_dashboard_scenarios.py. +- [ ] T033 Add thin compile/validate/resolve/generate tools in agent/src/ss_tools/agent/tools.py. +- [ ] T034 Verify agent schemas cannot carry code, SQL, raw expected metrics, custom tools, or artifact paths. +- [ ] T035 Run quickstart, JSON/OpenAPI schema validation, scoped/full backend tests, and ruff. +- [ ] T036 Audit all 19 cases, direct SQL/code bans, contract anchors, ATTN_1–4, and unresolved relations. + +## Dependencies + +T001–T005 → US1 → US2; US3 can follow compiler, US4 follows validator, draft pack follows validation/resolution. 039 begins only after ScenarioResponse and DraftPack fixtures are stable. + +#endregion DashboardScenarioModel.Tasks diff --git a/specs/038-dashboard-scenario-model/traceability.md b/specs/038-dashboard-scenario-model/traceability.md new file mode 100644 index 000000000..52527bdb3 --- /dev/null +++ b/specs/038-dashboard-scenario-model/traceability.md @@ -0,0 +1,19 @@ +#region DashboardScenarioModel.Traceability [C:3] [TYPE ADR] [SEMANTICS traceability,scenario,requirements] +@BRIEF Requirement-to-contract-to-task-to-test matrix for feature 038. + +| Requirement | Contract | Tasks | Test | +|---|---|---|---| +| AGSCN-FR-001, AGSCN-FR-002 | ScenarioGraph.Compiler.Compile, schema | T005–T012 | golden scenario snapshots | +| AGSCN-FR-003 | tool/action registry and validator | T010–T016 | all tool categories/unknown tool | +| AGSCN-FR-004 | Validator baseline invariant | T014–T016 | raw expected and missing baseline | +| AGSCN-FR-005 | ScenarioGraph.Validator.Validate | T013–T017 | invalid fixture matrix | +| AGSCN-FR-006 | Catalog.Load/CapabilityMapper.Map | T001–T009 | 19-case classification | +| AGSCN-FR-007 | capability templates | T007–T012 | manual/unsupported mapping | +| AGSCN-FR-008 | Compiler/Serializer | T011, T018–T020 | repeated/shuffled byte snapshots | +| AGSCN-FR-009 | bounded intent/API schema | T021–T029 | rejected low-level/code inputs | + +## Downstream + +039 renders ScenarioResponse and DraftPack; it does not reimplement validation, coverage, or parameter resolution. + +#endregion DashboardScenarioModel.Traceability diff --git a/specs/038-dashboard-scenario-model/ux_reference.md b/specs/038-dashboard-scenario-model/ux_reference.md index 4d64326a3..9c9bca5be 100644 --- a/specs/038-dashboard-scenario-model/ux_reference.md +++ b/specs/038-dashboard-scenario-model/ux_reference.md @@ -2,7 +2,7 @@ @BRIEF UX reference for reviewing a generated DashboardTestScenario graph as a goal-oriented test flow. **Feature Branch**: `038-dashboard-scenario-model` -**Created**: 2026-07-07 | **Status**: Draft +**Created**: 2026-07-07 | **Status**: Ready for Implementation ## 1. User Persona & Context diff --git a/specs/039-dashboard-scenario-ui/checklists/requirements.md b/specs/039-dashboard-scenario-ui/checklists/requirements.md index 14c161369..d1570574f 100644 --- a/specs/039-dashboard-scenario-ui/checklists/requirements.md +++ b/specs/039-dashboard-scenario-ui/checklists/requirements.md @@ -23,3 +23,12 @@ - [x] CHK009 UI entities are defined for entry action, workspace state, preview, parameters, baselines, artifacts, and confirmations. - [x] CHK010 Success criteria are measurable through model, component, and browser tests. - [x] CHK011 UX copy explicitly excludes direct SQL as a validation path. + +## Implementation Package + +- [x] CHK012 Research and implementation plan resolve all design decisions. +- [x] CHK013 Data model, component, API-UX, screen, state, and design contracts are present. +- [x] CHK014 Quickstart defines model, component, and browser-driven verification. +- [x] CHK015 Traceability maps every functional requirement to contracts, tasks, and tests. +- [x] CHK016 Tasks use exact repository paths, dependency order, and test-first sequencing. +- [x] CHK017 Semantic anchors and cross-package ownership boundaries pass validation. diff --git a/specs/039-dashboard-scenario-ui/contracts/modules.md b/specs/039-dashboard-scenario-ui/contracts/modules.md new file mode 100644 index 000000000..3370deba4 --- /dev/null +++ b/specs/039-dashboard-scenario-ui/contracts/modules.md @@ -0,0 +1,114 @@ +#region DashboardScenarioUi.Modules [C:5] [TYPE ADR] [SEMANTICS contracts,dashboard-testing,scenario,ux] +@BRIEF Model, API client, entry action, workspace, parameter, baseline, artifact, and confirmation UI contracts. +@RELATION DEPENDS_ON -> [DashboardScenarioUi.DataModel] +@RELATION DEPENDS_ON -> [AgentRuns.Model] +@RELATION DEPENDS_ON -> [ScenarioGraph.Api] + +// #region DashboardTesting.ApiClient [C:3] [TYPE Module] [SEMANTICS dashboard-testing,api,types] +// @defgroup DashboardTesting Typed fetchApi wrappers for 036–038 scenario endpoints. +// @RELATION DEPENDS_ON -> [ApiModule] +// @DATA_CONTRACT OpenAPI DTOs -> TypeScript DTOs with no any at public boundaries. +// @INVARIANT 401/403/409/422/5xx retain typed code/status; no silent empty fallback. +// #endregion DashboardTesting.ApiClient + +// #region DashboardTesting.WorkspaceModel [C:5] [TYPE Model] [SEMANTICS dashboard-testing,scenario,workspace,model] +// @defgroup DashboardTesting Screen model for scenario review, resolution, baseline impact, and draft preview. +// @STATE unavailable | bootstrapping | analyzing | needs_parameters | scenario_ready | generating | draft_ready | waiting_approval | saved | failed | disconnected +// @ACTION initialize(context, agentRun) — enter scenario mode for valid v2 intent only. +// @ACTION applyScenarioResponse(response) — atomically replace revision/validation. +// @ACTION updateParameter(name, value) — local declared typed edit. +// @ACTION applyParameters() — resolve dirty parameters against base revision. +// @ACTION generateDraftPack() — request 038 safe template pack. +// @ACTION selectArtifact(id) / loadPreview(id) — side-effect-free preview. +// @ACTION requestSave(ids) / requestBaselineApproval(candidate) — delegate gate creation. +// @INVARIANT AgentRunModel is sole authority for run/stages/gate; workspace never parses chat prose. +// @INVARIANT canRequestSave is false for preview_only, invalid artifact, blockers, or unresolved required input. +// @RELATION DEPENDS_ON -> [DashboardTesting.ApiClient] +// @RELATION DEPENDS_ON -> [AgentRuns.Model] +// @RELATION BINDS_TO -> [DashboardTesting.Workspace] +// @RATIONALE One composed screen model makes cross-panel readiness testable without DOM. +// @REJECTED Independent component fetch/state — causes revision and gate races. +// #endregion DashboardTesting.WorkspaceModel + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +#endregion DashboardScenarioUi.Modules diff --git a/specs/039-dashboard-scenario-ui/contracts/ux/alternatives.md b/specs/039-dashboard-scenario-ui/contracts/ux/alternatives.md new file mode 100644 index 000000000..e8d183130 --- /dev/null +++ b/specs/039-dashboard-scenario-ui/contracts/ux/alternatives.md @@ -0,0 +1,15 @@ +#region DashboardScenarioUi.Alternatives [C:3] [TYPE ADR] [SEMANTICS ux,alternatives,dashboard-testing] +@BRIEF Evaluated UX alternatives and rejection rationale. + +| Decision | Selected | Rejected | +|---|---|---| +| Entry | Business action from dashboard | Tool/artifact dropdown: exposes implementation | +| Surface | Existing /agent route in scenario mode | New disconnected route: duplicates run/chat/recovery | +| State | Composed screen model | Component-local fetch/state: revision races | +| Graph | Visual DAG plus full step table | Canvas-only graph: inaccessible and hard to test | +| Parameters | Typed declared form and revision resolve | Free-form chat only: ambiguous and non-validated | +| Drafts | File tree plus safe preview/download | Write then review Git diff: mutation occurs too early | +| Confirmation | Reuse bound 036 gate/card | New modal/local confirmation: bypasses audit binding | +| Baseline | Explicit approved/stale/missing/candidate states | Boolean pass/fail: hides uncertainty | + +#endregion DashboardScenarioUi.Alternatives diff --git a/specs/039-dashboard-scenario-ui/contracts/ux/api-ux.md b/specs/039-dashboard-scenario-ui/contracts/ux/api-ux.md new file mode 100644 index 000000000..80ca3f137 --- /dev/null +++ b/specs/039-dashboard-scenario-ui/contracts/ux/api-ux.md @@ -0,0 +1,27 @@ +#region DashboardScenarioUi.ApiUx [C:3] [TYPE ADR] [SEMANTICS ux,api,dashboard-testing] +@BRIEF UI request/event sequence across 036, 037, and 038. + +## Main Sequence + +1. Dashboard action opens /agent with UIContext v2. +2. Gradio emits agent_run_started; run panel appears. +3. Agent invokes 037 query-model inspection and emits inspect progress. +4. Agent submits bounded intent to 038 compile; workspace receives ScenarioResponse. +5. User resolves parameters via 038 resolve with base revision hash. +6. User requests draft pack; 038 registers 036 drafts. +7. User previews/downloads through 036 URLs. +8. Save or baseline approval creates 036 gate; decision/consume is authoritative. + +## Error Routing + +| Code | UX | +|---|---| +| 401 | Session expired; privileged flow stops | +| 403 | Permission denied; no confirm | +| 404 | Dashboard/run/artifact unavailable with context | +| 409 | Recover run or reload scenario revision/gate | +| 422 | Focus fields/steps/findings | +| 504 | Superset timeout; retry/switch environment/manual | +| 5xx | Preserve run id/drafts and offer safe retry | + +#endregion DashboardScenarioUi.ApiUx diff --git a/specs/039-dashboard-scenario-ui/contracts/ux/dashboard-scenario-ux.md b/specs/039-dashboard-scenario-ui/contracts/ux/dashboard-scenario-ux.md new file mode 100644 index 000000000..0e0f479a1 --- /dev/null +++ b/specs/039-dashboard-scenario-ui/contracts/ux/dashboard-scenario-ux.md @@ -0,0 +1,33 @@ +#region DashboardScenarioUi.WorkspaceUx [C:4] [TYPE ADR] [SEMANTICS ux,dashboard-testing,workspace,scenario] +@BRIEF Detailed layout, state, feedback, recovery, and browser test contract for the complete scenario workspace. + +## Desktop Layout + +- Header: mode, dashboard/env context, run id/connection. +- Progress: seven named stages. +- Main two-column area: scenario/steps/coverage (wide) and parameters/baselines (narrow). +- Draft area: file tree left, preview right. +- Confirmation: existing blocking card bound to current gate. + +At widths below large breakpoint, panels stack in workflow order. Step table and file tree scroll within bounded containers; page retains one primary vertical scroll. + +## Feedback and Recovery + +- Context/permission failure occurs before analysis. +- Superset failure offers retry, environment switch, manual-only shell. +- Finding focuses related step/field. +- Stale baseline offers candidate discovery, never silent replace. +- Preview-only pack lists blockers at the save control. +- Disconnect shows stale timestamp and recover action without restarting. +- Deny preserves draft and explicitly states no repository/baseline change. + +## Browser Tests + +1. Three dashboards generate correct isolated context. +2. 18-step fixture at 1366×768 does not overlap/collapse actions. +3. Keyboard completes parameter form and confirmation. +4. Missing/stale baseline states and no-direct-SQL copy. +5. Invalid artifact blocks save; download keeps repository unchanged. +6. Reload recovers run/scenario/drafts. + +#endregion DashboardScenarioUi.WorkspaceUx diff --git a/specs/039-dashboard-scenario-ui/contracts/ux/decisions.md b/specs/039-dashboard-scenario-ui/contracts/ux/decisions.md new file mode 100644 index 000000000..bc3ed9597 --- /dev/null +++ b/specs/039-dashboard-scenario-ui/contracts/ux/decisions.md @@ -0,0 +1,15 @@ +#region DashboardScenarioUi.UxDecisions [C:3] [TYPE ADR] [SEMANTICS ux,decisions,dashboard-testing] +@BRIEF Final UI decisions for implementation handoff. + +1. Keep ordinary AI and add one clearly labelled scenario-generation action. +2. Scenario mode is determined only by valid UIContext v2 intent. +3. Reuse AgentRun progress/recovery and ConfirmationCard; no duplicate lifecycles. +4. Present business objective, phases, steps, coverage, then tool evidence. +5. Parameter changes create immutable revisions and do not rerun inspection. +6. Show all unsupported/manual/needs-context cases. +7. Preview/download remains side-effect free; save eligibility comes from DraftPack. +8. Baseline approval requires visible provenance/diff and reason. +9. “Superset API; no direct SQL” is persistent in relevant comparison/step views. +10. Accessible step table is complete even if graph visualization is unavailable. + +#endregion DashboardScenarioUi.UxDecisions diff --git a/specs/039-dashboard-scenario-ui/contracts/ux/design-tokens.md b/specs/039-dashboard-scenario-ui/contracts/ux/design-tokens.md new file mode 100644 index 000000000..90bb81cab --- /dev/null +++ b/specs/039-dashboard-scenario-ui/contracts/ux/design-tokens.md @@ -0,0 +1,18 @@ +#region DashboardScenarioUi.DesignTokens [C:2] [TYPE ADR] [SEMANTICS ux,design-tokens,dashboard-testing] +@BRIEF Semantic tokens and shared components for scenario workspace. + +| Element/state | Contract | +|---|---| +| Workspace/cards | bg-surface-page/card, border-border, text-text | +| Secondary/collapsed | bg-surface-muted, text-text-muted/subtle | +| Active/info | bg-primary-light, text-primary | +| Ready/pass | bg-success-light, text-success | +| Warning/manual/stale | bg-warning-light, text-warning | +| Error/blocker/invalid | bg-destructive-light, border-destructive-ring, text-destructive | +| Buttons/forms | Button, Input, Select and shared form primitives from $lib/ui | +| Loading | Skeleton/Spinner from $lib/ui | +| Status | Icon + text + semantic token; never color only | + +No raw blue/green/red/gray/indigo Tailwind families in new page/domain components. Focus and disabled states come from shared primitives. + +#endregion DashboardScenarioUi.DesignTokens diff --git a/specs/039-dashboard-scenario-ui/contracts/ux/screen-models.md b/specs/039-dashboard-scenario-ui/contracts/ux/screen-models.md new file mode 100644 index 000000000..5fc9fef55 --- /dev/null +++ b/specs/039-dashboard-scenario-ui/contracts/ux/screen-models.md @@ -0,0 +1,29 @@ +#region DashboardScenarioUi.ScreenModels [C:4] [TYPE ADR] [SEMANTICS ux,screen-models,dashboard-testing] +@BRIEF Model composition and component bindings for the dashboard scenario workspace. + +## Composition + +~~~text +AgentChat.Model +└── AgentRuns.Model transport/recovery/gates/drafts + └── DashboardTesting.WorkspaceModel + ├── Scenario views 038 graph/validation/coverage + ├── Parameters 038 resolution + ├── Baseline impact 037 result/candidate + └── Artifact preview 038 manifest + 036 refs +~~~ + +AgentRuns.Model and WorkspaceModel are distinct ownership layers, not independent stores. The page creates/composes them once for the route visit. + +## Model-First Test Layers + +- L1 AgentRuns.Model: sequence, recovery, gate and drafts. +- L1 WorkspaceModel: FSM, typed parameter drafts, readiness, immutable revision apply, preview lifecycle. +- L2 components: DOM, accessibility, layout, feedback/recovery. +- E2E: dashboard action through deny/confirm. + +## Decomposition Gate + +At 400 lines or 40 public methods, split baseline/artifact selection into submodels while preserving WorkspaceModel facade and invariants. + +#endregion DashboardScenarioUi.ScreenModels diff --git a/specs/039-dashboard-scenario-ui/data-model.md b/specs/039-dashboard-scenario-ui/data-model.md new file mode 100644 index 000000000..9ccc75de8 --- /dev/null +++ b/specs/039-dashboard-scenario-ui/data-model.md @@ -0,0 +1,56 @@ +#region DashboardScenarioUi.DataModel [C:4] [TYPE ADR] [SEMANTICS data-model,ux,scenario,workspace] +@BRIEF Frontend workspace projection, parameter draft, selection, and derived-action contracts. +@RELATION DEPENDS_ON -> [DashboardScenarioModel.DataModel] +@RELATION DEPENDS_ON -> [AgentTestStabilization.DataModel] + +## ScenarioWorkspaceState + +| Atom | Type | Owner/source | +|---|---|---| +| mode | scenario or ordinary chat | UIContext v2 intent | +| scenario | DashboardTestScenario/null | 038 response | +| validation | ScenarioValidationResult/null | 038 response | +| draftPack | DraftPack/null | 038 response | +| selectedPhaseId/selectedStepId | string/null | local navigation | +| parameterDrafts | map name → typed draft | initialized from scenario parameters | +| parameterErrors | map name → message code | local validation | +| baselineSummary | approved/stale/missing/candidate groups | 037 response | +| selectedArtifactId | UUID/null | local preview | +| preview | loading/content/error | 036 opaque preview | +| domainError | typed code/detail/recovery | API result | +| agentRun | AgentRuns.Model reference | 036 authority | + +## Derived Values + +- workspaceState from intent, AgentRun status/stage, scenario, draft pack, and domain error; +- canApplyParameters when changed drafts are valid; +- canGenerateDraft when scenario exists and required parameters/selectors are resolved; +- canRequestSave when draftPack.status=save_eligible, artifacts valid, and no blockers; +- baselineApprovalReady when candidate selected and non-blank reason; +- progress stages from AgentRunModel only. + +## ParameterDraft + +Fields: name, type, value, initial_value, required, validation, dirty, affected_step_ids. Decimal remains a string until API validation. Dates use YYYY-MM-DD input representation. + +Apply payload contains base_revision_hash and only dirty declared changes. Successful response atomically replaces scenario/validation and reinitializes drafts. + +## BaselineImpactItem + +Fields: result identity, status approved/stale/missing/candidate, baseline/candidate id, filters summary/hash, actual/expected/diff, stale dimensions, provenance, approval eligibility, warnings. + +## ArtifactPreviewState + +File tree is DraftPack manifest joined with 036 DraftArtifact refs by artifact key/id. Content preview is bounded text or safe structured rendering; binary files show metadata/download only. Blob/object URLs are released on selection change and destroy. + +## UI Invariants + +1. Domain status never derives from assistant text. +2. Graph/validation are server-authoritative immutable revisions. +3. Parameter edits cannot modify undeclared fields. +4. Save and approve controls delegate to 036 pending gate. +5. Download/preview never set persisted state. +6. Direct SQL is absent from actions and explanatory alternatives. +7. Switching run/context resets all scenario-local selections and drafts. + +#endregion DashboardScenarioUi.DataModel diff --git a/specs/039-dashboard-scenario-ui/plan.md b/specs/039-dashboard-scenario-ui/plan.md new file mode 100644 index 000000000..3b8653646 --- /dev/null +++ b/specs/039-dashboard-scenario-ui/plan.md @@ -0,0 +1,69 @@ +# Implementation Plan: Dashboard Scenario UI + +**Branch**: 039-dashboard-scenario-ui | **Date**: 2026-07-13 | **Spec**: [spec.md](./spec.md) + +## Summary + +Add a dashboard-level scenario action and a model-first workspace inside /agent. The UI composes 036 run/recovery/gate state, 037 baseline results, and 038 scenario/draft DTOs. Users review a business flow, resolve typed parameters, inspect coverage and draft artifacts, then explicitly save or approve. + +## Technical Context + +**Language/Version**: TypeScript, Svelte 5.56 runes, SvelteKit, Tailwind semantic tokens +**Dependencies**: existing fetchApi, AgentChat model family, $lib/ui, i18n, Vitest 4, Testing Library, Playwright +**Storage**: no frontend persistence of authoritative domain data; run id may be route/session recovery hint only +**Testing**: L1 model no-render, L2 component UX, Playwright browser flow +**Performance Goals**: parameter readiness projection under 200ms; 15–100 steps usable at 1366px; no duplicate stream re-render +**Constraints**: no Svelte 4 syntax, no component-owned business state, no prose parsing, no raw colors, no direct SQL copy/action +**Scale**: 100 steps, 50 params, 50 drafts, 19 coverage rows + +## Constitution Check + +| Principle | Result | +|---|---| +| Semantic contracts | PASS — model/actions and C3+ components contracted | +| Decision memory | PASS — composed model, entry action, accessible graph/table, no tool picker | +| Module discipline | PASS — model plus small domain components; decomposition threshold enforced | +| RBAC | PASS — entry/start/save/approve responses render distinct permission state | +| Svelte 5 | PASS — .svelte.ts model, $state/$derived/$effect only where appropriate | +| TDD | PASS — L1 model before L2 components before E2E | +| Attention | PASS — DashboardTesting.* hierarchy and shared dashboard-testing semantics | + +## Project Structure + +~~~text +frontend/src/ +├── lib/api/dashboard-testing.ts +├── lib/models/DashboardScenarioWorkspaceModel.svelte.ts +├── lib/models/__tests__/DashboardScenarioWorkspaceModel.test.ts +├── lib/components/dashboard-testing/ +│ ├── ScenarioWorkspace.svelte +│ ├── ScenarioProgressStrip.svelte +│ ├── ScenarioSummary.svelte +│ ├── ScenarioGraph.svelte +│ ├── ScenarioStepTable.svelte +│ ├── ScenarioCoverage.svelte +│ ├── ParameterPanel.svelte +│ ├── BaselineImpactPanel.svelte +│ ├── ArtifactPreviewPanel.svelte +│ └── __tests__/ +├── routes/dashboards/[id]/components/DashboardHeader.svelte +├── routes/agent/+page.svelte +└── lib/i18n/locales/{ru,en}/dashboard-testing.json +~~~ + +## Delivery Phases + +1. Types/API client and model fixtures. +2. Dashboard entry/context/RBAC path. +3. Workspace model and progress/scenario/coverage views. +4. Parameter and baseline resolution. +5. Draft preview and reused 036 confirmation flow. +6. Responsive/a11y/E2E/regression gates. + +## No New Backend Domain Logic + +039 may require endpoint wiring fixes but must not implement a second validator, comparison engine, artifact generator, approval store, or run lifecycle. Those remain 036–038. + +## Complexity Tracking + +ScenarioWorkspace is decomposed before 400 lines or 40 public model methods. The visual graph must not absorb the accessible step table or parameter logic. diff --git a/specs/039-dashboard-scenario-ui/quickstart.md b/specs/039-dashboard-scenario-ui/quickstart.md new file mode 100644 index 000000000..27898d541 --- /dev/null +++ b/specs/039-dashboard-scenario-ui/quickstart.md @@ -0,0 +1,41 @@ +# Quickstart: Dashboard Scenario UI + +## Prerequisites + +Stable fixtures and passing contracts from 036, 037, and 038. Backend, agent, and frontend running; one execution user and one approver. + +## Test Order + +~~~bash +cd frontend +npm run test -- --run src/lib/models/__tests__/DashboardScenarioWorkspaceModel.test.ts +npm run test -- --run src/lib/components/dashboard-testing +npx playwright test e2e/tests/dashboard-scenario-ui.e2e.js +npm run lint +npm run build +~~~ + +## Manual End-to-End + +1. Open dashboard 42 with environment selected. +2. Click “Создать сценарий тестирования”; verify exact dashboard/env/intent in /agent. +3. Confirm run id appears before analysis tool activity. +4. Review 18-step scenario: phases, step table, graph, all 19 coverage rows, warnings/blockers. +5. Verify relevant steps state Superset API/no direct SQL. +6. Resolve date, counterparty, and baseline choice; only affected steps update. +7. Generate draft; preview each text artifact and download without Git changes. +8. Verify preview-only/invalid draft blocks save. +9. Request repository save, inspect exact paths/warnings, deny, and verify no write. +10. Request baseline approval; blank reason must fail locally/server-side. +11. Confirm with reason; verify approved provenance and consumed gate. +12. Reload mid-run/draft and recover the same state. +13. Repeat with unauthorized user; no run or confirm control depending on denied operation. + +## Exit Gates + +- Correct context for three dashboards and no stale reuse. +- 18-step fixture usable at 1366px and keyboard accessible. +- Parameter readiness projection under 200ms in L1 tests. +- 100% save/approve actions use 036 gate. +- No direct-SQL action/copy. +- Frontend tests/lint/build and Playwright smoke pass. diff --git a/specs/039-dashboard-scenario-ui/research.md b/specs/039-dashboard-scenario-ui/research.md new file mode 100644 index 000000000..16d09296e --- /dev/null +++ b/specs/039-dashboard-scenario-ui/research.md @@ -0,0 +1,68 @@ +#region DashboardScenarioUi.Research [C:4] [TYPE ADR] [SEMANTICS research,ux,scenario,workspace,svelte] +@BRIEF Phase 0 decisions for dashboard entry, composed scenario workspace model, parameters, baselines, drafts, and approvals. +@RELATION DEPENDS_ON -> [DashboardScenarioUi.Spec] +@RELATION DEPENDS_ON -> [AgentRuns.Model] +@RELATION DEPENDS_ON -> [ScenarioGraph.Api] +@RATIONALE 039 is a typed consumer of 036–038 contracts; duplicating lifecycle or validation in components would create conflicting truth. +@REJECTED A wizard that asks users to choose Playwright, SQL, or XLSX — rejected because the compiler chooses a cross-tool business scenario. + +## 1. Existing Frontend Fact Check + +- Dashboard detail is model-first: DashboardDetailModel.svelte.ts plus thin +page.svelte. +- DashboardHeader.svelte already has an ordinary AI link carrying dashboard UIContext v1. +- /agent uses AgentChatModel.svelte.ts and Gradio stream processing. +- Shared UI components and semantic tokens are mandatory; current dashboard page has some legacy raw buttons that this feature must not copy into new components. +- Existing Gradio submit already places serialized UIContext last. + +## 2. Entry Action + +- **Decision**: Add a distinct Button/link labelled “Создать сценарий тестирования” in DashboardHeader. +- It routes to /agent with objectType=dashboard, id/name/env/route, contextVersion=2, and intent=build_dashboard_test_scenario. +- Existing ordinary AI chat remains available as a separate general-purpose action. +- Missing environment opens/focuses environment selection and does not start a run. +- RBAC denial is visible before or at run creation; no speculative workspace. + +## 3. Workspace Composition + +- **Decision**: DashboardScenarioWorkspaceModel.svelte.ts composes/references AgentRunModel and typed 037/038 API client. +- AgentRunModel owns transport/run recovery, stages, drafts, and gates. +- WorkspaceModel owns scenario response, selected step/phase, parameter drafts, baseline summaries, preview selection, and domain-derived readiness. +- Components read one workspace model; none parses chat prose or recomputes graph validity. + +## 4. State Machine + +States: unavailable, bootstrapping, analyzing, needs_parameters, scenario_ready, generating, draft_ready, waiting_approval, saved, failed, disconnected. + +Transitions are driven by structured AgentRun events and authoritative ScenarioResponse/DraftPack. A sequence gap delegates recovery to AgentRunModel. + +## 5. Parameters + +- Local type/required validation gives immediate feedback. +- Apply sends only declared resolution changes with base revision hash. +- Unaffected step ids/order remain; no full analysis restart. +- Reset returns to server-provided defaults/source, not empty values blindly. +- Baseline choice is a typed parameter linked to 037 statuses. + +## 6. Scenario Presentation + +- Summary and phase/step table are primary. +- A visual dependency graph is supplementary; the table provides complete accessible semantics. +- Tool categories are visible for trust but not user-selected. +- All 19 checklist cases remain visible in coverage, including manual/unsupported/needs-context. +- Superset metric steps state “Superset API; no direct SQL”. + +## 7. Draft and Approval + +- File tree shows intended relative paths, validation, warnings, and unresolved markers. +- Preview/download use 036 opaque URLs and do not mutate repository. +- Save is disabled for preview_only or invalid drafts. +- Repository save and baseline approval reuse the 036 confirmation card, including exact paths/hash/warnings and mandatory baseline reason. + +## 8. Responsive and Accessibility + +- Target: no collapse at 1366px for 15+ steps; below large breakpoint use stacked panels. +- Keyboard: entry, phase/step navigation, parameter form, file tree, preview, confirmation. +- Focus moves to first invalid parameter/finding and returns after modal/card action. +- Status uses text/icon and ARIA, never color alone. + +#endregion DashboardScenarioUi.Research diff --git a/specs/039-dashboard-scenario-ui/spec.md b/specs/039-dashboard-scenario-ui/spec.md index ae3b721fb..8fc6b6f4d 100644 --- a/specs/039-dashboard-scenario-ui/spec.md +++ b/specs/039-dashboard-scenario-ui/spec.md @@ -13,7 +13,7 @@ @SEMANTICS: spec, requirements, feature, ux, agent, scenario, dashboard-testing, artifacts, baseline **Feature Branch**: `039-dashboard-scenario-ui` -**Created**: 2026-07-07 | **Status**: Draft +**Created**: 2026-07-07 | **Status**: Ready for Implementation **Input**: "Provide the user-facing dashboard test scenario generation experience. From a dashboard page the user starts Create test scenario, the agent analyzes the dashboard, proposes a unique scenario graph, collects missing business parameters, previews generated artifacts and baseline impacts, and requires HITL confirmation before saving files or approving baselines." ## User Scenarios diff --git a/specs/039-dashboard-scenario-ui/tasks.md b/specs/039-dashboard-scenario-ui/tasks.md new file mode 100644 index 000000000..933a8a666 --- /dev/null +++ b/specs/039-dashboard-scenario-ui/tasks.md @@ -0,0 +1,74 @@ +#region DashboardScenarioUi.Tasks [C:3] [TYPE ADR] [SEMANTICS tasks,dashboard-testing,frontend] +@BRIEF Model-first TDD backlog for dashboard scenario generation UI. + +## Phase 1 — Types, Fixtures, API Client + +- [ ] T001 Materialize stable 036 AgentRun, 037 baseline, and 038 ScenarioResponse/DraftPack fixtures into frontend/src/lib/models/__fixtures__/dashboard-testing/. +- [ ] T002 Generate or hand-author strict TypeScript DTOs in frontend/src/lib/types/dashboard-testing.ts from upstream OpenAPI/JSON schemas. +- [ ] T003 Write failing API error mapping tests for 401/403/404/409/422/504/5xx in frontend/src/lib/api/__tests__/dashboard-testing.test.ts. +- [ ] T004 Implement frontend/src/lib/api/dashboard-testing.ts using fetchApi and typed errors. +- [ ] T005 Add frontend/src/lib/i18n/locales/ru/dashboard-testing.json and frontend/src/lib/i18n/locales/en/dashboard-testing.json; register them in frontend/src/lib/i18n/index.svelte.ts and prohibit state derivation from localized strings. + +## Phase 2 — US1 Dashboard Entry + +- [ ] T006 [US1] Write failing DashboardHeader entry tests in frontend/src/routes/dashboards/[id]/components/__tests__/DashboardHeader.ux.test.ts for three ids, env missing, permission denied, and ordinary AI link preservation. +- [ ] T007 [US1] Extend frontend/src/lib/models/DashboardDetailModel.svelte.ts with scenarioHref/context v2 derivation. +- [ ] T008 [US1] Add $lib/ui Button/link “Создать сценарий тестирования” to frontend/src/routes/dashboards/[id]/components/DashboardHeader.svelte. +- [ ] T009 [US1] Extend AgentChatTypes.UIContext and AgentChatModel.setUIContextFromParams for contextVersion=2 and intent. +- [ ] T010 [US1] Initialize scenario mode in routes/agent/+page.svelte only for valid v2 intent. +- [ ] T011 [US1] Add missing-env and permission-denied recovery without starting AgentRun. + +**Checkpoint**: One click opens correct scenario mode; three dashboard contexts never reuse stale id/env. + +## Phase 3 — US2 Progress and Scenario Review + +- [ ] T012 [US2] Write failing L1 FSM/sequence/recovery fixture tests in frontend/src/lib/models/__tests__/DashboardScenarioWorkspaceModel.test.ts. +- [ ] T013 [US2] Implement frontend/src/lib/models/DashboardScenarioWorkspaceModel.svelte.ts composed with AgentRunModel. +- [ ] T014 [US2] Write failing L2 ScenarioProgressStrip states/a11y tests in frontend/src/lib/components/agent/dashboard-testing/__tests__/ScenarioProgressStrip.ux.test.ts. +- [ ] T015 [US2] Implement frontend/src/lib/components/agent/dashboard-testing/ScenarioWorkspace.svelte and ScenarioProgressStrip.svelte. +- [ ] T016 [US2] Handle disconnected/gap recovery via AgentRunModel only. +- [ ] T017 [US2] Verify ordinary chat renders no scenario workspace. +- [ ] T018 [US2] Write failing 18-step summary/graph/table/coverage tests in frontend/src/lib/components/agent/dashboard-testing/__tests__/ScenarioViews.ux.test.ts. +- [ ] T019 [US2] Implement ScenarioSummary.svelte, ScenarioGraph.svelte, ScenarioStepTable.svelte, and ScenarioCoverage.svelte under frontend/src/lib/components/agent/dashboard-testing/. +- [ ] T020 [US2] Display tool categories read-only and explicit Superset API/no-direct-SQL statement. +- [ ] T021 [US2] Add complete accessible linear table fallback and finding focus links. +- [ ] T022 [US2] Cover all 19 automated/manual/unsupported/needs-context rows. + +## Phase 4 — US3 Parameters and Baselines + +- [ ] T023 [US3] Write failing no-render typed parameter/readiness/revision tests in frontend/src/lib/models/__tests__/DashboardScenarioWorkspaceModel.parameters.test.ts, including under-200ms assertion. +- [ ] T024 [US3] Implement WorkspaceModel parameter drafts, local validation, dirty-only resolve payload, and atomic revision apply. +- [ ] T025 [US3] Write failing ParameterPanel L2 tests in frontend/src/lib/components/agent/dashboard-testing/__tests__/ParameterPanel.ux.test.ts for date/decimal/enum/baseline choice/error focus/reset/conflict. +- [ ] T026 [US3] Implement frontend/src/lib/components/agent/dashboard-testing/ParameterPanel.svelte using $lib/ui form controls. +- [ ] T027 [US3] Verify resolution changes only affected steps and never restarts inspect stage. +- [ ] T028 [US3] Write failing approved/stale/missing/candidate/inconclusive tests in frontend/src/lib/components/agent/dashboard-testing/__tests__/BaselineImpactPanel.ux.test.ts. +- [ ] T029 [US3] Implement frontend/src/lib/components/agent/dashboard-testing/BaselineImpactPanel.svelte with provenance, diff, policy, stale dimensions, and no-direct-SQL copy. +- [ ] T030 [US3] Wire discovery candidate action without direct approval or catalog mutation. + +## Phase 5 — US4 Draft Preview + +- [ ] T031 [US4] Write failing tests in frontend/src/lib/components/agent/dashboard-testing/__tests__/ArtifactPreviewPanel.ux.test.ts for tree, text/binary preview, warning, invalid, preview-only, save eligibility, and URL cleanup. +- [ ] T032 [US4] Implement WorkspaceModel draft generation/selection/preview lifecycle. +- [ ] T033 [US4] Implement frontend/src/lib/components/agent/dashboard-testing/ArtifactPreviewPanel.svelte with bounded preview and opaque download URL. +- [ ] T034 [US4] Prove preview/download does not mark persisted or change repository state. + +## Phase 6 — US5 Save and Approval + +- [ ] T035 [US5] Write failing integration tests in frontend/src/lib/components/assistant/__tests__/dashboard_scenario_confirmation.integration.test.ts binding save/baseline candidate to existing 036 ConfirmationCard. +- [ ] T036 [US5] Extend frontend/src/lib/components/assistant/ConfirmationCard.svelte for exact paths, warnings, value/filter fingerprint, and required reason without a second modal. +- [ ] T037 [US5] Cover deny, blank reason, permission denial, payload conflict, consumed success, and preserved drafts. + +## Phase 7 — Responsive, A11y, E2E, Quality + +- [ ] T038 Add 1366×768 and narrow viewport component/browser assertions with 18-step fixture. +- [ ] T039 Add keyboard/focus/ARIA tests for entry, step navigation, parameters, file tree, and confirmation. +- [ ] T040 Add frontend/e2e/tests/dashboard-scenario-ui.e2e.js for entry → scenario → resolve → draft → deny/approve → reload. +- [ ] T041 Scan new UI/locales for SQL-as-option language and raw Tailwind color families. +- [ ] T042 Run all 033/035/036 agent model/components regressions plus 039 L1/L2 suites. +- [ ] T043 Run quickstart, frontend test/lint/build, Playwright, and semantic ATTN/anchor audits. + +## Dependencies + +T001–T005 → US1 → run/progress → scenario views → parameters/baselines → drafts → confirmation → E2E. Tests precede each implementation group. + +#endregion DashboardScenarioUi.Tasks diff --git a/specs/039-dashboard-scenario-ui/traceability.md b/specs/039-dashboard-scenario-ui/traceability.md new file mode 100644 index 000000000..49ccfa1ee --- /dev/null +++ b/specs/039-dashboard-scenario-ui/traceability.md @@ -0,0 +1,23 @@ +#region DashboardScenarioUi.Traceability [C:3] [TYPE ADR] [SEMANTICS traceability,dashboard-testing,ux] +@BRIEF Requirement-to-model/component/task/test matrix for feature 039. + +| Requirement | Model/component | Tasks | Test | +|---|---|---|---| +| AGUI-FR-001, AGUI-FR-002 | EntryAction, WorkspaceModel.initialize | T006–T011 | three-context entry/RBAC | +| AGUI-FR-003 | AgentRuns.Model, Progress | T012–T017 | structured stage/gap recovery | +| AGUI-FR-004 | ScenarioViews | T018–T022 | 18-step/coverage/layout | +| AGUI-FR-005 | Parameters, WorkspaceModel | T023–T027 | typed resolution under 200ms | +| AGUI-FR-006 | BaselineImpact | T028–T030 | approved/stale/missing/candidate | +| AGUI-FR-007 | ArtifactPreview | T031–T034 | tree/preview/invalid/download | +| AGUI-FR-008 | ConfirmationBinding | T035–T037 | deny/reason/RBAC/gate | +| AGUI-FR-009 | ScenarioViews/BaselineImpact | T020, T029, T041 | no-direct-SQL copy scan | +| AGUI-FR-010 | all components | T038–T043 | L1/L2/a11y/Playwright | + +## Contract Sources + +- Run/progress/draft/gate: 036. +- Query/baseline status: 037. +- Scenario/validation/resolution/draft pack: 038. +- 039 contains presentation and client orchestration only. + +#endregion DashboardScenarioUi.Traceability diff --git a/specs/039-dashboard-scenario-ui/ux_reference.md b/specs/039-dashboard-scenario-ui/ux_reference.md index cc4a33142..d8e538494 100644 --- a/specs/039-dashboard-scenario-ui/ux_reference.md +++ b/specs/039-dashboard-scenario-ui/ux_reference.md @@ -5,7 +5,7 @@ @REJECTED Primary dropdown options for Playwright, SQL, and XLSX were rejected because they expose implementation details and conflict with unique cross-tool scenario generation. **Feature Branch**: `039-dashboard-scenario-ui` -**Created**: 2026-07-07 | **Status**: Draft +**Created**: 2026-07-07 | **Status**: Ready for Implementation ## 1. User Persona & Context