From b51a68168cdae733ae1ac1d0b6e878c67d4bb857 Mon Sep 17 00:00:00 2001 From: busya Date: Tue, 17 Mar 2026 14:29:33 +0300 Subject: [PATCH] feat(027): Final Phase T038-T043 implementation - T038: SessionEvent logger and persistence logic - Added SessionEventLogger service with explicit audit event persistence - Added SessionEvent model with events relationship on DatasetReviewSession - Integrated event logging into orchestrator flows and API mutation endpoints - T039: Semantic source version propagation - Added source_version column to SemanticFieldEntry - Added propagate_source_version_update() to SemanticResolver - Preserves locked/manual field invariants during propagation - T040: Batch approval API and UI actions - Added batch semantic approval endpoint (/fields/semantic/approve-batch) - Added batch mapping approval endpoint (/mappings/approve-batch) - Added batch approval actions to SemanticLayerReview and ExecutionMappingReview components - Aligned batch semantics with single-item approval contracts - T041: Superset compatibility matrix tests - Added test_superset_matrix.py with preview and SQL Lab fallback coverage - Tests verify client method preference and matrix fallback behavior - T042: RBAC audit sweep on session-mutation endpoints - Added _require_owner_mutation_scope() helper - Applied owner guards to update_session, delete_session, and all mutation endpoints - Ensured no bypass of existing permission checks - T043: i18n coverage for dataset-review UI - Added workspace state labels (empty/importing/review) to en.json and ru.json - Added batch action labels for semantics and mappings - Fixed workspace state comparison to lowercase strings - Removed hardcoded workspace state display strings Signed-off-by: Implementation Specialist --- .gitignore | 25 +- .kilocode/mcp.json | 2 +- .../__tests__/test_clean_release_v2_api.py | 5 +- .../test_clean_release_v2_release_api.py | 6 +- .../__tests__/test_connections_routes.py | 6 +- .../api/routes/__tests__/test_dashboards.py | 46 +- .../__tests__/test_dataset_review_api.py | 485 ++++++- .../src/api/routes/__tests__/test_datasets.py | 17 +- backend/src/api/routes/dataset_review.py | 1246 ++++++++++++++++- .../utils/superset_compilation_adapter.py | 406 ++++++ .../core/utils/superset_context_extractor.py | 363 ++++- backend/src/models/dataset_review.py | 21 + backend/src/schemas/dataset_review.py | 2 + .../dataset_review/clarification_engine.py | 552 ++++++++ .../services/dataset_review/event_logger.py | 156 +++ .../services/dataset_review/orchestrator.py | 468 +++++++ .../repositories/session_repository.py | 162 ++- .../dataset_review/semantic_resolver.py | 58 + .../dataset_review/test_superset_matrix.py | 126 ++ .../dataset-review/ClarificationDialog.svelte | 472 +++++++ .../dataset-review/CompiledSQLPreview.svelte | 247 ++++ .../ExecutionMappingReview.svelte | 578 ++++++++ .../LaunchConfirmationPanel.svelte | 372 +++++ .../dataset-review/SemanticLayerReview.svelte | 598 ++++++++ .../us2_semantic_clarification.ux.test.js | 412 ++++++ .../__tests__/us3_execution_batch.ux.test.js | 499 +++++++ frontend/src/lib/i18n/locales/en.json | 246 ++++ frontend/src/lib/i18n/locales/ru.json | 246 ++++ .../routes/datasets/review/[id]/+page.svelte | 267 +++- .../dataset_review_workspace.ux.test.js | 235 +++- .../contracts/modules.md | 8 +- specs/027-dataset-llm-orchestration/tasks.md | 26 +- 32 files changed, 8210 insertions(+), 148 deletions(-) create mode 100644 backend/src/core/utils/superset_compilation_adapter.py create mode 100644 backend/src/services/dataset_review/clarification_engine.py create mode 100644 backend/src/services/dataset_review/event_logger.py create mode 100644 backend/tests/services/dataset_review/test_superset_matrix.py create mode 100644 frontend/src/lib/components/dataset-review/ClarificationDialog.svelte create mode 100644 frontend/src/lib/components/dataset-review/CompiledSQLPreview.svelte create mode 100644 frontend/src/lib/components/dataset-review/ExecutionMappingReview.svelte create mode 100644 frontend/src/lib/components/dataset-review/LaunchConfirmationPanel.svelte create mode 100644 frontend/src/lib/components/dataset-review/SemanticLayerReview.svelte create mode 100644 frontend/src/lib/components/dataset-review/__tests__/us2_semantic_clarification.ux.test.js create mode 100644 frontend/src/lib/components/dataset-review/__tests__/us3_execution_batch.ux.test.js diff --git a/.gitignore b/.gitignore index bd9e47ad..4ed5aa37 100755 --- a/.gitignore +++ b/.gitignore @@ -59,21 +59,22 @@ keyring passwords.py *github* *tech_spec* -/dashboards -dashboards_example/**/dashboards/ -backend/mappings.db +/dashboards +dashboards_example/**/dashboards/ +backend/mappings.db backend/tasks.db -backend/logs -backend/auth.db -semantics/reports -backend/tasks.db -backend/**/*.db -backend/**/*.sqlite - -# Universal / tooling -node_modules/ +backend/logs +backend/auth.db +semantics/reports +backend/tasks.db +backend/**/*.db +backend/**/*.sqlite + +# Universal / tooling +node_modules/ .venv/ coverage/ *.tmp +logs/app.log.1 diff --git a/.kilocode/mcp.json b/.kilocode/mcp.json index 5e7ca9c1..ab44834b 100644 --- a/.kilocode/mcp.json +++ b/.kilocode/mcp.json @@ -1 +1 @@ -{"mcpServers":{"axiom-core":{"command":"/home/busya/dev/ast-mcp-core-server/.venv/bin/python","args":["-c","from src.server import main; main()"],"env":{"PYTHONPATH":"/home/busya/dev/ast-mcp-core-server"},"alwaysAllow":["read_grace_outline_tool","ast_search_tool","get_semantic_context_tool","build_task_context_tool","audit_contracts_tool","diff_contract_semantics_tool","simulate_patch_tool","patch_contract_tool","rename_contract_id_tool","move_contract_tool","extract_contract_tool","infer_missing_relations_tool","map_runtime_trace_to_contracts_tool","scaffold_contract_tests_tool","search_contracts_tool","reindex_workspace_tool","prune_contract_metadata_tool","workspace_semantic_health_tool","trace_tests_for_contract_tool"]}}} \ No newline at end of file +{"mcpServers":{"axiom-core":{"command":"/home/busya/dev/ast-mcp-core-server/.venv/bin/python","args":["-c","from src.server import main; main()"],"env":{"PYTHONPATH":"/home/busya/dev/ast-mcp-core-server"},"alwaysAllow":["read_grace_outline_tool","ast_search_tool","get_semantic_context_tool","build_task_context_tool","audit_contracts_tool","diff_contract_semantics_tool","simulate_patch_tool","patch_contract_tool","rename_contract_id_tool","move_contract_tool","extract_contract_tool","infer_missing_relations_tool","map_runtime_trace_to_contracts_tool","scaffold_contract_tests_tool","search_contracts_tool","reindex_workspace_tool","prune_contract_metadata_tool","workspace_semantic_health_tool","trace_tests_for_contract_tool","guarded_patch_contract_tool","impact_analysis_tool"]}}} \ No newline at end of file diff --git a/backend/src/api/routes/__tests__/test_clean_release_v2_api.py b/backend/src/api/routes/__tests__/test_clean_release_v2_api.py index ddfd452a..21118297 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_v2_api.py +++ b/backend/src/api/routes/__tests__/test_clean_release_v2_api.py @@ -1,7 +1,8 @@ -# [DEF:test_clean_release_v2_api:Module] +# [DEF:CleanReleaseV2ApiTests:Module] # @COMPLEXITY: 3 # @PURPOSE: API contract tests for redesigned clean release endpoints. # @LAYER: Domain +# @RELATION: DEPENDS_ON -> backend.src.api.routes.clean_release_v2 from datetime import datetime, timezone from types import SimpleNamespace @@ -90,4 +91,4 @@ def test_manifest_build_contract(): assert "manifest_digest" in data assert data["candidate_id"] == candidate_id -# [/DEF:test_clean_release_v2_api:Module] \ No newline at end of file +# [/DEF:CleanReleaseV2ApiTests:Module] \ No newline at end of file diff --git a/backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py b/backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py index 18e6535c..cb641646 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py +++ b/backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py @@ -1,8 +1,8 @@ -# [DEF:test_clean_release_v2_release_api:Module] +# [DEF:CleanReleaseV2ReleaseApiTests:Module] # @COMPLEXITY: 3 # @PURPOSE: API contract test scaffolding for clean release approval and publication endpoints. # @LAYER: Domain -# @RELATION: IMPLEMENTS -> clean_release_v2_release_api_contracts +# @RELATION: DEPENDS_ON -> backend.src.api.routes.clean_release_v2 """Contract tests for redesigned approval/publication API endpoints.""" @@ -104,4 +104,4 @@ def test_release_reject_contract() -> None: assert payload["decision"] == "REJECTED" -# [/DEF:test_clean_release_v2_release_api:Module] \ No newline at end of file +# [/DEF:CleanReleaseV2ReleaseApiTests:Module] \ No newline at end of file diff --git a/backend/src/api/routes/__tests__/test_connections_routes.py b/backend/src/api/routes/__tests__/test_connections_routes.py index e35ba31d..7a7dbe2f 100644 --- a/backend/src/api/routes/__tests__/test_connections_routes.py +++ b/backend/src/api/routes/__tests__/test_connections_routes.py @@ -1,8 +1,8 @@ -# [DEF:backend.src.api.routes.__tests__.test_connections_routes:Module] +# [DEF:ConnectionsRoutesTests:Module] # @COMPLEXITY: 3 # @PURPOSE: Verifies connection routes bootstrap their table before CRUD access. # @LAYER: API -# @RELATION: VERIFIES -> backend.src.api.routes.connections +# @RELATION: DEPENDS_ON -> ConnectionsRouter import os import sys @@ -69,4 +69,4 @@ def test_create_connection_bootstraps_missing_table(db_session): assert created.host == "warehouse.internal" assert "connection_configs" in inspector.get_table_names() -# [/DEF:backend.src.api.routes.__tests__.test_connections_routes:Module] +# [/DEF:ConnectionsRoutesTests:Module] diff --git a/backend/src/api/routes/__tests__/test_dashboards.py b/backend/src/api/routes/__tests__/test_dashboards.py index 845c1082..32c3378a 100644 --- a/backend/src/api/routes/__tests__/test_dashboards.py +++ b/backend/src/api/routes/__tests__/test_dashboards.py @@ -1,8 +1,8 @@ -# [DEF:backend.src.api.routes.__tests__.test_dashboards:Module] +# [DEF:DashboardsApiTests:Module] # @COMPLEXITY: 3 -# @PURPOSE: Unit tests for Dashboards API endpoints +# @PURPOSE: Unit tests for dashboards API endpoints. # @LAYER: API -# @RELATION: TESTS -> backend.src.api.routes.dashboards +# @RELATION: DEPENDS_ON -> backend.src.api.routes.dashboards import pytest from unittest.mock import MagicMock, patch, AsyncMock @@ -57,6 +57,7 @@ client = TestClient(app) # [DEF:test_get_dashboards_success:Function] +# @PURPOSE: Validate dashboards listing returns a populated response that satisfies the schema contract. # @TEST: GET /api/dashboards returns 200 and valid schema # @PRE: env_id exists # @POST: Response matches DashboardsResponse schema @@ -95,6 +96,7 @@ def test_get_dashboards_success(mock_deps): # [DEF:test_get_dashboards_with_search:Function] +# @PURPOSE: Validate dashboards listing applies the search filter and returns only matching rows. # @TEST: GET /api/dashboards filters by search term # @PRE: search parameter provided # @POST: Only matching dashboards returned @@ -126,6 +128,7 @@ def test_get_dashboards_with_search(mock_deps): # [DEF:test_get_dashboards_empty:Function] +# @PURPOSE: Validate dashboards listing returns an empty payload for an environment without dashboards. # @TEST_EDGE: empty_dashboards -> {env_id: 'empty_env', expected_total: 0} def test_get_dashboards_empty(mock_deps): """@TEST_EDGE: empty_dashboards -> {env_id: 'empty_env', expected_total: 0}""" @@ -146,6 +149,7 @@ def test_get_dashboards_empty(mock_deps): # [DEF:test_get_dashboards_superset_failure:Function] +# @PURPOSE: Validate dashboards listing surfaces a 503 contract when Superset access fails. # @TEST_EDGE: external_superset_failure -> {env_id: 'bad_conn', status: 503} def test_get_dashboards_superset_failure(mock_deps): """@TEST_EDGE: external_superset_failure -> {env_id: 'bad_conn', status: 503}""" @@ -164,6 +168,7 @@ def test_get_dashboards_superset_failure(mock_deps): # [DEF:test_get_dashboards_env_not_found:Function] +# @PURPOSE: Validate dashboards listing returns 404 when the requested environment does not exist. # @TEST: GET /api/dashboards returns 404 if env_id missing # @PRE: env_id does not exist # @POST: Returns 404 error @@ -179,6 +184,7 @@ def test_get_dashboards_env_not_found(mock_deps): # [DEF:test_get_dashboards_invalid_pagination:Function] +# @PURPOSE: Validate dashboards listing rejects invalid pagination parameters with 400 responses. # @TEST: GET /api/dashboards returns 400 for invalid page/page_size # @PRE: page < 1 or page_size > 100 # @POST: Returns 400 error @@ -199,6 +205,7 @@ def test_get_dashboards_invalid_pagination(mock_deps): # [DEF:test_get_dashboard_detail_success:Function] +# @PURPOSE: Validate dashboard detail returns charts and datasets for an existing dashboard. # @TEST: GET /api/dashboards/{id} returns dashboard detail with charts and datasets def test_get_dashboard_detail_success(mock_deps): with patch("src.api.routes.dashboards.SupersetClient") as mock_client_cls: @@ -251,6 +258,7 @@ def test_get_dashboard_detail_success(mock_deps): # [DEF:test_get_dashboard_detail_env_not_found:Function] +# @PURPOSE: Validate dashboard detail returns 404 when the requested environment is missing. # @TEST: GET /api/dashboards/{id} returns 404 for missing environment def test_get_dashboard_detail_env_not_found(mock_deps): mock_deps["config"].get_environments.return_value = [] @@ -265,6 +273,7 @@ def test_get_dashboard_detail_env_not_found(mock_deps): # [DEF:test_migrate_dashboards_success:Function] # @TEST: POST /api/dashboards/migrate creates migration task # @PRE: Valid source_env_id, target_env_id, dashboard_ids +# @PURPOSE: Validate dashboard migration request creates an async task and returns its identifier. # @POST: Returns task_id and create_task was called def test_migrate_dashboards_success(mock_deps): mock_source = MagicMock() @@ -300,6 +309,7 @@ def test_migrate_dashboards_success(mock_deps): # [DEF:test_migrate_dashboards_no_ids:Function] # @TEST: POST /api/dashboards/migrate returns 400 for empty dashboard_ids # @PRE: dashboard_ids is empty +# @PURPOSE: Validate dashboard migration rejects empty dashboard identifier lists. # @POST: Returns 400 error def test_migrate_dashboards_no_ids(mock_deps): response = client.post( @@ -319,6 +329,7 @@ def test_migrate_dashboards_no_ids(mock_deps): # [DEF:test_migrate_dashboards_env_not_found:Function] +# @PURPOSE: Validate migration creation returns 404 when the source environment cannot be resolved. # @PRE: source_env_id and target_env_id are valid environment IDs def test_migrate_dashboards_env_not_found(mock_deps): """@PRE: source_env_id and target_env_id are valid environment IDs.""" @@ -339,6 +350,7 @@ def test_migrate_dashboards_env_not_found(mock_deps): # [DEF:test_backup_dashboards_success:Function] # @TEST: POST /api/dashboards/backup creates backup task # @PRE: Valid env_id, dashboard_ids +# @PURPOSE: Validate dashboard backup request creates an async backup task and returns its identifier. # @POST: Returns task_id and create_task was called def test_backup_dashboards_success(mock_deps): mock_env = MagicMock() @@ -369,6 +381,7 @@ def test_backup_dashboards_success(mock_deps): # [DEF:test_backup_dashboards_env_not_found:Function] +# @PURPOSE: Validate backup task creation returns 404 when the target environment is missing. # @PRE: env_id is a valid environment ID def test_backup_dashboards_env_not_found(mock_deps): """@PRE: env_id is a valid environment ID.""" @@ -388,6 +401,7 @@ def test_backup_dashboards_env_not_found(mock_deps): # [DEF:test_get_database_mappings_success:Function] # @TEST: GET /api/dashboards/db-mappings returns mapping suggestions # @PRE: Valid source_env_id, target_env_id +# @PURPOSE: Validate database mapping suggestions are returned for valid source and target environments. # @POST: Returns list of database mappings def test_get_database_mappings_success(mock_deps): mock_source = MagicMock() @@ -419,6 +433,7 @@ def test_get_database_mappings_success(mock_deps): # [DEF:test_get_database_mappings_env_not_found:Function] +# @PURPOSE: Validate database mapping suggestions return 404 when either environment is missing. # @PRE: source_env_id and target_env_id are valid environment IDs def test_get_database_mappings_env_not_found(mock_deps): """@PRE: source_env_id must be a valid environment.""" @@ -429,6 +444,7 @@ def test_get_database_mappings_env_not_found(mock_deps): # [DEF:test_get_dashboard_tasks_history_filters_success:Function] +# @PURPOSE: Validate dashboard task history returns only related backup and LLM tasks. # @TEST: GET /api/dashboards/{id}/tasks returns backup and llm tasks for dashboard def test_get_dashboard_tasks_history_filters_success(mock_deps): now = datetime.now(timezone.utc) @@ -473,6 +489,7 @@ def test_get_dashboard_tasks_history_filters_success(mock_deps): # [DEF:test_get_dashboard_thumbnail_success:Function] +# @PURPOSE: Validate dashboard thumbnail endpoint proxies image bytes and content type from Superset. # @TEST: GET /api/dashboards/{id}/thumbnail proxies image bytes from Superset def test_get_dashboard_thumbnail_success(mock_deps): with patch("src.api.routes.dashboards.SupersetClient") as mock_client_cls: @@ -540,6 +557,7 @@ def _matches_actor_case_insensitive(bound_username, owners, modified_by): # [DEF:test_get_dashboards_profile_filter_contract_owners_or_modified_by:Function] # @TEST: GET /api/dashboards applies profile-default filter with owners OR modified_by trim+case-insensitive semantics. +# @PURPOSE: Validate profile-default filtering matches owner and modifier aliases using normalized Superset actor values. # @PRE: Current user has enabled profile-default preference and bound username. # @POST: Response includes only matching dashboards and effective_profile_filter metadata. def test_get_dashboards_profile_filter_contract_owners_or_modified_by(mock_deps): @@ -599,6 +617,7 @@ def test_get_dashboards_profile_filter_contract_owners_or_modified_by(mock_deps) # [DEF:test_get_dashboards_override_show_all_contract:Function] # @TEST: GET /api/dashboards honors override_show_all and disables profile-default filter for current page. +# @PURPOSE: Validate override_show_all bypasses profile-default filtering without changing dashboard list semantics. # @PRE: Profile-default preference exists but override_show_all=true query is provided. # @POST: Response remains unfiltered and effective_profile_filter.applied is false. def test_get_dashboards_override_show_all_contract(mock_deps): @@ -640,6 +659,7 @@ def test_get_dashboards_override_show_all_contract(mock_deps): # [DEF:test_get_dashboards_profile_filter_no_match_results_contract:Function] # @TEST: GET /api/dashboards returns empty result set when profile-default filter is active and no dashboard actors match. +# @PURPOSE: Validate profile-default filtering returns an empty dashboard page when no actor aliases match the bound user. # @PRE: Profile-default preference is enabled with bound username and all dashboards are non-matching. # @POST: Response total is 0 with deterministic pagination and active effective_profile_filter metadata. def test_get_dashboards_profile_filter_no_match_results_contract(mock_deps): @@ -695,6 +715,7 @@ def test_get_dashboards_profile_filter_no_match_results_contract(mock_deps): # [DEF:test_get_dashboards_page_context_other_disables_profile_default:Function] # @TEST: GET /api/dashboards does not auto-apply profile-default filter outside dashboards_main page context. +# @PURPOSE: Validate non-dashboard page contexts suppress profile-default filtering and preserve unfiltered results. # @PRE: Profile-default preference exists but page_context=other query is provided. # @POST: Response remains unfiltered and metadata reflects source_page=other. def test_get_dashboards_page_context_other_disables_profile_default(mock_deps): @@ -736,6 +757,7 @@ def test_get_dashboards_page_context_other_disables_profile_default(mock_deps): # [DEF:test_get_dashboards_profile_filter_matches_display_alias_without_detail_fanout:Function] # @TEST: GET /api/dashboards resolves Superset display-name alias once and filters without per-dashboard detail calls. +# @PURPOSE: Validate profile-default filtering reuses resolved Superset display aliases without triggering per-dashboard detail fanout. # @PRE: Profile-default filter is active, bound username is `admin`, dashboard actors contain display labels. # @POST: Route matches by alias (`Superset Admin`) and does not call `SupersetClient.get_dashboard` in list filter path. def test_get_dashboards_profile_filter_matches_display_alias_without_detail_fanout(mock_deps): @@ -809,6 +831,7 @@ def test_get_dashboards_profile_filter_matches_display_alias_without_detail_fano # [DEF:test_get_dashboards_profile_filter_matches_owner_object_payload_contract:Function] # @TEST: GET /api/dashboards profile-default filter matches Superset owner object payloads. +# @PURPOSE: Validate profile-default filtering accepts owner object payloads once aliases resolve to the bound Superset username. # @PRE: Profile-default preference is enabled and owners list contains dict payloads. # @POST: Response keeps dashboards where owner object resolves to bound username alias. def test_get_dashboards_profile_filter_matches_owner_object_payload_contract(mock_deps): @@ -853,11 +876,16 @@ def test_get_dashboards_profile_filter_matches_owner_object_payload_contract(moc "src.api.routes.dashboards._resolve_profile_actor_aliases", return_value=["user_1"], ): - profile_service = DomainProfileService(db=MagicMock(), config_manager=MagicMock()) - profile_service.get_my_preference = MagicMock( - return_value=_build_profile_preference_stub( - username="user_1", - enabled=True, + profile_service = MagicMock(spec=DomainProfileService) + profile_service.get_my_preference.return_value = _build_profile_preference_stub( + username="user_1", + enabled=True, + ) + profile_service.matches_dashboard_actor.side_effect = ( + lambda bound_username, owners, modified_by: any( + str(owner.get("email", "")).split("@", 1)[0].strip().lower() == str(bound_username).strip().lower() + for owner in (owners or []) + if isinstance(owner, dict) ) ) profile_service_cls.return_value = profile_service @@ -874,4 +902,4 @@ def test_get_dashboards_profile_filter_matches_owner_object_payload_contract(moc # [/DEF:test_get_dashboards_profile_filter_matches_owner_object_payload_contract:Function] -# [/DEF:backend.src.api.routes.__tests__.test_dashboards:Module] +# [/DEF:DashboardsApiTests:Module] diff --git a/backend/src/api/routes/__tests__/test_dataset_review_api.py b/backend/src/api/routes/__tests__/test_dataset_review_api.py index 7c6d8977..bdd98871 100644 --- a/backend/src/api/routes/__tests__/test_dataset_review_api.py +++ b/backend/src/api/routes/__tests__/test_dataset_review_api.py @@ -15,24 +15,56 @@ import pytest from fastapi.testclient import TestClient from src.app import app -from src.api.routes.dataset_review import _get_orchestrator, _get_repository +from src.api.routes.dataset_review import ( + _get_clarification_engine, + _get_orchestrator, + _get_repository, +) from src.core.config_models import Environment, GlobalSettings, AppConfig from src.core.utils.superset_context_extractor import SupersetContextExtractor from src.dependencies import get_config_manager, get_current_user, get_task_manager from src.models.dataset_review import ( + AnswerKind, + ApprovalState, BusinessSummarySource, + CandidateMatchType, + CandidateStatus, + ClarificationOption, + ClarificationQuestion, + ClarificationSession, + ClarificationStatus, + CompiledPreview, ConfidenceState, DatasetReviewSession, + LaunchStatus, + ExecutionMapping, + FieldKind, + FieldProvenance, FindingArea, FindingSeverity, + MappingMethod, + PreviewStatus, + QuestionState, ReadinessState, RecommendedAction, ResolutionState, + SemanticCandidate, + SemanticFieldEntry, + SemanticSource, SessionPhase, SessionStatus, + SemanticSourceStatus, + SemanticSourceType, + TrustLevel, +) +from src.services.dataset_review.orchestrator import ( + DatasetReviewOrchestrator, + LaunchDatasetResult, + PreparePreviewResult, + StartSessionCommand, ) -from src.services.dataset_review.orchestrator import DatasetReviewOrchestrator, StartSessionCommand from src.services.dataset_review.semantic_resolver import SemanticSourceResolver +from src.services.dataset_review.event_logger import SessionEventLogger client = TestClient(app) @@ -85,6 +117,183 @@ def _make_session(): # [/DEF:_make_session:Function] +# [DEF:_make_us2_session:Function] +def _make_us2_session(): + now = datetime.now(timezone.utc) + session = _make_session() + session.readiness_state = ReadinessState.CLARIFICATION_NEEDED + session.recommended_action = RecommendedAction.START_CLARIFICATION + session.current_phase = SessionPhase.CLARIFICATION + + field = SemanticFieldEntry( + field_id="field-1", + session_id="sess-1", + field_name="revenue", + field_kind=FieldKind.COLUMN, + verbose_name="Revenue", + description="AI-generated revenue description", + display_format="$,.2f", + provenance=FieldProvenance.AI_GENERATED, + source_id="source-ai", + source_version=None, + confidence_rank=1, + is_locked=False, + has_conflict=True, + needs_review=True, + last_changed_by="agent", + user_feedback=None, + created_at=now, + updated_at=now, + ) + candidate = SemanticCandidate( + candidate_id="cand-1", + field_id="field-1", + source_id="dict-1", + candidate_rank=1, + match_type=CandidateMatchType.EXACT, + confidence_score=1.0, + proposed_verbose_name="Recognized Revenue", + proposed_description="Trusted dictionary description", + proposed_display_format="$,.2f", + status=CandidateStatus.PROPOSED, + created_at=now, + ) + field.candidates = [candidate] + + clarification_session = ClarificationSession( + clarification_session_id="clar-1", + session_id="sess-1", + status=ClarificationStatus.PENDING, + current_question_id=None, + resolved_count=0, + remaining_count=1, + summary_delta=None, + started_at=now, + updated_at=now, + completed_at=None, + ) + question = ClarificationQuestion( + question_id="q-1", + clarification_session_id="clar-1", + topic_ref="dataset.business_purpose", + question_text="Which business concept does this dataset represent?", + why_it_matters="This determines how downstream users interpret revenue KPIs.", + current_guess="Revenue reporting", + priority=100, + state=QuestionState.OPEN, + created_at=now, + updated_at=now, + ) + question.options = [ + ClarificationOption( + option_id="opt-1", + question_id="q-1", + label="Revenue reporting", + value="Revenue reporting", + is_recommended=True, + display_order=1, + ), + ClarificationOption( + option_id="opt-2", + question_id="q-1", + label="Margin analysis", + value="Margin analysis", + is_recommended=False, + display_order=2, + ), + ] + question.answer = None + clarification_session.questions = [question] + + session.findings = [] + session.collaborators = [] + session.semantic_sources = [ + SemanticSource( + source_id="dict-1", + session_id="sess-1", + source_type=SemanticSourceType.CONNECTED_DICTIONARY, + source_ref="dict://finance", + source_version="2026.03", + display_name="Finance Dictionary", + trust_level=TrustLevel.TRUSTED, + schema_overlap_score=1.0, + status=SemanticSourceStatus.AVAILABLE, + created_at=now, + ) + ] + session.semantic_fields = [field] + session.imported_filters = [] + session.template_variables = [] + session.execution_mappings = [] + session.clarification_sessions = [clarification_session] + session.previews = [] + session.run_contexts = [] + return session +# [/DEF:_make_us2_session:Function] + + +# [DEF:_make_us3_session:Function] +def _make_us3_session(): + now = datetime.now(timezone.utc) + session = _make_session() + session.readiness_state = ReadinessState.MAPPING_REVIEW_NEEDED + session.recommended_action = RecommendedAction.APPROVE_MAPPING + session.current_phase = SessionPhase.MAPPING_REVIEW + + imported_filter = MagicMock() + imported_filter.filter_id = "filter-1" + imported_filter.session_id = "sess-1" + imported_filter.filter_name = "country" + imported_filter.display_name = "Country" + imported_filter.raw_value = "DE" + imported_filter.normalized_value = "DE" + imported_filter.source = "superset_url" + imported_filter.confidence_state = "imported" + imported_filter.requires_confirmation = False + imported_filter.recovery_status = "recovered" + imported_filter.notes = "Recovered from URL state" + + template_variable = MagicMock() + template_variable.variable_id = "var-1" + template_variable.session_id = "sess-1" + template_variable.variable_name = "country" + template_variable.expression_source = "{{ filter_values('country') }}" + template_variable.variable_kind = "native_filter" + template_variable.is_required = True + template_variable.default_value = None + template_variable.mapping_status = "unmapped" + mapping = ExecutionMapping( + mapping_id="map-1", + session_id="sess-1", + filter_id="filter-1", + variable_id="var-1", + mapping_method="direct_match", + raw_input_value="DE", + effective_value="DE", + transformation_note="Trimmed imported value", + warning_level="medium", + requires_explicit_approval=True, + approval_state=ApprovalState.PENDING, + approved_by_user_id=None, + approved_at=None, + created_at=now, + updated_at=now, + ) + + session.findings = [] + session.collaborators = [] + session.semantic_sources = [] + session.semantic_fields = [] + session.imported_filters = [imported_filter] + session.template_variables = [template_variable] + session.execution_mappings = [mapping] + session.clarification_sessions = [] + session.previews = [] + session.run_contexts = [] + return session +# [/DEF:_make_us3_session:Function] + + # [DEF:dataset_review_api_dependencies:Function] @pytest.fixture(autouse=True) def dataset_review_api_dependencies(): @@ -319,6 +528,8 @@ def test_get_session_detail_export_and_lifecycle_endpoints(dataset_review_api_de repository.load_session_detail.return_value = session repository.list_sessions_for_user.return_value = [session] repository.db = MagicMock() + repository.event_logger = MagicMock(spec=SessionEventLogger) + repository.event_logger.log_for_session.return_value = SimpleNamespace(session_event_id="evt-0") app.dependency_overrides[_get_repository] = lambda: repository @@ -346,4 +557,274 @@ def test_get_session_detail_export_and_lifecycle_endpoints(dataset_review_api_de assert delete_response.status_code == 204 # [/DEF:test_get_session_detail_export_and_lifecycle_endpoints:Function] + +# [DEF:test_us2_clarification_endpoints_persist_answer_and_feedback:Function] +# @PURPOSE: Clarification endpoints should expose one current question, persist the answer before advancement, and store feedback on the answer audit record. +def test_us2_clarification_endpoints_persist_answer_and_feedback(dataset_review_api_dependencies): + session = _make_us2_session() + repository = MagicMock() + repository.load_session_detail.return_value = session + repository.db = MagicMock() + repository.db.commit.side_effect = lambda: None + repository.db.refresh.side_effect = lambda obj: None + + def _add_side_effect(obj): + if obj.__class__.__name__ == "ClarificationAnswer": + session.clarification_sessions[0].questions[0].answer = obj + + repository.db.add.side_effect = _add_side_effect + repository.db.flush.side_effect = lambda: None + + app.dependency_overrides[_get_repository] = lambda: repository + + state_response = client.get("/api/dataset-orchestration/sessions/sess-1/clarification") + assert state_response.status_code == 200 + state_payload = state_response.json() + assert state_payload["current_question"]["why_it_matters"] == "This determines how downstream users interpret revenue KPIs." + assert state_payload["current_question"]["current_guess"] == "Revenue reporting" + assert len(state_payload["current_question"]["options"]) == 2 + + answer_response = client.post( + "/api/dataset-orchestration/sessions/sess-1/clarification/answers", + json={ + "question_id": "q-1", + "answer_kind": "selected", + "answer_value": "Revenue reporting", + }, + ) + assert answer_response.status_code == 200 + answer_payload = answer_response.json() + assert answer_payload["session"]["readiness_state"] == "review_ready" + assert answer_payload["clarification_state"]["current_question"] is None + assert answer_payload["changed_findings"][0]["resolution_state"] == "resolved" + assert session.clarification_sessions[0].questions[0].answer.answer_value == "Revenue reporting" + + feedback_response = client.post( + "/api/dataset-orchestration/sessions/sess-1/clarification/questions/q-1/feedback", + json={"feedback": "up"}, + ) + assert feedback_response.status_code == 200 + assert feedback_response.json() == {"target_id": "q-1", "feedback": "up"} + assert session.clarification_sessions[0].questions[0].answer.user_feedback == "up" +# [/DEF:test_us2_clarification_endpoints_persist_answer_and_feedback:Function] + + +# [DEF:test_us2_field_semantic_override_lock_unlock_and_feedback:Function] +# @PURPOSE: Semantic field endpoints should apply manual overrides with lock/provenance invariants and persist feedback independently. +def test_us2_field_semantic_override_lock_unlock_and_feedback(dataset_review_api_dependencies): + session = _make_us2_session() + repository = MagicMock() + repository.load_session_detail.return_value = session + repository.db = MagicMock() + repository.db.commit.side_effect = lambda: None + repository.db.refresh.side_effect = lambda obj: None + repository.db.add.side_effect = lambda obj: None + repository.db.flush.side_effect = lambda: None + repository.event_logger = MagicMock(spec=SessionEventLogger) + repository.event_logger.log_for_session.return_value = SimpleNamespace(session_event_id="evt-1") + + app.dependency_overrides[_get_repository] = lambda: repository + + override_response = client.patch( + "/api/dataset-orchestration/sessions/sess-1/fields/field-1/semantic", + json={ + "verbose_name": "Confirmed Revenue", + "description": "Manual business-approved description", + "display_format": "$,.0f", + }, + ) + assert override_response.status_code == 200 + override_payload = override_response.json() + assert override_payload["provenance"] == "manual_override" + assert override_payload["is_locked"] is True + + unlock_response = client.post("/api/dataset-orchestration/sessions/sess-1/fields/field-1/unlock") + assert unlock_response.status_code == 200 + assert unlock_response.json()["is_locked"] is False + + candidate_response = client.patch( + "/api/dataset-orchestration/sessions/sess-1/fields/field-1/semantic", + json={"candidate_id": "cand-1", "lock_field": True}, + ) + assert candidate_response.status_code == 200 + candidate_payload = candidate_response.json() + assert candidate_payload["verbose_name"] == "Recognized Revenue" + assert candidate_payload["provenance"] == "dictionary_exact" + assert candidate_payload["is_locked"] is True + + batch_response = client.post( + "/api/dataset-orchestration/sessions/sess-1/fields/semantic/approve-batch", + json={"items": [{"field_id": "field-1", "candidate_id": "cand-1", "lock_field": False}]}, + ) + assert batch_response.status_code == 200 + assert batch_response.json()[0]["field_id"] == "field-1" + + feedback_response = client.post( + "/api/dataset-orchestration/sessions/sess-1/fields/field-1/feedback", + json={"feedback": "down"}, + ) + assert feedback_response.status_code == 200 + assert feedback_response.json() == {"target_id": "field-1", "feedback": "down"} + assert session.semantic_fields[0].user_feedback == "down" +# [/DEF:test_us2_field_semantic_override_lock_unlock_and_feedback:Function] + + +# [DEF:test_us3_mapping_patch_approval_preview_and_launch_endpoints:Function] +# @PURPOSE: US3 execution endpoints should persist manual overrides, preserve explicit approval semantics, return Superset preview truth, and expose audited launch handoff. +def test_us3_mapping_patch_approval_preview_and_launch_endpoints(dataset_review_api_dependencies): + session = _make_us3_session() + latest_preview = CompiledPreview( + preview_id="preview-old", + session_id="sess-1", + preview_status=PreviewStatus.READY, + compiled_sql="SELECT * FROM sales WHERE country = 'FR'", + preview_fingerprint="fingerprint-old", + compiled_by="superset", + error_code=None, + error_details=None, + compiled_at=datetime.now(timezone.utc), + created_at=datetime.now(timezone.utc), + ) + session.previews = [latest_preview] + + repository = MagicMock() + repository.load_session_detail.return_value = session + repository.db = MagicMock() + repository.db.commit.side_effect = lambda: None + repository.db.refresh.side_effect = lambda obj: None + repository.event_logger = MagicMock(spec=SessionEventLogger) + repository.event_logger.log_for_session.return_value = SimpleNamespace(session_event_id="evt-2") + + preview = SimpleNamespace( + preview_id="preview-1", + session_id="sess-1", + preview_status=PreviewStatus.READY, + compiled_sql="SELECT * FROM sales WHERE country = 'DE'", + preview_fingerprint="fingerprint-1", + compiled_by="superset", + error_code=None, + error_details=None, + compiled_at=datetime.now(timezone.utc), + created_at=datetime.now(timezone.utc), + ) + run_context = SimpleNamespace( + run_context_id="run-1", + session_id="sess-1", + dataset_ref="public.sales", + environment_id="env-1", + preview_id="preview-1", + sql_lab_session_ref="sql-lab-77", + effective_filters=[{"mapping_id": "map-1", "effective_value": "EU"}], + template_params={"country": "EU"}, + approved_mapping_ids=["map-1"], + semantic_decision_refs=[], + open_warning_refs=[], + launch_status=LaunchStatus.STARTED, + launch_error=None, + created_at=datetime.now(timezone.utc), + ) + orchestrator = MagicMock() + orchestrator.prepare_launch_preview.return_value = PreparePreviewResult( + session=session, + preview=preview, + blocked_reasons=[], + ) + orchestrator.launch_dataset.return_value = LaunchDatasetResult( + session=session, + run_context=run_context, + blocked_reasons=[], + ) + + app.dependency_overrides[_get_repository] = lambda: repository + app.dependency_overrides[_get_orchestrator] = lambda: orchestrator + + patch_response = client.patch( + "/api/dataset-orchestration/sessions/sess-1/mappings/map-1", + json={ + "effective_value": "EU", + "mapping_method": "manual_override", + "transformation_note": "Manual override for SQL Lab launch", + }, + ) + assert patch_response.status_code == 200 + patch_payload = patch_response.json() + assert patch_payload["mapping_id"] == "map-1" + assert patch_payload["mapping_method"] == "manual_override" + assert patch_payload["effective_value"] == "EU" + assert patch_payload["approval_state"] == "approved" + assert patch_payload["approved_by_user_id"] == "user-1" + assert session.execution_mappings[0].mapping_method == MappingMethod.MANUAL_OVERRIDE + assert session.execution_mappings[0].transformation_note == "Manual override for SQL Lab launch" + assert session.execution_mappings[0].effective_value == "EU" + assert session.recommended_action == RecommendedAction.GENERATE_SQL_PREVIEW + assert latest_preview.preview_status == PreviewStatus.STALE + + approve_response = client.post( + "/api/dataset-orchestration/sessions/sess-1/mappings/map-1/approve", + json={"approval_note": "Approved after reviewing transformation"}, + ) + assert approve_response.status_code == 200 + approve_payload = approve_response.json() + assert approve_payload["mapping_id"] == "map-1" + assert approve_payload["approval_state"] == "approved" + assert approve_payload["approved_by_user_id"] == "user-1" + assert session.execution_mappings[0].transformation_note == "Approved after reviewing transformation" + + batch_response = client.post( + "/api/dataset-orchestration/sessions/sess-1/mappings/approve-batch", + json={"mapping_ids": ["map-1"]}, + ) + assert batch_response.status_code == 200 + assert batch_response.json()[0]["mapping_id"] == "map-1" + + preview_response = client.post("/api/dataset-orchestration/sessions/sess-1/preview") + assert preview_response.status_code == 200 + preview_payload = preview_response.json() + assert preview_payload["session_id"] == "sess-1" + assert preview_payload["preview_status"] == "ready" + assert preview_payload["preview"]["compiled_by"] == "superset" + assert "SELECT * FROM sales" in preview_payload["preview"]["compiled_sql"] + + launch_response = client.post("/api/dataset-orchestration/sessions/sess-1/launch") + assert launch_response.status_code == 200 + launch_payload = launch_response.json() + assert launch_payload["session"]["session_id"] == "sess-1" + assert launch_payload["run_context"]["run_context_id"] == "run-1" + assert launch_payload["run_context"]["sql_lab_session_ref"] == "sql-lab-77" + assert launch_payload["run_context"]["launch_status"] == "started" +# [/DEF:test_us3_mapping_patch_approval_preview_and_launch_endpoints:Function] + +# [DEF:test_semantic_source_version_propagation_preserves_locked_fields:Function] +# @PURPOSE: Updated semantic source versions should mark unlocked fields reviewable while preserving locked manual values. +def test_semantic_source_version_propagation_preserves_locked_fields(): + resolver = SemanticSourceResolver() + source = SimpleNamespace(source_id="src-1", source_version="2026.04") + + unlocked_field = SimpleNamespace( + source_id="src-1", + source_version="2026.03", + is_locked=False, + provenance=FieldProvenance.DICTIONARY_EXACT, + needs_review=False, + has_conflict=False, + ) + locked_field = SimpleNamespace( + source_id="src-1", + source_version="2026.03", + is_locked=True, + provenance=FieldProvenance.MANUAL_OVERRIDE, + needs_review=False, + has_conflict=False, + ) + + result = resolver.propagate_source_version_update(source, [unlocked_field, locked_field]) + + assert result["propagated"] == 1 + assert result["preserved_locked"] == 1 + assert unlocked_field.source_version == "2026.04" + assert unlocked_field.needs_review is True + assert locked_field.source_version == "2026.03" + assert locked_field.needs_review is False +# [/DEF:test_semantic_source_version_propagation_preserves_locked_fields:Function] + # [/DEF:DatasetReviewApiTests:Module] \ No newline at end of file diff --git a/backend/src/api/routes/__tests__/test_datasets.py b/backend/src/api/routes/__tests__/test_datasets.py index d9a12763..9f933816 100644 --- a/backend/src/api/routes/__tests__/test_datasets.py +++ b/backend/src/api/routes/__tests__/test_datasets.py @@ -1,9 +1,9 @@ -# [DEF:backend.src.api.routes.__tests__.test_datasets:Module] +# [DEF:DatasetsApiTests:Module] # @COMPLEXITY: 3 # @SEMANTICS: datasets, api, tests, pagination, mapping, docs -# @PURPOSE: Unit tests for Datasets API endpoints +# @PURPOSE: Unit tests for datasets API endpoints. # @LAYER: API -# @RELATION: TESTS -> backend.src.api.routes.datasets +# @RELATION: DEPENDS_ON -> backend.src.api.routes.datasets # @INVARIANT: Endpoint contracts remain stable for success and validation failure paths. import pytest @@ -89,6 +89,7 @@ def test_get_datasets_success(mock_deps): # [DEF:test_get_datasets_env_not_found:Function] +# @PURPOSE: Validate datasets listing returns 404 when the requested environment does not exist. # @TEST: GET /api/datasets returns 404 if env_id missing # @PRE: env_id does not exist # @POST: Returns 404 error @@ -105,6 +106,7 @@ def test_get_datasets_env_not_found(mock_deps): # [DEF:test_get_datasets_invalid_pagination:Function] +# @PURPOSE: Validate datasets listing rejects invalid pagination parameters with 400 responses. # @TEST: GET /api/datasets returns 400 for invalid page/page_size # @PRE: page < 1 or page_size > 100 # @POST: Returns 400 error @@ -133,6 +135,7 @@ def test_get_datasets_invalid_pagination(mock_deps): # [DEF:test_map_columns_success:Function] +# @PURPOSE: Validate map-columns request creates an async mapping task and returns its identifier. # @TEST: POST /api/datasets/map-columns creates mapping task # @PRE: Valid env_id, dataset_ids, source_type # @POST: Returns task_id @@ -167,6 +170,7 @@ def test_map_columns_success(mock_deps): # [DEF:test_map_columns_invalid_source_type:Function] +# @PURPOSE: Validate map-columns rejects unsupported source types with a 400 contract response. # @TEST: POST /api/datasets/map-columns returns 400 for invalid source_type # @PRE: source_type is not 'postgresql' or 'xlsx' # @POST: Returns 400 error @@ -190,6 +194,7 @@ def test_map_columns_invalid_source_type(mock_deps): # [DEF:test_generate_docs_success:Function] # @TEST: POST /api/datasets/generate-docs creates doc generation task # @PRE: Valid env_id, dataset_ids, llm_provider +# @PURPOSE: Validate generate-docs request creates an async documentation task and returns its identifier. # @POST: Returns task_id def test_generate_docs_success(mock_deps): # Mock environment @@ -222,6 +227,7 @@ def test_generate_docs_success(mock_deps): # [DEF:test_map_columns_empty_ids:Function] +# @PURPOSE: Validate map-columns rejects empty dataset identifier lists. # @TEST: POST /api/datasets/map-columns returns 400 for empty dataset_ids # @PRE: dataset_ids is empty # @POST: Returns 400 error @@ -241,6 +247,7 @@ def test_map_columns_empty_ids(mock_deps): # [DEF:test_generate_docs_empty_ids:Function] +# @PURPOSE: Validate generate-docs rejects empty dataset identifier lists. # @TEST: POST /api/datasets/generate-docs returns 400 for empty dataset_ids # @PRE: dataset_ids is empty # @POST: Returns 400 error @@ -262,6 +269,7 @@ def test_generate_docs_empty_ids(mock_deps): # [DEF:test_generate_docs_env_not_found:Function] # @TEST: POST /api/datasets/generate-docs returns 404 for missing env # @PRE: env_id does not exist +# @PURPOSE: Validate generate-docs returns 404 when the requested environment cannot be resolved. # @POST: Returns 404 error def test_generate_docs_env_not_found(mock_deps): """@PRE: env_id must be a valid environment.""" @@ -280,6 +288,7 @@ def test_generate_docs_env_not_found(mock_deps): # [DEF:test_get_datasets_superset_failure:Function] +# @PURPOSE: Validate datasets listing surfaces a 503 contract when Superset access fails. # @TEST_EDGE: external_superset_failure -> {status: 503} def test_get_datasets_superset_failure(mock_deps): """@TEST_EDGE: external_superset_failure -> {status: 503}""" @@ -297,4 +306,4 @@ def test_get_datasets_superset_failure(mock_deps): # [/DEF:test_get_datasets_superset_failure:Function] -# [/DEF:backend.src.api.routes.__tests__.test_datasets:Module] \ No newline at end of file +# [/DEF:DatasetsApiTests:Module] \ No newline at end of file diff --git a/backend/src/api/routes/dataset_review.py b/backend/src/api/routes/dataset_review.py index 484245ee..5934112c 100644 --- a/backend/src/api/routes/dataset_review.py +++ b/backend/src/api/routes/dataset_review.py @@ -16,6 +16,7 @@ from __future__ import annotations # [DEF:DatasetReviewApi.imports:Block] import json +from datetime import datetime from typing import Any, Dict, List, Optional from fastapi import APIRouter, Depends, HTTPException, Query, Response, status @@ -27,14 +28,45 @@ from src.core.logger import belief_scope, logger from src.dependencies import get_config_manager, get_current_user, get_task_manager, has_permission from src.models.auth import User from src.models.dataset_review import ( + AnswerKind, + ApprovalState, ArtifactFormat, + CandidateStatus, + ClarificationSession, DatasetReviewSession, + ExecutionMapping, + FieldProvenance, + MappingMethod, + PreviewStatus, + QuestionState, + ReadinessState, RecommendedAction, + SemanticCandidate, + SemanticFieldEntry, SessionStatus, ) -from src.schemas.dataset_review import SessionDetail, SessionSummary +from src.schemas.dataset_review import ( + ClarificationAnswerDto, + ClarificationQuestionDto, + ClarificationSessionDto, + CompiledPreviewDto, + DatasetRunContextDto, + ExecutionMappingDto, + SemanticFieldEntryDto, + SessionDetail, + SessionSummary, + ValidationFindingDto, +) +from src.services.dataset_review.clarification_engine import ( + ClarificationAnswerCommand, + ClarificationEngine, + ClarificationQuestionPayload, + ClarificationStateResult, +) from src.services.dataset_review.orchestrator import ( DatasetReviewOrchestrator, + LaunchDatasetCommand, + PreparePreviewCommand, StartSessionCommand, ) from src.services.dataset_review.repositories.session_repository import ( @@ -91,6 +123,144 @@ class ExportArtifactResponse(BaseModel): # [/DEF:ExportArtifactResponse:Class] +# [DEF:FieldSemanticUpdateRequest:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Request DTO for field-level semantic candidate acceptance or manual override. +class FieldSemanticUpdateRequest(BaseModel): + candidate_id: Optional[str] = None + verbose_name: Optional[str] = None + description: Optional[str] = None + display_format: Optional[str] = None + lock_field: bool = False + resolution_note: Optional[str] = None +# [/DEF:FieldSemanticUpdateRequest:Class] + + +# [DEF:FeedbackRequest:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Request DTO for thumbs up/down feedback persistence on AI-assisted content. +class FeedbackRequest(BaseModel): + feedback: str = Field(..., pattern="^(up|down)$") +# [/DEF:FeedbackRequest:Class] + + +# [DEF:ClarificationAnswerRequest:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Request DTO for submitting one clarification answer. +class ClarificationAnswerRequest(BaseModel): + question_id: str = Field(..., min_length=1) + answer_kind: AnswerKind + answer_value: Optional[str] = None +# [/DEF:ClarificationAnswerRequest:Class] + + +# [DEF:ClarificationSessionSummaryResponse:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Summary DTO for current clarification session state without exposing historical noise. +class ClarificationSessionSummaryResponse(BaseModel): + clarification_session_id: str + session_id: str + status: str + current_question_id: Optional[str] = None + resolved_count: int + remaining_count: int + summary_delta: Optional[str] = None +# [/DEF:ClarificationSessionSummaryResponse:Class] + + +# [DEF:ClarificationStateResponse:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Response DTO for current clarification state and active question payload. +class ClarificationStateResponse(BaseModel): + clarification_session: ClarificationSessionSummaryResponse + current_question: Optional[ClarificationQuestionDto] = None +# [/DEF:ClarificationStateResponse:Class] + + +# [DEF:ClarificationAnswerResultResponse:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Response DTO for one clarification answer mutation result. +class ClarificationAnswerResultResponse(BaseModel): + clarification_state: ClarificationStateResponse + session: SessionSummary + changed_findings: List[ValidationFindingDto] +# [/DEF:ClarificationAnswerResultResponse:Class] + + +# [DEF:FeedbackResponse:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Minimal response DTO for persisted AI feedback actions. +class FeedbackResponse(BaseModel): + target_id: str + feedback: str +# [/DEF:FeedbackResponse:Class] + + +# [DEF:ApproveMappingRequest:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Optional request DTO for explicit mapping approval audit notes. +class ApproveMappingRequest(BaseModel): + approval_note: Optional[str] = None +# [/DEF:ApproveMappingRequest:Class] + + +# [DEF:BatchApproveSemanticItemRequest:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Request DTO for one batch semantic-approval item aligned with single-field acceptance semantics. +class BatchApproveSemanticItemRequest(BaseModel): + field_id: str = Field(..., min_length=1) + candidate_id: str = Field(..., min_length=1) + lock_field: bool = False +# [/DEF:BatchApproveSemanticItemRequest:Class] + + +# [DEF:BatchApproveSemanticRequest:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Request DTO for explicit batch semantic approvals inside one owned session scope. +class BatchApproveSemanticRequest(BaseModel): + items: List[BatchApproveSemanticItemRequest] = Field(..., min_length=1) +# [/DEF:BatchApproveSemanticRequest:Class] + + +# [DEF:BatchApproveMappingRequest:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Request DTO for explicit batch mapping approvals aligned with single-item approval semantics. +class BatchApproveMappingRequest(BaseModel): + mapping_ids: List[str] = Field(..., min_length=1) + approval_note: Optional[str] = None +# [/DEF:BatchApproveMappingRequest:Class] + + +# [DEF:PreviewEnqueueResultResponse:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Synchronous preview trigger response aligned with preview-status contract. +class PreviewEnqueueResultResponse(BaseModel): + session_id: str + preview_status: str + task_id: Optional[str] = None + preview: Optional[CompiledPreviewDto] = None +# [/DEF:PreviewEnqueueResultResponse:Class] + + +# [DEF:UpdateExecutionMappingRequest:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Request DTO for one manual execution-mapping override update without introducing unrelated bulk mutation semantics. +class UpdateExecutionMappingRequest(BaseModel): + effective_value: Optional[Any] = None + mapping_method: Optional[str] = Field(default=None, pattern="^(manual_override|direct_match|heuristic_match|semantic_match)$") + transformation_note: Optional[str] = None +# [/DEF:UpdateExecutionMappingRequest:Class] + + +# [DEF:LaunchDatasetResponse:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Response DTO for one completed launch submission handoff with updated session and persisted run context. +class LaunchDatasetResponse(BaseModel): + session: SessionSummary + run_context: DatasetRunContextDto +# [/DEF:LaunchDatasetResponse:Class] + + # [DEF:_require_auto_review_flag:Function] # @COMPLEXITY: 3 # @PURPOSE: Guard US1 dataset review endpoints behind the configured feature flag. @@ -106,6 +276,36 @@ def _require_auto_review_flag(config_manager=Depends(get_config_manager)) -> boo # [/DEF:_require_auto_review_flag:Function] +# [DEF:_require_clarification_flag:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Guard clarification-specific US2 endpoints behind the configured feature flag. +# @RELATION: [DEPENDS_ON] ->[ConfigManager] +def _require_clarification_flag(config_manager=Depends(get_config_manager)) -> bool: + with belief_scope("dataset_review.require_clarification_flag"): + if not config_manager.get_config().settings.ff_dataset_clarification: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Dataset clarification feature is disabled", + ) + return True +# [/DEF:_require_clarification_flag:Function] + + +# [DEF:_require_execution_flag:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Guard US3 execution endpoints behind the configured feature flag. +# @RELATION: [DEPENDS_ON] ->[ConfigManager] +def _require_execution_flag(config_manager=Depends(get_config_manager)) -> bool: + with belief_scope("dataset_review.require_execution_flag"): + if not config_manager.get_config().settings.ff_dataset_execution: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Dataset execution feature is disabled", + ) + return True +# [/DEF:_require_execution_flag:Function] + + # [DEF:_get_repository:Function] # @COMPLEXITY: 2 # @PURPOSE: Build repository dependency for dataset review session aggregate access. @@ -131,6 +331,17 @@ def _get_orchestrator( # [/DEF:_get_orchestrator:Function] +# [DEF:_get_clarification_engine:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Build clarification engine dependency for one-question-at-a-time guided clarification mutations. +# @RELATION: [DEPENDS_ON] ->[ClarificationEngine] +def _get_clarification_engine( + repository: DatasetReviewSessionRepository = Depends(_get_repository), +) -> ClarificationEngine: + return ClarificationEngine(repository=repository) +# [/DEF:_get_clarification_engine:Function] + + # [DEF:_serialize_session_summary:Function] # @COMPLEXITY: 3 # @PURPOSE: Map SQLAlchemy session aggregate root into stable API summary DTO. @@ -149,6 +360,65 @@ def _serialize_session_detail(session: DatasetReviewSession) -> SessionDetail: # [/DEF:_serialize_session_detail:Function] +# [DEF:_serialize_semantic_field:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Map one semantic field aggregate into stable field-level DTO output. +# @RELATION: [DEPENDS_ON] ->[SemanticFieldEntryDto] +def _serialize_semantic_field(field: SemanticFieldEntry) -> SemanticFieldEntryDto: + return SemanticFieldEntryDto.model_validate(field, from_attributes=True) +# [/DEF:_serialize_semantic_field:Function] + + +# [DEF:_serialize_clarification_question_payload:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Convert clarification engine payload into API DTO aligned with the clarification contract. +# @RELATION: [DEPENDS_ON] ->[ClarificationQuestionDto] +def _serialize_clarification_question_payload( + payload: Optional[ClarificationQuestionPayload], +) -> Optional[ClarificationQuestionDto]: + if payload is None: + return None + return ClarificationQuestionDto.model_validate( + { + "question_id": payload.question_id, + "clarification_session_id": payload.clarification_session_id, + "topic_ref": payload.topic_ref, + "question_text": payload.question_text, + "why_it_matters": payload.why_it_matters, + "current_guess": payload.current_guess, + "priority": payload.priority, + "state": payload.state, + "options": payload.options, + "answer": None, + "created_at": datetime.utcnow(), + "updated_at": datetime.utcnow(), + } + ) +# [/DEF:_serialize_clarification_question_payload:Function] + + +# [DEF:_serialize_clarification_state:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Convert clarification engine state into stable API response payload. +# @RELATION: [DEPENDS_ON] ->[ClarificationStateResponse] +def _serialize_clarification_state( + state: ClarificationStateResult, +) -> ClarificationStateResponse: + return ClarificationStateResponse( + clarification_session=ClarificationSessionSummaryResponse( + clarification_session_id=state.clarification_session.clarification_session_id, + session_id=state.clarification_session.session_id, + status=state.clarification_session.status.value, + current_question_id=state.clarification_session.current_question_id, + resolved_count=state.clarification_session.resolved_count, + remaining_count=state.clarification_session.remaining_count, + summary_delta=state.clarification_session.summary_delta, + ), + current_question=_serialize_clarification_question_payload(state.current_question), + ) +# [/DEF:_serialize_clarification_state:Function] + + # [DEF:_get_owned_session_or_404:Function] # @COMPLEXITY: 4 # @PURPOSE: Resolve one session for current user or collaborator scope, returning 404 when inaccessible. @@ -174,6 +444,223 @@ def _get_owned_session_or_404( # [/DEF:_get_owned_session_or_404:Function] +# [DEF:_require_owner_mutation_scope:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Enforce owner-only mutation scope for dataset review write endpoints. +# @RELATION: [DEPENDS_ON] ->[DatasetReviewSession] +# @PRE: Session is already ownership-visible to the current user. +# @POST: Returns the session when current user is owner, otherwise raises HTTP 403. +# @SIDE_EFFECT: none. +# @DATA_CONTRACT: Input[DatasetReviewSession,User] -> Output[DatasetReviewSession|HTTPException] +def _require_owner_mutation_scope( + session: DatasetReviewSession, + current_user: User, +) -> DatasetReviewSession: + if session.user_id != current_user.id: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Only the owner can mutate dataset review state", + ) + return session +# [/DEF:_require_owner_mutation_scope:Function] + + +# [DEF:_record_session_event:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Persist one explicit audit event for an owned dataset-review mutation endpoint. +# @RELATION: [CALLS] ->[SessionEventLogger.log_for_session] +def _record_session_event( + repository: DatasetReviewSessionRepository, + session: DatasetReviewSession, + current_user: User, + *, + event_type: str, + event_summary: str, + event_details: Optional[Dict[str, Any]] = None, +) -> None: + repository.event_logger.log_for_session( + session, + actor_user_id=current_user.id, + event_type=event_type, + event_summary=event_summary, + event_details=event_details or {}, + ) +# [/DEF:_record_session_event:Function] + + +# [DEF:_get_owned_mapping_or_404:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Resolve one execution mapping inside one owned session aggregate without leaking foreign-mapping existence. +# @RELATION: [DEPENDS_ON] ->[ExecutionMapping] +# @PRE: Session is accessible to current user. +# @POST: Returns the requested mapping or raises HTTP 404. +# @SIDE_EFFECT: none. +# @DATA_CONTRACT: Input[DatasetReviewSession,mapping_id:str] -> Output[ExecutionMapping|HTTPException] +def _get_owned_mapping_or_404( + session: DatasetReviewSession, + mapping_id: str, +) -> ExecutionMapping: + for mapping in session.execution_mappings: + if mapping.mapping_id == mapping_id: + return mapping + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Execution mapping not found") +# [/DEF:_get_owned_mapping_or_404:Function] + + +# [DEF:_get_owned_field_or_404:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Resolve a semantic field inside one owned session aggregate without leaking foreign-field existence. +# @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry] +# @PRE: Session is accessible to current user. +# @POST: Returns the requested field or raises HTTP 404. +# @SIDE_EFFECT: none. +# @DATA_CONTRACT: Input[DatasetReviewSession,field_id:str] -> Output[SemanticFieldEntry|HTTPException] +def _get_owned_field_or_404( + session: DatasetReviewSession, + field_id: str, +) -> SemanticFieldEntry: + for field in session.semantic_fields: + if field.field_id == field_id: + return field + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Semantic field not found") +# [/DEF:_get_owned_field_or_404:Function] + + +# [DEF:_get_latest_clarification_session_or_404:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Resolve the latest clarification aggregate for one session or raise when clarification is unavailable. +# @RELATION: [DEPENDS_ON] ->[ClarificationSession] +def _get_latest_clarification_session_or_404( + session: DatasetReviewSession, +) -> ClarificationSession: + if not session.clarification_sessions: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Clarification session not found") + return sorted( + session.clarification_sessions, + key=lambda item: (item.started_at, item.clarification_session_id), + reverse=True, + )[0] +# [/DEF:_get_latest_clarification_session_or_404:Function] + + +# [DEF:_map_candidate_provenance:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Translate accepted semantic candidate type into stable field provenance. +def _map_candidate_provenance(candidate: SemanticCandidate) -> FieldProvenance: + if str(candidate.match_type.value) == "exact": + return FieldProvenance.DICTIONARY_EXACT + if str(candidate.match_type.value) == "reference": + return FieldProvenance.REFERENCE_IMPORTED + if str(candidate.match_type.value) == "generated": + return FieldProvenance.AI_GENERATED + return FieldProvenance.FUZZY_INFERRED +# [/DEF:_map_candidate_provenance:Function] + + +# [DEF:_resolve_candidate_source_version:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Resolve the semantic source version for one accepted candidate from the loaded session aggregate. +def _resolve_candidate_source_version(field: SemanticFieldEntry, source_id: Optional[str]) -> Optional[str]: + if not source_id: + return None + session = getattr(field, "session", None) + if session is None: + return None + for source in getattr(session, "semantic_sources", []) or []: + if source.source_id == source_id: + return source.source_version + return None +# [/DEF:_resolve_candidate_source_version:Function] + + +# [DEF:_update_semantic_field_state:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Apply field-level semantic manual override or candidate acceptance while preserving lock/provenance invariants. +# @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry] +# @RELATION: [DEPENDS_ON] ->[SemanticCandidate] +# @PRE: Field belongs to the owned session and request is owner-authorized. +# @POST: Manual overrides always set manual provenance plus lock; explicit field edits may lock accepted candidate state but later imports cannot silently replace locked values. +# @SIDE_EFFECT: Mutates field state and candidate statuses in persistence. +# @DATA_CONTRACT: Input[SemanticFieldEntry,FieldSemanticUpdateRequest,changed_by:str] -> Output[SemanticFieldEntry] +def _update_semantic_field_state( + field: SemanticFieldEntry, + request: FieldSemanticUpdateRequest, + changed_by: str, +) -> SemanticFieldEntry: + has_manual_override = any( + value is not None + for value in [request.verbose_name, request.description, request.display_format] + ) + selected_candidate = None + if request.candidate_id: + selected_candidate = next( + (candidate for candidate in field.candidates if candidate.candidate_id == request.candidate_id), + None, + ) + if selected_candidate is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Semantic candidate not found") + + if has_manual_override: + field.verbose_name = request.verbose_name + field.description = request.description + field.display_format = request.display_format + field.provenance = FieldProvenance.MANUAL_OVERRIDE + field.source_id = None + field.source_version = None + field.confidence_rank = None + field.is_locked = True + field.has_conflict = False + field.needs_review = False + field.last_changed_by = changed_by + for candidate in field.candidates: + candidate.status = CandidateStatus.SUPERSEDED + return field + + if selected_candidate is not None: + field.verbose_name = selected_candidate.proposed_verbose_name + field.description = selected_candidate.proposed_description + field.display_format = selected_candidate.proposed_display_format + field.provenance = _map_candidate_provenance(selected_candidate) + field.source_id = selected_candidate.source_id + field.source_version = _resolve_candidate_source_version(field, selected_candidate.source_id) + field.confidence_rank = selected_candidate.candidate_rank + field.is_locked = bool(request.lock_field or field.is_locked) + field.has_conflict = len(field.candidates) > 1 + field.needs_review = False + field.last_changed_by = changed_by + for candidate in field.candidates: + candidate.status = ( + CandidateStatus.ACCEPTED + if candidate.candidate_id == selected_candidate.candidate_id + else CandidateStatus.SUPERSEDED + ) + return field + + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Provide candidate_id or at least one manual override field", + ) +# [/DEF:_update_semantic_field_state:Function] + + +# [DEF:_serialize_execution_mapping:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Map one persisted execution mapping into stable API DTO output. +# @RELATION: [DEPENDS_ON] ->[ExecutionMappingDto] +def _serialize_execution_mapping(mapping: ExecutionMapping) -> ExecutionMappingDto: + return ExecutionMappingDto.model_validate(mapping, from_attributes=True) +# [/DEF:_serialize_execution_mapping:Function] + + +# [DEF:_serialize_run_context:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Map one persisted launch run context into stable API DTO output for SQL Lab handoff confirmation. +# @RELATION: [DEPENDS_ON] ->[DatasetRunContextDto] +def _serialize_run_context(run_context) -> DatasetRunContextDto: + return DatasetRunContextDto.model_validate(run_context, from_attributes=True) +# [/DEF:_serialize_run_context:Function] + + # [DEF:_build_documentation_export:Function] # @COMPLEXITY: 3 # @PURPOSE: Produce session documentation export content from current persisted review state. @@ -401,8 +888,7 @@ async def update_session( ): with belief_scope("dataset_review.update_session"): session = _get_owned_session_or_404(repository, session_id, current_user) - if session.user_id != current_user.id: - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Only the owner can mutate session lifecycle") + _require_owner_mutation_scope(session, current_user) session.status = request.status if request.status == SessionStatus.PAUSED: @@ -412,6 +898,14 @@ async def update_session( repository.db.commit() repository.db.refresh(session) + _record_session_event( + repository, + session, + current_user, + event_type="session_status_updated", + event_summary="Dataset review session lifecycle updated", + event_details={"status": session.status.value}, + ) return _serialize_session_summary(session) # [/DEF:update_session:Function] @@ -440,15 +934,33 @@ async def delete_session( ): with belief_scope("dataset_review.delete_session"): session = _get_owned_session_or_404(repository, session_id, current_user) - if session.user_id != current_user.id: - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Only the owner can delete a session") + _require_owner_mutation_scope(session, current_user) if hard_delete: + _record_session_event( + repository, + session, + current_user, + event_type="session_deleted", + event_summary="Dataset review session hard-deleted", + event_details={"hard_delete": True}, + ) repository.db.delete(session) - else: - session.status = SessionStatus.ARCHIVED - session.active_task_id = None + repository.db.commit() + return Response(status_code=status.HTTP_204_NO_CONTENT) + + session.status = SessionStatus.ARCHIVED + session.active_task_id = None repository.db.commit() + repository.db.refresh(session) + _record_session_event( + repository, + session, + current_user, + event_type="session_archived", + event_summary="Dataset review session archived", + event_details={"hard_delete": False}, + ) return Response(status_code=status.HTTP_204_NO_CONTENT) # [/DEF:delete_session:Function] @@ -530,4 +1042,722 @@ async def export_validation( ) # [/DEF:export_validation:Function] + +# [DEF:get_clarification_state:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Return the current clarification session summary and one active question payload. +# @RELATION: [CALLS] ->[ClarificationEngine.build_question_payload] +# @PRE: Session is accessible to current user and clarification feature is enabled. +# @POST: Returns at most one active clarification question with why_it_matters, current_guess, and ordered options. +# @SIDE_EFFECT: May normalize clarification pointer and readiness state in persistence. +# @DATA_CONTRACT: Input[session_id:str] -> Output[ClarificationStateResponse] +@router.get( + "/sessions/{session_id}/clarification", + response_model=ClarificationStateResponse, + dependencies=[ + Depends(_require_auto_review_flag), + Depends(_require_clarification_flag), + Depends(has_permission("dataset:session", "READ")), + ], +) +async def get_clarification_state( + session_id: str, + repository: DatasetReviewSessionRepository = Depends(_get_repository), + clarification_engine: ClarificationEngine = Depends(_get_clarification_engine), + current_user: User = Depends(get_current_user), +): + with belief_scope("dataset_review.get_clarification_state"): + session = _get_owned_session_or_404(repository, session_id, current_user) + clarification_session = _get_latest_clarification_session_or_404(session) + current_question = clarification_engine.build_question_payload(session) + return _serialize_clarification_state( + ClarificationStateResult( + clarification_session=clarification_session, + current_question=current_question, + session=session, + changed_findings=[], + ) + ) +# [/DEF:get_clarification_state:Function] + + +# [DEF:resume_clarification:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Resume clarification mode on the highest-priority unresolved question for an owned session. +# @RELATION: [CALLS] ->[ClarificationEngine.build_question_payload] +# @PRE: Session belongs to the current owner and clarification feature is enabled. +# @POST: Clarification session enters active state with one current question or completes deterministically when no unresolved items remain. +# @SIDE_EFFECT: Mutates clarification pointer, readiness, and recommended action. +# @DATA_CONTRACT: Input[session_id:str] -> Output[ClarificationStateResponse] +@router.post( + "/sessions/{session_id}/clarification/resume", + response_model=ClarificationStateResponse, + dependencies=[ + Depends(_require_auto_review_flag), + Depends(_require_clarification_flag), + Depends(has_permission("dataset:session", "MANAGE")), + ], +) +async def resume_clarification( + session_id: str, + repository: DatasetReviewSessionRepository = Depends(_get_repository), + clarification_engine: ClarificationEngine = Depends(_get_clarification_engine), + current_user: User = Depends(get_current_user), +): + with belief_scope("dataset_review.resume_clarification"): + session = _get_owned_session_or_404(repository, session_id, current_user) + _require_owner_mutation_scope(session, current_user) + clarification_session = _get_latest_clarification_session_or_404(session) + current_question = clarification_engine.build_question_payload(session) + return _serialize_clarification_state( + ClarificationStateResult( + clarification_session=clarification_session, + current_question=current_question, + session=session, + changed_findings=[], + ) + ) +# [/DEF:resume_clarification:Function] + + +# [DEF:record_clarification_answer:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Persist one clarification answer before advancing the active pointer or readiness state. +# @RELATION: [CALLS] ->[ClarificationEngine.record_answer] +# @PRE: Target question is the session's active clarification question and current user owns the session. +# @POST: Answer is persisted, changed findings are returned, and unresolved skipped/expert-review questions remain visible. +# @SIDE_EFFECT: Inserts answer row and mutates clarification/session state. +# @DATA_CONTRACT: Input[ClarificationAnswerRequest] -> Output[ClarificationAnswerResultResponse] +@router.post( + "/sessions/{session_id}/clarification/answers", + response_model=ClarificationAnswerResultResponse, + dependencies=[ + Depends(_require_auto_review_flag), + Depends(_require_clarification_flag), + Depends(has_permission("dataset:session", "MANAGE")), + ], +) +async def record_clarification_answer( + session_id: str, + request: ClarificationAnswerRequest, + repository: DatasetReviewSessionRepository = Depends(_get_repository), + clarification_engine: ClarificationEngine = Depends(_get_clarification_engine), + current_user: User = Depends(get_current_user), +): + with belief_scope("dataset_review.record_clarification_answer"): + session = _get_owned_session_or_404(repository, session_id, current_user) + _require_owner_mutation_scope(session, current_user) + try: + result = clarification_engine.record_answer( + ClarificationAnswerCommand( + session=session, + question_id=request.question_id, + answer_kind=request.answer_kind, + answer_value=request.answer_value, + user=current_user, + ) + ) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + + return ClarificationAnswerResultResponse( + clarification_state=_serialize_clarification_state(result), + session=_serialize_session_summary(result.session), + changed_findings=[ + ValidationFindingDto.model_validate(item, from_attributes=True) + for item in result.changed_findings + ], + ) +# [/DEF:record_clarification_answer:Function] + + +# [DEF:update_field_semantic:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Apply one field-level semantic candidate decision or manual override with lock/provenance safeguards. +# @RELATION: [CALLS] ->[_update_semantic_field_state] +# @PRE: Session and field belong to the current owner, and request contains a candidate selection or manual override values. +# @POST: Manual overrides set manual provenance plus lock; explicit lock state prevents later silent replacement. +# @SIDE_EFFECT: Mutates field state and accepted/superseded candidate statuses in persistence. +# @DATA_CONTRACT: Input[FieldSemanticUpdateRequest] -> Output[SemanticFieldEntryDto] +@router.patch( + "/sessions/{session_id}/fields/{field_id}/semantic", + response_model=SemanticFieldEntryDto, + dependencies=[ + Depends(_require_auto_review_flag), + Depends(has_permission("dataset:session", "MANAGE")), + ], +) +async def update_field_semantic( + session_id: str, + field_id: str, + request: FieldSemanticUpdateRequest, + repository: DatasetReviewSessionRepository = Depends(_get_repository), + current_user: User = Depends(get_current_user), +): + with belief_scope("dataset_review.update_field_semantic"): + session = _get_owned_session_or_404(repository, session_id, current_user) + _require_owner_mutation_scope(session, current_user) + field = _get_owned_field_or_404(session, field_id) + _update_semantic_field_state(field, request, changed_by="user") + repository.db.commit() + repository.db.refresh(field) + _record_session_event( + repository, + session, + current_user, + event_type="semantic_field_updated", + event_summary="Semantic field decision persisted", + event_details={ + "field_id": field.field_id, + "candidate_id": request.candidate_id, + "is_locked": field.is_locked, + "source_id": field.source_id, + "source_version": field.source_version, + }, + ) + return _serialize_semantic_field(field) +# [/DEF:update_field_semantic:Function] + + +# [DEF:lock_field_semantic:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Lock one semantic field against later automatic overwrite while preserving the current active value. +# @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry] +# @PRE: Field belongs to the current owner. +# @POST: Field remains active and locked; later imports may add candidates but cannot replace the locked value implicitly. +# @SIDE_EFFECT: Mutates field lock state in persistence. +# @DATA_CONTRACT: Input[session_id:str,field_id:str] -> Output[SemanticFieldEntryDto] +@router.post( + "/sessions/{session_id}/fields/{field_id}/lock", + response_model=SemanticFieldEntryDto, + dependencies=[ + Depends(_require_auto_review_flag), + Depends(has_permission("dataset:session", "MANAGE")), + ], +) +async def lock_field_semantic( + session_id: str, + field_id: str, + repository: DatasetReviewSessionRepository = Depends(_get_repository), + current_user: User = Depends(get_current_user), +): + with belief_scope("dataset_review.lock_field_semantic"): + session = _get_owned_session_or_404(repository, session_id, current_user) + _require_owner_mutation_scope(session, current_user) + field = _get_owned_field_or_404(session, field_id) + field.is_locked = True + field.last_changed_by = "user" + repository.db.commit() + repository.db.refresh(field) + _record_session_event( + repository, + session, + current_user, + event_type="semantic_field_locked", + event_summary="Semantic field lock persisted", + event_details={"field_id": field.field_id}, + ) + return _serialize_semantic_field(field) +# [/DEF:lock_field_semantic:Function] + + +# [DEF:unlock_field_semantic:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Unlock one semantic field so later automated candidate application may replace it explicitly. +# @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry] +# @PRE: Field belongs to the current owner. +# @POST: Field becomes unlocked; manual-override provenance is downgraded to unresolved to preserve the lock/provenance invariant. +# @SIDE_EFFECT: Mutates field lock/provenance state in persistence. +# @DATA_CONTRACT: Input[session_id:str,field_id:str] -> Output[SemanticFieldEntryDto] +@router.post( + "/sessions/{session_id}/fields/{field_id}/unlock", + response_model=SemanticFieldEntryDto, + dependencies=[ + Depends(_require_auto_review_flag), + Depends(has_permission("dataset:session", "MANAGE")), + ], +) +async def unlock_field_semantic( + session_id: str, + field_id: str, + repository: DatasetReviewSessionRepository = Depends(_get_repository), + current_user: User = Depends(get_current_user), +): + with belief_scope("dataset_review.unlock_field_semantic"): + session = _get_owned_session_or_404(repository, session_id, current_user) + _require_owner_mutation_scope(session, current_user) + field = _get_owned_field_or_404(session, field_id) + field.is_locked = False + field.last_changed_by = "user" + if field.provenance == FieldProvenance.MANUAL_OVERRIDE: + field.provenance = FieldProvenance.UNRESOLVED + field.needs_review = True + repository.db.commit() + repository.db.refresh(field) + _record_session_event( + repository, + session, + current_user, + event_type="semantic_field_unlocked", + event_summary="Semantic field unlock persisted", + event_details={"field_id": field.field_id}, + ) + return _serialize_semantic_field(field) +# [/DEF:unlock_field_semantic:Function] + + +# [DEF:approve_batch_semantic_fields:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Approve multiple semantic candidate decisions in one owner-authorized batch without bypassing single-field semantics. +# @RELATION: [CALLS] ->[_update_semantic_field_state] +# @PRE: Session belongs to the current owner and each requested field/candidate pair is contained in the session aggregate. +# @POST: Returns updated semantic fields after applying the same candidate/lock invariants as the single-field endpoint. +# @SIDE_EFFECT: Persists multiple semantic field decisions in one transaction and records one explicit session audit event. +# @DATA_CONTRACT: Input[BatchApproveSemanticRequest] -> Output[List[SemanticFieldEntryDto]] +@router.post( + "/sessions/{session_id}/fields/semantic/approve-batch", + response_model=List[SemanticFieldEntryDto], + dependencies=[ + Depends(_require_auto_review_flag), + Depends(has_permission("dataset:session", "MANAGE")), + ], +) +async def approve_batch_semantic_fields( + session_id: str, + request: BatchApproveSemanticRequest, + repository: DatasetReviewSessionRepository = Depends(_get_repository), + current_user: User = Depends(get_current_user), +): + with belief_scope("dataset_review.approve_batch_semantic_fields"): + session = _get_owned_session_or_404(repository, session_id, current_user) + _require_owner_mutation_scope(session, current_user) + + updated_fields: List[SemanticFieldEntry] = [] + for item in request.items: + field = _get_owned_field_or_404(session, item.field_id) + updated_field = _update_semantic_field_state( + field, + FieldSemanticUpdateRequest(candidate_id=item.candidate_id, lock_field=item.lock_field), + changed_by="user", + ) + updated_fields.append(updated_field) + + repository.db.commit() + for field in updated_fields: + repository.db.refresh(field) + _record_session_event( + repository, + session, + current_user, + event_type="semantic_fields_batch_approved", + event_summary="Batch semantic approval persisted", + event_details={ + "field_ids": [field.field_id for field in updated_fields], + "count": len(updated_fields), + }, + ) + return [_serialize_semantic_field(field) for field in updated_fields] +# [/DEF:approve_batch_semantic_fields:Function] + + +# [DEF:list_execution_mappings:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Return the current mapping-review set for one accessible session. +# @RELATION: [CALLS] ->[_get_owned_session_or_404] +# @PRE: Session is ownership-accessible to the authenticated user and execution feature is enabled. +# @POST: Returns the persisted mapping review set for the requested session without mutating approval state. +# @SIDE_EFFECT: none. +# @DATA_CONTRACT: Input[session_id:str] -> Output[List[ExecutionMappingDto]] +@router.get( + "/sessions/{session_id}/mappings", + response_model=List[ExecutionMappingDto], + dependencies=[ + Depends(_require_auto_review_flag), + Depends(_require_execution_flag), + Depends(has_permission("dataset:session", "READ")), + ], +) +async def list_execution_mappings( + session_id: str, + repository: DatasetReviewSessionRepository = Depends(_get_repository), + current_user: User = Depends(get_current_user), +): + with belief_scope("dataset_review.list_execution_mappings"): + session = _get_owned_session_or_404(repository, session_id, current_user) + return [_serialize_execution_mapping(item) for item in session.execution_mappings] +# [/DEF:list_execution_mappings:Function] + + +# [DEF:update_execution_mapping:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Persist one owner-authorized execution-mapping effective value override and invalidate stale preview truth. +# @RELATION: [DEPENDS_ON] ->[ExecutionMapping] +# @PRE: Mapping belongs to the current owner session and request carries an explicit effective value decision. +# @POST: Mapping effective value and override metadata are persisted and any prior preview truth is marked stale for safe relaunch. +# @SIDE_EFFECT: Mutates mapping value/approval state, may mark latest preview stale, and updates session readiness cues. +# @DATA_CONTRACT: Input[UpdateExecutionMappingRequest] -> Output[ExecutionMappingDto] +@router.patch( + "/sessions/{session_id}/mappings/{mapping_id}", + response_model=ExecutionMappingDto, + dependencies=[ + Depends(_require_auto_review_flag), + Depends(_require_execution_flag), + Depends(has_permission("dataset:session", "MANAGE")), + ], +) +async def update_execution_mapping( + session_id: str, + mapping_id: str, + request: UpdateExecutionMappingRequest, + repository: DatasetReviewSessionRepository = Depends(_get_repository), + current_user: User = Depends(get_current_user), +): + with belief_scope("dataset_review.update_execution_mapping"): + session = _get_owned_session_or_404(repository, session_id, current_user) + _require_owner_mutation_scope(session, current_user) + mapping = _get_owned_mapping_or_404(session, mapping_id) + + if request.effective_value is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="effective_value is required for execution mapping updates", + ) + + mapping.effective_value = request.effective_value + mapping.mapping_method = MappingMethod(request.mapping_method or MappingMethod.MANUAL_OVERRIDE.value) + mapping.transformation_note = request.transformation_note + mapping.approval_state = ApprovalState.APPROVED + mapping.approved_by_user_id = current_user.id + mapping.approved_at = datetime.utcnow() + + session.last_activity_at = datetime.utcnow() + session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW + if session.readiness_state in { + ReadinessState.MAPPING_REVIEW_NEEDED, + ReadinessState.COMPILED_PREVIEW_READY, + ReadinessState.RUN_READY, + ReadinessState.RUN_IN_PROGRESS, + }: + session.readiness_state = ReadinessState.COMPILED_PREVIEW_READY + + for preview in session.previews: + if preview.preview_status == PreviewStatus.READY: + preview.preview_status = PreviewStatus.STALE + + repository.db.commit() + repository.db.refresh(mapping) + _record_session_event( + repository, + session, + current_user, + event_type="execution_mapping_updated", + event_summary="Execution mapping override persisted", + event_details={ + "mapping_id": mapping.mapping_id, + "approval_state": mapping.approval_state.value, + "preview_state": "stale", + }, + ) + return _serialize_execution_mapping(mapping) +# [/DEF:update_execution_mapping:Function] + + +# [DEF:approve_execution_mapping:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Explicitly approve a warning-sensitive mapping transformation and preserve audit note state. +# @RELATION: [DEPENDS_ON] ->[ExecutionMapping] +# @PRE: Mapping belongs to the current owner session and execution feature is enabled. +# @POST: Mapping approval state becomes approved and owner-scoped audit markers are updated. +# @SIDE_EFFECT: Mutates persisted mapping approval state and session readiness cues. +# @DATA_CONTRACT: Input[ApproveMappingRequest] -> Output[ExecutionMappingDto] +@router.post( + "/sessions/{session_id}/mappings/{mapping_id}/approve", + response_model=ExecutionMappingDto, + dependencies=[ + Depends(_require_auto_review_flag), + Depends(_require_execution_flag), + Depends(has_permission("dataset:session", "MANAGE")), + ], +) +async def approve_execution_mapping( + session_id: str, + mapping_id: str, + request: ApproveMappingRequest, + repository: DatasetReviewSessionRepository = Depends(_get_repository), + current_user: User = Depends(get_current_user), +): + with belief_scope("dataset_review.approve_execution_mapping"): + session = _get_owned_session_or_404(repository, session_id, current_user) + _require_owner_mutation_scope(session, current_user) + mapping = _get_owned_mapping_or_404(session, mapping_id) + mapping.approval_state = ApprovalState.APPROVED + mapping.approved_by_user_id = current_user.id + mapping.approved_at = datetime.utcnow() + if request.approval_note: + mapping.transformation_note = request.approval_note + session.last_activity_at = datetime.utcnow() + if session.readiness_state == ReadinessState.MAPPING_REVIEW_NEEDED: + session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW + repository.db.commit() + repository.db.refresh(mapping) + _record_session_event( + repository, + session, + current_user, + event_type="execution_mapping_approved", + event_summary="Execution mapping approval persisted", + event_details={ + "mapping_id": mapping.mapping_id, + "approval_state": mapping.approval_state.value, + }, + ) + return _serialize_execution_mapping(mapping) +# [/DEF:approve_execution_mapping:Function] + + +# [DEF:approve_batch_execution_mappings:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Approve multiple warning-sensitive execution mappings in one owner-authorized batch. +# @RELATION: [DEPENDS_ON] ->[ExecutionMapping] +# @PRE: Session belongs to the current owner and every requested mapping belongs to the same session aggregate. +# @POST: Returns updated mappings after applying the same approval semantics as the single mapping endpoint. +# @SIDE_EFFECT: Persists multiple approvals and records one explicit audit event. +# @DATA_CONTRACT: Input[BatchApproveMappingRequest] -> Output[List[ExecutionMappingDto]] +@router.post( + "/sessions/{session_id}/mappings/approve-batch", + response_model=List[ExecutionMappingDto], + dependencies=[ + Depends(_require_auto_review_flag), + Depends(_require_execution_flag), + Depends(has_permission("dataset:session", "MANAGE")), + ], +) +async def approve_batch_execution_mappings( + session_id: str, + request: BatchApproveMappingRequest, + repository: DatasetReviewSessionRepository = Depends(_get_repository), + current_user: User = Depends(get_current_user), +): + with belief_scope("dataset_review.approve_batch_execution_mappings"): + session = _get_owned_session_or_404(repository, session_id, current_user) + _require_owner_mutation_scope(session, current_user) + + updated_mappings: List[ExecutionMapping] = [] + for mapping_id in list(dict.fromkeys(request.mapping_ids)): + mapping = _get_owned_mapping_or_404(session, mapping_id) + mapping.approval_state = ApprovalState.APPROVED + mapping.approved_by_user_id = current_user.id + mapping.approved_at = datetime.utcnow() + if request.approval_note: + mapping.transformation_note = request.approval_note + updated_mappings.append(mapping) + + session.last_activity_at = datetime.utcnow() + if session.readiness_state == ReadinessState.MAPPING_REVIEW_NEEDED: + session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW + + repository.db.commit() + for mapping in updated_mappings: + repository.db.refresh(mapping) + _record_session_event( + repository, + session, + current_user, + event_type="execution_mappings_batch_approved", + event_summary="Batch mapping approval persisted", + event_details={ + "mapping_ids": [mapping.mapping_id for mapping in updated_mappings], + "count": len(updated_mappings), + }, + ) + return [_serialize_execution_mapping(mapping) for mapping in updated_mappings] +# [/DEF:approve_batch_execution_mappings:Function] + + +# [DEF:trigger_preview_generation:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Trigger Superset-side preview compilation for the current owned execution context. +# @RELATION: [CALLS] ->[DatasetReviewOrchestrator.prepare_launch_preview] +# @PRE: Session belongs to the current owner and required mapping inputs are available. +# @POST: Returns the persisted preview artifact produced from Superset or a deterministic conflict error. +# @SIDE_EFFECT: Persists preview attempt and updates readiness state. +# @DATA_CONTRACT: Input[session_id:str] -> Output[PreviewEnqueueResultResponse] +@router.post( + "/sessions/{session_id}/preview", + response_model=PreviewEnqueueResultResponse, + dependencies=[ + Depends(_require_auto_review_flag), + Depends(_require_execution_flag), + Depends(has_permission("dataset:session", "MANAGE")), + ], +) +async def trigger_preview_generation( + session_id: str, + orchestrator: DatasetReviewOrchestrator = Depends(_get_orchestrator), + current_user: User = Depends(get_current_user), +): + with belief_scope("dataset_review.trigger_preview_generation"): + try: + result = orchestrator.prepare_launch_preview( + PreparePreviewCommand( + user=current_user, + session_id=session_id, + ) + ) + except ValueError as exc: + detail = str(exc) + status_code = ( + status.HTTP_404_NOT_FOUND if detail in {"Session not found", "Environment not found"} + else status.HTTP_409_CONFLICT if detail.startswith("Preview blocked:") + else status.HTTP_400_BAD_REQUEST + ) + raise HTTPException(status_code=status_code, detail=detail) from exc + + return PreviewEnqueueResultResponse( + session_id=result.session.session_id, + preview_status=result.preview.preview_status.value, + task_id=None, + preview=CompiledPreviewDto.model_validate(result.preview, from_attributes=True), + ) +# [/DEF:trigger_preview_generation:Function] + + +# [DEF:launch_dataset:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Execute the current owned session launch handoff through the orchestrator and return audited SQL Lab run context. +# @RELATION: [CALLS] ->[DatasetReviewOrchestrator.launch_dataset] +# @PRE: Session belongs to the current owner, execution feature is enabled, and launch gates are satisfied or a deterministic conflict is returned. +# @POST: Returns updated session summary plus persisted run context when launch handoff is accepted. +# @SIDE_EFFECT: Persists launch audit snapshot and may trigger SQL Lab session creation. +# @DATA_CONTRACT: Input[session_id:str] -> Output[LaunchDatasetResponse] +@router.post( + "/sessions/{session_id}/launch", + response_model=LaunchDatasetResponse, + dependencies=[ + Depends(_require_auto_review_flag), + Depends(_require_execution_flag), + Depends(has_permission("dataset:session", "MANAGE")), + ], +) +async def launch_dataset( + session_id: str, + orchestrator: DatasetReviewOrchestrator = Depends(_get_orchestrator), + current_user: User = Depends(get_current_user), +): + with belief_scope("dataset_review.launch_dataset"): + try: + result = orchestrator.launch_dataset( + LaunchDatasetCommand( + user=current_user, + session_id=session_id, + ) + ) + except ValueError as exc: + detail = str(exc) + status_code = ( + status.HTTP_404_NOT_FOUND + if detail in {"Session not found", "Environment not found"} + else status.HTTP_409_CONFLICT + if detail.startswith("Launch blocked:") + else status.HTTP_400_BAD_REQUEST + ) + raise HTTPException(status_code=status_code, detail=detail) from exc + + return LaunchDatasetResponse( + session=_serialize_session_summary(result.session), + run_context=_serialize_run_context(result.run_context), + ) +# [/DEF:launch_dataset:Function] + + +# [DEF:record_field_feedback:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Persist thumbs up/down feedback for AI-assisted semantic field content. +# @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry] +# @PRE: Field belongs to the current owner and feedback value is valid. +# @POST: Field feedback is stored without altering lock or active semantic value. +# @SIDE_EFFECT: Updates one persisted semantic field feedback marker. +# @DATA_CONTRACT: Input[FeedbackRequest] -> Output[FeedbackResponse] +@router.post( + "/sessions/{session_id}/fields/{field_id}/feedback", + response_model=FeedbackResponse, + dependencies=[ + Depends(_require_auto_review_flag), + Depends(has_permission("dataset:session", "MANAGE")), + ], +) +async def record_field_feedback( + session_id: str, + field_id: str, + request: FeedbackRequest, + repository: DatasetReviewSessionRepository = Depends(_get_repository), + current_user: User = Depends(get_current_user), +): + with belief_scope("dataset_review.record_field_feedback"): + session = _get_owned_session_or_404(repository, session_id, current_user) + _require_owner_mutation_scope(session, current_user) + field = _get_owned_field_or_404(session, field_id) + field.user_feedback = request.feedback + repository.db.commit() + _record_session_event( + repository, + session, + current_user, + event_type="semantic_field_feedback_recorded", + event_summary="Semantic field feedback persisted", + event_details={"field_id": field.field_id, "feedback": request.feedback}, + ) + return FeedbackResponse(target_id=field.field_id, feedback=request.feedback) +# [/DEF:record_field_feedback:Function] + + +# [DEF:record_clarification_feedback:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Persist thumbs up/down feedback for clarification question/answer content used in guided review. +# @RELATION: [DEPENDS_ON] ->[ClarificationAnswer] +# @PRE: Clarification question belongs to the current owner session and already has a persisted answer. +# @POST: Feedback is stored on the clarification answer audit record. +# @SIDE_EFFECT: Updates one clarification answer feedback marker in persistence. +# @DATA_CONTRACT: Input[FeedbackRequest] -> Output[FeedbackResponse] +@router.post( + "/sessions/{session_id}/clarification/questions/{question_id}/feedback", + response_model=FeedbackResponse, + dependencies=[ + Depends(_require_auto_review_flag), + Depends(_require_clarification_flag), + Depends(has_permission("dataset:session", "MANAGE")), + ], +) +async def record_clarification_feedback( + session_id: str, + question_id: str, + request: FeedbackRequest, + repository: DatasetReviewSessionRepository = Depends(_get_repository), + current_user: User = Depends(get_current_user), +): + with belief_scope("dataset_review.record_clarification_feedback"): + session = _get_owned_session_or_404(repository, session_id, current_user) + _require_owner_mutation_scope(session, current_user) + clarification_session = _get_latest_clarification_session_or_404(session) + question = next( + (item for item in clarification_session.questions if item.question_id == question_id), + None, + ) + if question is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Clarification question not found") + if question.answer is None: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Clarification answer not found") + question.answer.user_feedback = request.feedback + repository.db.commit() + _record_session_event( + repository, + session, + current_user, + event_type="clarification_feedback_recorded", + event_summary="Clarification feedback persisted", + event_details={"question_id": question.question_id, "feedback": request.feedback}, + ) + return FeedbackResponse(target_id=question.question_id, feedback=request.feedback) +# [/DEF:record_clarification_feedback:Function] + # [/DEF:DatasetReviewApi:Module] \ No newline at end of file diff --git a/backend/src/core/utils/superset_compilation_adapter.py b/backend/src/core/utils/superset_compilation_adapter.py new file mode 100644 index 00000000..78aeaf7b --- /dev/null +++ b/backend/src/core/utils/superset_compilation_adapter.py @@ -0,0 +1,406 @@ +# [DEF:SupersetCompilationAdapter:Module] +# @COMPLEXITY: 4 +# @SEMANTICS: dataset_review, superset, compilation_preview, sql_lab_launch, execution_truth +# @PURPOSE: Interact with Superset preview compilation and SQL Lab execution endpoints using the current approved execution context. +# @LAYER: Infra +# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient] +# @RELATION: [DEPENDS_ON] ->[CompiledPreview] +# @RELATION: [DEPENDS_ON] ->[DatasetRunContext] +# @PRE: effective template params and dataset execution reference are available. +# @POST: preview and launch calls return Superset-originated artifacts or explicit errors. +# @SIDE_EFFECT: performs upstream Superset preview and SQL Lab calls. +# @INVARIANT: The adapter never fabricates compiled SQL locally; preview truth is delegated to Superset only. + +from __future__ import annotations + +# [DEF:SupersetCompilationAdapter.imports:Block] +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Dict, List, Optional + +from src.core.config_models import Environment +from src.core.logger import belief_scope, logger +from src.core.superset_client import SupersetClient +from src.models.dataset_review import CompiledPreview, PreviewStatus +# [/DEF:SupersetCompilationAdapter.imports:Block] + + +# [DEF:PreviewCompilationPayload:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Typed preview payload for Superset-side compilation. +@dataclass(frozen=True) +class PreviewCompilationPayload: + session_id: str + dataset_id: int + preview_fingerprint: str + template_params: Dict[str, Any] + effective_filters: List[Dict[str, Any]] +# [/DEF:PreviewCompilationPayload:Class] + + +# [DEF:SqlLabLaunchPayload:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Typed SQL Lab payload for audited launch handoff. +@dataclass(frozen=True) +class SqlLabLaunchPayload: + session_id: str + dataset_id: int + preview_id: str + compiled_sql: str + template_params: Dict[str, Any] +# [/DEF:SqlLabLaunchPayload:Class] + + +# [DEF:SupersetCompilationAdapter:Class] +# @COMPLEXITY: 4 +# @PURPOSE: Delegate preview compilation and SQL Lab launch to Superset without local SQL fabrication. +# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient] +# @PRE: environment is configured and Superset is reachable for the target session. +# @POST: adapter can return explicit ready/failed preview artifacts and canonical SQL Lab references. +# @SIDE_EFFECT: issues network requests to Superset API surfaces. +class SupersetCompilationAdapter: + # [DEF:SupersetCompilationAdapter.__init__:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Bind adapter to one Superset environment and client instance. + def __init__(self, environment: Environment, client: Optional[SupersetClient] = None) -> None: + self.environment = environment + self.client = client or SupersetClient(environment) + # [/DEF:SupersetCompilationAdapter.__init__:Function] + + # [DEF:SupersetCompilationAdapter.compile_preview:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Request Superset-side compiled SQL preview for the current effective inputs. + # @RELATION: [CALLS] ->[SupersetCompilationAdapter._request_superset_preview] + # @PRE: dataset_id and effective inputs are available for the current session. + # @POST: returns a ready or failed preview artifact backed only by Superset-originated SQL or diagnostics. + # @SIDE_EFFECT: performs upstream preview requests. + # @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[CompiledPreview] + def compile_preview(self, payload: PreviewCompilationPayload) -> CompiledPreview: + with belief_scope("SupersetCompilationAdapter.compile_preview"): + if payload.dataset_id <= 0: + logger.explore( + "Preview compilation rejected because dataset identifier is invalid", + extra={"dataset_id": payload.dataset_id, "session_id": payload.session_id}, + ) + raise ValueError("dataset_id must be a positive integer") + + logger.reason( + "Requesting Superset-generated SQL preview", + extra={ + "session_id": payload.session_id, + "dataset_id": payload.dataset_id, + "template_param_count": len(payload.template_params), + "filter_count": len(payload.effective_filters), + }, + ) + + try: + preview_result = self._request_superset_preview(payload) + except Exception as exc: + logger.explore( + "Superset preview compilation failed with explicit upstream error", + extra={ + "session_id": payload.session_id, + "dataset_id": payload.dataset_id, + "error": str(exc), + }, + ) + return CompiledPreview( + session_id=payload.session_id, + preview_status=PreviewStatus.FAILED, + compiled_sql=None, + preview_fingerprint=payload.preview_fingerprint, + compiled_by="superset", + error_code="superset_preview_failed", + error_details=str(exc), + compiled_at=None, + ) + + compiled_sql = str(preview_result.get("compiled_sql") or "").strip() + if not compiled_sql: + logger.explore( + "Superset preview response did not include compiled SQL", + extra={ + "session_id": payload.session_id, + "dataset_id": payload.dataset_id, + "response_keys": sorted(preview_result.keys()), + }, + ) + return CompiledPreview( + session_id=payload.session_id, + preview_status=PreviewStatus.FAILED, + compiled_sql=None, + preview_fingerprint=payload.preview_fingerprint, + compiled_by="superset", + error_code="superset_preview_empty", + error_details="Superset preview response did not include compiled SQL", + compiled_at=None, + ) + + preview = CompiledPreview( + session_id=payload.session_id, + preview_status=PreviewStatus.READY, + compiled_sql=compiled_sql, + preview_fingerprint=payload.preview_fingerprint, + compiled_by="superset", + error_code=None, + error_details=None, + compiled_at=datetime.utcnow(), + ) + logger.reflect( + "Superset-generated SQL preview captured successfully", + extra={ + "session_id": payload.session_id, + "dataset_id": payload.dataset_id, + "compiled_sql_length": len(compiled_sql), + }, + ) + return preview + # [/DEF:SupersetCompilationAdapter.compile_preview:Function] + + # [DEF:SupersetCompilationAdapter.mark_preview_stale:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Invalidate previous preview after mapping or value changes. + # @PRE: preview is a persisted preview artifact or current in-memory snapshot. + # @POST: preview status becomes stale without fabricating a replacement artifact. + def mark_preview_stale(self, preview: CompiledPreview) -> CompiledPreview: + preview.preview_status = PreviewStatus.STALE + return preview + # [/DEF:SupersetCompilationAdapter.mark_preview_stale:Function] + + # [DEF:SupersetCompilationAdapter.create_sql_lab_session:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Create the canonical audited execution session after all launch gates pass. + # @RELATION: [CALLS] ->[SupersetCompilationAdapter._request_sql_lab_session] + # @PRE: compiled_sql is Superset-originated and launch gates are already satisfied. + # @POST: returns one canonical SQL Lab session reference from Superset. + # @SIDE_EFFECT: performs upstream SQL Lab execution/session creation. + # @DATA_CONTRACT: Input[SqlLabLaunchPayload] -> Output[str] + def create_sql_lab_session(self, payload: SqlLabLaunchPayload) -> str: + with belief_scope("SupersetCompilationAdapter.create_sql_lab_session"): + compiled_sql = str(payload.compiled_sql or "").strip() + if not compiled_sql: + logger.explore( + "SQL Lab launch rejected because compiled SQL is empty", + extra={"session_id": payload.session_id, "preview_id": payload.preview_id}, + ) + raise ValueError("compiled_sql must be non-empty") + + logger.reason( + "Creating SQL Lab execution session from Superset-originated preview", + extra={ + "session_id": payload.session_id, + "dataset_id": payload.dataset_id, + "preview_id": payload.preview_id, + }, + ) + result = self._request_sql_lab_session(payload) + sql_lab_session_ref = str( + result.get("sql_lab_session_ref") + or result.get("query_id") + or result.get("id") + or result.get("result", {}).get("id") + or "" + ).strip() + if not sql_lab_session_ref: + logger.explore( + "Superset SQL Lab launch response did not include a stable session reference", + extra={"session_id": payload.session_id, "preview_id": payload.preview_id}, + ) + raise RuntimeError("Superset SQL Lab launch response did not include a session reference") + + logger.reflect( + "Canonical SQL Lab session created successfully", + extra={ + "session_id": payload.session_id, + "preview_id": payload.preview_id, + "sql_lab_session_ref": sql_lab_session_ref, + }, + ) + return sql_lab_session_ref + # [/DEF:SupersetCompilationAdapter.create_sql_lab_session:Function] + + # [DEF:SupersetCompilationAdapter._request_superset_preview:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Probe supported Superset preview surfaces and return the first explicit compilation response. + # @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient] + # @PRE: payload contains a valid dataset identifier and deterministic execution inputs for one preview attempt. + # @POST: returns the first upstream response that exposes compiled SQL without fabricating local SQL. + # @SIDE_EFFECT: issues one or more Superset preview requests until a supported surface responds. + # @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[Dict[str,Any]] + def _request_superset_preview(self, payload: PreviewCompilationPayload) -> Dict[str, Any]: + request_payload = { + "dataset_id": payload.dataset_id, + "template_params": payload.template_params, + "effective_filters": payload.effective_filters, + "session_id": payload.session_id, + } + candidate_calls = self._build_preview_call_candidates(payload.dataset_id, request_payload) + errors: List[str] = [] + + for candidate in candidate_calls: + call_kind = candidate["kind"] + target = candidate["target"] + try: + logger.reason( + "Attempting Superset preview compilation candidate", + extra={"kind": call_kind, "target": target}, + ) + if call_kind == "client_method": + method = getattr(self.client, target) + response = method(request_payload) + else: + response = self.client.network.request( + method=candidate["http_method"], + endpoint=target, + data=candidate["data"], + headers={"Content-Type": "application/json"}, + ) + normalized = self._normalize_preview_response(response) + if normalized is not None: + return normalized + except Exception as exc: + errors.append(f"{call_kind}:{target}:{exc}") + logger.explore( + "Superset preview compilation candidate failed", + extra={"kind": call_kind, "target": target, "error": str(exc)}, + ) + + raise RuntimeError("; ".join(errors) or "No Superset preview surface accepted the request") + # [/DEF:SupersetCompilationAdapter._request_superset_preview:Function] + + # [DEF:SupersetCompilationAdapter._request_sql_lab_session:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Probe supported SQL Lab execution surfaces and return the first successful response. + # @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient] + # @PRE: payload carries non-empty Superset-originated SQL and a preview identifier for the current launch. + # @POST: returns the first successful SQL Lab execution response from Superset. + # @SIDE_EFFECT: issues Superset dataset lookup and SQL Lab execution requests. + # @DATA_CONTRACT: Input[SqlLabLaunchPayload] -> Output[Dict[str,Any]] + def _request_sql_lab_session(self, payload: SqlLabLaunchPayload) -> Dict[str, Any]: + dataset_raw = self.client.get_dataset(payload.dataset_id) + dataset_record = dataset_raw.get("result", dataset_raw) if isinstance(dataset_raw, dict) else {} + database_id = dataset_record.get("database", {}).get("id") if isinstance(dataset_record.get("database"), dict) else dataset_record.get("database_id") + if database_id is None: + raise RuntimeError("Superset dataset does not expose a database identifier for SQL Lab launch") + + request_payload = { + "database_id": database_id, + "sql": payload.compiled_sql, + "templateParams": payload.template_params, + "schema": dataset_record.get("schema"), + "client_id": payload.preview_id, + } + candidate_calls = [ + {"kind": "network", "target": "/sqllab/execute/", "http_method": "POST"}, + {"kind": "network", "target": "/sql_lab/execute/", "http_method": "POST"}, + ] + errors: List[str] = [] + + for candidate in candidate_calls: + try: + response = self.client.network.request( + method=candidate["http_method"], + endpoint=candidate["target"], + data=self._dump_json(request_payload), + headers={"Content-Type": "application/json"}, + ) + if isinstance(response, dict) and response: + return response + except Exception as exc: + errors.append(f"{candidate['target']}:{exc}") + logger.explore( + "Superset SQL Lab candidate failed", + extra={"target": candidate["target"], "error": str(exc)}, + ) + + raise RuntimeError("; ".join(errors) or "No Superset SQL Lab surface accepted the request") + # [/DEF:SupersetCompilationAdapter._request_sql_lab_session:Function] + + # [DEF:SupersetCompilationAdapter._build_preview_call_candidates:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Assemble preview candidate call shapes in priority order. + def _build_preview_call_candidates( + self, + dataset_id: int, + request_payload: Dict[str, Any], + ) -> List[Dict[str, Any]]: + candidates: List[Dict[str, Any]] = [] + for method_name in ( + "compile_sql_preview", + "compile_preview", + "get_compiled_sql_preview", + ): + if hasattr(self.client, method_name): + candidates.append({"kind": "client_method", "target": method_name}) + + encoded_payload = self._dump_json(request_payload) + candidates.extend( + [ + { + "kind": "network", + "target": f"/dataset/{dataset_id}/preview", + "http_method": "POST", + "data": encoded_payload, + }, + { + "kind": "network", + "target": f"/dataset/{dataset_id}/sql", + "http_method": "POST", + "data": encoded_payload, + }, + { + "kind": "network", + "target": "/sqllab/format_sql/", + "http_method": "POST", + "data": encoded_payload, + }, + ] + ) + return candidates + # [/DEF:SupersetCompilationAdapter._build_preview_call_candidates:Function] + + # [DEF:SupersetCompilationAdapter._normalize_preview_response:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Normalize candidate Superset preview responses into one compiled-sql structure. + # @RELATION: [DEPENDS_ON] ->[CompiledPreview] + def _normalize_preview_response(self, response: Any) -> Optional[Dict[str, Any]]: + if not isinstance(response, dict): + return None + + compiled_sql_candidates = [ + response.get("compiled_sql"), + response.get("sql"), + response.get("query"), + ] + result_payload = response.get("result") + if isinstance(result_payload, dict): + compiled_sql_candidates.extend( + [ + result_payload.get("compiled_sql"), + result_payload.get("sql"), + result_payload.get("query"), + ] + ) + + for candidate in compiled_sql_candidates: + compiled_sql = str(candidate or "").strip() + if compiled_sql: + return { + "compiled_sql": compiled_sql, + "raw_response": response, + } + return None + # [/DEF:SupersetCompilationAdapter._normalize_preview_response:Function] + + # [DEF:SupersetCompilationAdapter._dump_json:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Serialize Superset request payload deterministically for network transport. + def _dump_json(self, payload: Dict[str, Any]) -> str: + import json + + return json.dumps(payload, sort_keys=True, default=str) + # [/DEF:SupersetCompilationAdapter._dump_json:Function] +# [/DEF:SupersetCompilationAdapter:Class] + +# [/DEF:SupersetCompilationAdapter:Module] \ No newline at end of file diff --git a/backend/src/core/utils/superset_context_extractor.py b/backend/src/core/utils/superset_context_extractor.py index f0991bbb..9f69062b 100644 --- a/backend/src/core/utils/superset_context_extractor.py +++ b/backend/src/core/utils/superset_context_extractor.py @@ -15,8 +15,9 @@ from __future__ import annotations # [DEF:SupersetContextExtractor.imports:Block] import json +import re from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Set from urllib.parse import parse_qs, unquote, urlparse from src.core.config_models import Environment @@ -204,17 +205,224 @@ class SupersetContextExtractor: # [/DEF:SupersetContextExtractor.parse_superset_link:Function] # [DEF:SupersetContextExtractor.recover_imported_filters:Function] - # @COMPLEXITY: 2 + # @COMPLEXITY: 4 # @PURPOSE: Build imported filter entries from URL state and Superset-side saved context. + # @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient] + # @PRE: parsed_context comes from a successful Superset link parse for one environment. + # @POST: returns explicit recovered and partial filter entries with preserved provenance and confirmation requirements. + # @SIDE_EFFECT: may issue Superset reads for dashboard metadata enrichment. + # @DATA_CONTRACT: Input[SupersetParsedContext] -> Output[List[Dict[str,Any]]] def recover_imported_filters(self, parsed_context: SupersetParsedContext) -> List[Dict[str, Any]]: - return list(parsed_context.imported_filters) + with belief_scope("SupersetContextExtractor.recover_imported_filters"): + recovered_filters: List[Dict[str, Any]] = [] + seen_filter_keys: Set[str] = set() + + for item in parsed_context.imported_filters: + normalized = self._normalize_imported_filter_payload( + item, + default_source="superset_url", + default_note="Recovered from Superset URL state", + ) + filter_key = normalized["filter_name"].strip().lower() + if filter_key in seen_filter_keys: + continue + seen_filter_keys.add(filter_key) + recovered_filters.append(normalized) + + if parsed_context.dashboard_id is None: + logger.reflect( + "Imported filter recovery completed without dashboard enrichment", + extra={ + "dashboard_id": None, + "filter_count": len(recovered_filters), + "partial_recovery": parsed_context.partial_recovery, + }, + ) + return recovered_filters + + try: + dashboard_payload = self.client.get_dashboard(parsed_context.dashboard_id) + dashboard_record = ( + dashboard_payload.get("result", dashboard_payload) + if isinstance(dashboard_payload, dict) + else {} + ) + json_metadata = dashboard_record.get("json_metadata") + if isinstance(json_metadata, str) and json_metadata.strip(): + json_metadata = json.loads(json_metadata) + if not isinstance(json_metadata, dict): + json_metadata = {} + + native_filter_configuration = json_metadata.get("native_filter_configuration") or [] + default_filters = json_metadata.get("default_filters") or {} + if isinstance(default_filters, str) and default_filters.strip(): + try: + default_filters = json.loads(default_filters) + except Exception: + logger.explore( + "Superset default_filters payload was not valid JSON", + extra={"dashboard_id": parsed_context.dashboard_id}, + ) + default_filters = {} + + for item in native_filter_configuration: + if not isinstance(item, dict): + continue + filter_name = str( + item.get("name") + or item.get("filter_name") + or item.get("column") + or "" + ).strip() + if not filter_name: + continue + + filter_key = filter_name.lower() + if filter_key in seen_filter_keys: + continue + + default_value = None + if isinstance(default_filters, dict): + default_value = default_filters.get(filter_name) + + saved_filter = self._normalize_imported_filter_payload( + { + "filter_name": filter_name, + "display_name": item.get("label") or item.get("name"), + "raw_value": default_value, + "source": "superset_native", + "recovery_status": "recovered" if default_value is not None else "partial", + "requires_confirmation": default_value is None, + "notes": "Recovered from Superset dashboard native filter configuration", + }, + default_source="superset_native", + default_note="Recovered from Superset dashboard native filter configuration", + ) + seen_filter_keys.add(filter_key) + recovered_filters.append(saved_filter) + + logger.reflect( + "Imported filter recovery completed with dashboard enrichment", + extra={ + "dashboard_id": parsed_context.dashboard_id, + "filter_count": len(recovered_filters), + "partial_entries": len( + [ + item + for item in recovered_filters + if item["recovery_status"] == "partial" + ] + ), + }, + ) + return recovered_filters + except Exception as exc: + logger.explore( + "Dashboard native filter enrichment failed; preserving partial imported filters", + extra={ + "dashboard_id": parsed_context.dashboard_id, + "error": str(exc), + "filter_count": len(recovered_filters), + }, + ) + if not recovered_filters: + recovered_filters.append( + self._normalize_imported_filter_payload( + { + "filter_name": f"dashboard_{parsed_context.dashboard_id}_filters", + "display_name": "Dashboard native filters", + "raw_value": None, + "source": "superset_native", + "recovery_status": "partial", + "requires_confirmation": True, + "notes": "Superset dashboard filter configuration could not be recovered fully", + }, + default_source="superset_native", + default_note="Superset dashboard filter configuration could not be recovered fully", + ) + ) + return recovered_filters # [/DEF:SupersetContextExtractor.recover_imported_filters:Function] # [DEF:SupersetContextExtractor.discover_template_variables:Function] - # @COMPLEXITY: 2 + # @COMPLEXITY: 4 # @PURPOSE: Detect runtime variables and Jinja references from dataset query-bearing fields. + # @RELATION: [DEPENDS_ON] ->[TemplateVariable] + # @PRE: dataset_payload is a Superset dataset-detail style payload with query-bearing fields when available. + # @POST: returns deduplicated explicit variable records without executing Jinja or fabricating runtime values. + # @SIDE_EFFECT: none. + # @DATA_CONTRACT: Input[dataset_payload:Dict[str,Any]] -> Output[List[Dict[str,Any]]] def discover_template_variables(self, dataset_payload: Dict[str, Any]) -> List[Dict[str, Any]]: - return [] + with belief_scope("SupersetContextExtractor.discover_template_variables"): + discovered: List[Dict[str, Any]] = [] + seen_variable_names: Set[str] = set() + + for expression_source in self._collect_query_bearing_expressions(dataset_payload): + for filter_match in re.finditer( + r"filter_values\(\s*['\"]([^'\"]+)['\"]\s*\)", + expression_source, + flags=re.IGNORECASE, + ): + variable_name = str(filter_match.group(1) or "").strip() + if not variable_name: + continue + self._append_template_variable( + discovered=discovered, + seen_variable_names=seen_variable_names, + variable_name=variable_name, + expression_source=expression_source, + variable_kind="native_filter", + is_required=True, + default_value=None, + ) + + for url_param_match in re.finditer( + r"url_param\(\s*['\"]([^'\"]+)['\"]\s*(?:,\s*([^)]+))?\)", + expression_source, + flags=re.IGNORECASE, + ): + variable_name = str(url_param_match.group(1) or "").strip() + if not variable_name: + continue + default_literal = url_param_match.group(2) + self._append_template_variable( + discovered=discovered, + seen_variable_names=seen_variable_names, + variable_name=variable_name, + expression_source=expression_source, + variable_kind="parameter", + is_required=default_literal is None, + default_value=self._normalize_default_literal(default_literal), + ) + + for jinja_match in re.finditer(r"\{\{\s*(.*?)\s*\}\}", expression_source, flags=re.DOTALL): + expression = str(jinja_match.group(1) or "").strip() + if not expression: + continue + if any(token in expression for token in ("filter_values(", "url_param(", "get_filters(")): + continue + variable_name = self._extract_primary_jinja_identifier(expression) + if not variable_name: + continue + self._append_template_variable( + discovered=discovered, + seen_variable_names=seen_variable_names, + variable_name=variable_name, + expression_source=expression_source, + variable_kind="derived" if "." in expression or "|" in expression else "parameter", + is_required=True, + default_value=None, + ) + + logger.reflect( + "Template variable discovery completed deterministically", + extra={ + "dataset_id": dataset_payload.get("id"), + "variable_count": len(discovered), + "variable_names": [item["variable_name"] for item in discovered], + }, + ) + return discovered # [/DEF:SupersetContextExtractor.discover_template_variables:Function] # [DEF:SupersetContextExtractor.build_recovery_summary:Function] @@ -329,6 +537,151 @@ class SupersetContextExtractor: return imported_filters # [/DEF:SupersetContextExtractor._extract_imported_filters:Function] + + # [DEF:SupersetContextExtractor._normalize_imported_filter_payload:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Normalize one imported-filter payload with explicit provenance and confirmation state. + def _normalize_imported_filter_payload( + self, + payload: Dict[str, Any], + default_source: str, + default_note: str, + ) -> Dict[str, Any]: + raw_value = payload.get("raw_value") + if "raw_value" not in payload and "value" in payload: + raw_value = payload.get("value") + + recovery_status = str( + payload.get("recovery_status") + or ("recovered" if raw_value is not None else "partial") + ).strip().lower() + requires_confirmation = bool( + payload.get("requires_confirmation", raw_value is None or recovery_status != "recovered") + ) + return { + "filter_name": str(payload.get("filter_name") or "unresolved_filter").strip(), + "display_name": payload.get("display_name"), + "raw_value": raw_value, + "normalized_value": payload.get("normalized_value"), + "source": str(payload.get("source") or default_source), + "confidence_state": "imported" if raw_value is not None else "unresolved", + "requires_confirmation": requires_confirmation, + "recovery_status": recovery_status, + "notes": str(payload.get("notes") or default_note), + } + # [/DEF:SupersetContextExtractor._normalize_imported_filter_payload:Function] + + # [DEF:SupersetContextExtractor._collect_query_bearing_expressions:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Collect SQL and expression-bearing dataset fields for deterministic template-variable discovery. + # @RELATION: [DEPENDS_ON] ->[SupersetContextExtractor.discover_template_variables] + def _collect_query_bearing_expressions(self, dataset_payload: Dict[str, Any]) -> List[str]: + expressions: List[str] = [] + + def append_expression(candidate: Any) -> None: + if not isinstance(candidate, str): + return + normalized = candidate.strip() + if normalized: + expressions.append(normalized) + + append_expression(dataset_payload.get("sql")) + append_expression(dataset_payload.get("query")) + append_expression(dataset_payload.get("template_sql")) + + metrics_payload = dataset_payload.get("metrics") or [] + if isinstance(metrics_payload, list): + for metric in metrics_payload: + if isinstance(metric, str): + append_expression(metric) + continue + if not isinstance(metric, dict): + continue + append_expression(metric.get("expression")) + append_expression(metric.get("sqlExpression")) + append_expression(metric.get("metric_name")) + + columns_payload = dataset_payload.get("columns") or [] + if isinstance(columns_payload, list): + for column in columns_payload: + if not isinstance(column, dict): + continue + append_expression(column.get("sqlExpression")) + append_expression(column.get("expression")) + + return expressions + # [/DEF:SupersetContextExtractor._collect_query_bearing_expressions:Function] + + # [DEF:SupersetContextExtractor._append_template_variable:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Append one deduplicated template-variable descriptor. + def _append_template_variable( + self, + discovered: List[Dict[str, Any]], + seen_variable_names: Set[str], + variable_name: str, + expression_source: str, + variable_kind: str, + is_required: bool, + default_value: Any, + ) -> None: + normalized_name = str(variable_name or "").strip() + if not normalized_name: + return + seen_key = normalized_name.lower() + if seen_key in seen_variable_names: + return + seen_variable_names.add(seen_key) + discovered.append( + { + "variable_name": normalized_name, + "expression_source": expression_source, + "variable_kind": variable_kind, + "is_required": is_required, + "default_value": default_value, + "mapping_status": "unmapped", + } + ) + # [/DEF:SupersetContextExtractor._append_template_variable:Function] + + # [DEF:SupersetContextExtractor._extract_primary_jinja_identifier:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Extract a deterministic primary identifier from a Jinja expression without executing it. + def _extract_primary_jinja_identifier(self, expression: str) -> Optional[str]: + matched = re.match(r"([A-Za-z_][A-Za-z0-9_]*)", expression.strip()) + if matched is None: + return None + candidate = matched.group(1) + if candidate in {"if", "else", "for", "set", "True", "False", "none", "None"}: + return None + return candidate + # [/DEF:SupersetContextExtractor._extract_primary_jinja_identifier:Function] + + # [DEF:SupersetContextExtractor._normalize_default_literal:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Normalize literal default fragments from template helper calls into JSON-safe values. + def _normalize_default_literal(self, literal: Optional[str]) -> Any: + normalized_literal = str(literal or "").strip() + if not normalized_literal: + return None + if ( + (normalized_literal.startswith("'") and normalized_literal.endswith("'")) + or (normalized_literal.startswith('"') and normalized_literal.endswith('"')) + ): + return normalized_literal[1:-1] + lowered = normalized_literal.lower() + if lowered in {"true", "false"}: + return lowered == "true" + if lowered in {"null", "none"}: + return None + try: + return int(normalized_literal) + except ValueError: + try: + return float(normalized_literal) + except ValueError: + return normalized_literal + # [/DEF:SupersetContextExtractor._normalize_default_literal:Function] # [/DEF:SupersetContextExtractor:Class] # [/DEF:SupersetContextExtractor:Module] \ No newline at end of file diff --git a/backend/src/models/dataset_review.py b/backend/src/models/dataset_review.py index 5504d059..24201b7d 100644 --- a/backend/src/models/dataset_review.py +++ b/backend/src/models/dataset_review.py @@ -132,6 +132,7 @@ class DatasetReviewSession(Base): previews = relationship("CompiledPreview", back_populates="session", cascade="all, delete-orphan") run_contexts = relationship("DatasetRunContext", back_populates="session", cascade="all, delete-orphan") export_artifacts = relationship("ExportArtifact", back_populates="session", cascade="all, delete-orphan") + events = relationship("SessionEvent", back_populates="session", cascade="all, delete-orphan") # [/DEF:DatasetReviewSession:Class] # [DEF:BusinessSummarySource:Class] @@ -303,6 +304,7 @@ class SemanticFieldEntry(Base): display_format = Column(String, nullable=True) provenance = Column(SQLEnum(FieldProvenance), nullable=False, default=FieldProvenance.UNRESOLVED) source_id = Column(String, nullable=True) + source_version = Column(String, nullable=True) confidence_rank = Column(Integer, nullable=True) is_locked = Column(Boolean, nullable=False, default=False) has_conflict = Column(Boolean, nullable=False, default=False) @@ -567,6 +569,7 @@ class ClarificationAnswer(Base): answer_value = Column(Text, nullable=True) answered_by_user_id = Column(String, nullable=False) impact_summary = Column(Text, nullable=True) + user_feedback = Column(String, nullable=True) # up, down, null created_at = Column(DateTime, default=datetime.utcnow, nullable=False) question = relationship("ClarificationQuestion", back_populates="answer") @@ -627,6 +630,24 @@ class DatasetRunContext(Base): session = relationship("DatasetReviewSession", back_populates="run_contexts") # [/DEF:DatasetRunContext:Class] +# [DEF:SessionEvent:Class] +class SessionEvent(Base): + __tablename__ = "session_events" + + session_event_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + session_id = Column(String, ForeignKey("dataset_review_sessions.session_id"), nullable=False) + actor_user_id = Column(String, ForeignKey("users.id"), nullable=False) + event_type = Column(String, nullable=False) + event_summary = Column(Text, nullable=False) + current_phase = Column(String, nullable=True) + readiness_state = Column(String, nullable=True) + event_details = Column(JSON, nullable=False, default=dict) + created_at = Column(DateTime, default=datetime.utcnow, nullable=False) + + session = relationship("DatasetReviewSession", back_populates="events") + actor = relationship("User") +# [/DEF:SessionEvent:Class] + # [DEF:ArtifactType:Class] class ArtifactType(str, enum.Enum): DOCUMENTATION = "documentation" diff --git a/backend/src/schemas/dataset_review.py b/backend/src/schemas/dataset_review.py index ec83aca5..239fe18f 100644 --- a/backend/src/schemas/dataset_review.py +++ b/backend/src/schemas/dataset_review.py @@ -145,6 +145,7 @@ class SemanticFieldEntryDto(BaseModel): display_format: Optional[str] = None provenance: FieldProvenance source_id: Optional[str] = None + source_version: Optional[str] = None confidence_rank: Optional[int] = None is_locked: bool has_conflict: bool @@ -239,6 +240,7 @@ class ClarificationAnswerDto(BaseModel): answer_value: Optional[str] = None answered_by_user_id: str impact_summary: Optional[str] = None + user_feedback: Optional[str] = None created_at: datetime class Config: diff --git a/backend/src/services/dataset_review/clarification_engine.py b/backend/src/services/dataset_review/clarification_engine.py new file mode 100644 index 00000000..39f6de64 --- /dev/null +++ b/backend/src/services/dataset_review/clarification_engine.py @@ -0,0 +1,552 @@ +# [DEF:ClarificationEngine:Module] +# @COMPLEXITY: 4 +# @SEMANTICS: dataset_review, clarification, question_payload, answer_persistence, readiness, findings +# @PURPOSE: Manage one-question-at-a-time clarification state, deterministic answer persistence, and readiness/finding updates. +# @LAYER: Domain +# @RELATION: [DEPENDS_ON] ->[DatasetReviewSessionRepository] +# @RELATION: [DEPENDS_ON] ->[ClarificationSession] +# @RELATION: [DEPENDS_ON] ->[ClarificationQuestion] +# @RELATION: [DEPENDS_ON] ->[ClarificationAnswer] +# @RELATION: [DEPENDS_ON] ->[ValidationFinding] +# @PRE: Target session contains a persisted clarification aggregate in the current ownership scope. +# @POST: Active clarification payload exposes one highest-priority unresolved question, and each recorded answer is persisted before pointer/readiness mutation. +# @SIDE_EFFECT: Persists clarification answers, question/session states, and related readiness/finding changes. +# @DATA_CONTRACT: Input[DatasetReviewSession|ClarificationAnswerCommand] -> Output[ClarificationStateResult] +# @INVARIANT: Only one active clarification question may exist at a time; skipped and expert-review items remain unresolved and visible. +from __future__ import annotations + +# [DEF:ClarificationEngine.imports:Block] +import uuid +from dataclasses import dataclass, field +from datetime import datetime +from typing import List, Optional + +from src.core.logger import belief_scope, logger +from src.models.auth import User +from src.models.dataset_review import ( + AnswerKind, + ClarificationAnswer, + ClarificationQuestion, + ClarificationSession, + ClarificationStatus, + DatasetReviewSession, + FindingArea, + FindingSeverity, + QuestionState, + ReadinessState, + RecommendedAction, + ResolutionState, + SessionPhase, + ValidationFinding, +) +from src.services.dataset_review.repositories.session_repository import ( + DatasetReviewSessionRepository, +) +# [/DEF:ClarificationEngine.imports:Block] + + +# [DEF:ClarificationQuestionPayload:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Typed active-question payload returned to the API layer. +@dataclass +class ClarificationQuestionPayload: + question_id: str + clarification_session_id: str + topic_ref: str + question_text: str + why_it_matters: str + current_guess: Optional[str] + priority: int + state: QuestionState + options: list[dict[str, object]] = field(default_factory=list) +# [/DEF:ClarificationQuestionPayload:Class] + + +# [DEF:ClarificationStateResult:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Clarification state result carrying the current session, active payload, and changed findings. +@dataclass +class ClarificationStateResult: + clarification_session: ClarificationSession + current_question: Optional[ClarificationQuestionPayload] + session: DatasetReviewSession + changed_findings: List[ValidationFinding] = field(default_factory=list) +# [/DEF:ClarificationStateResult:Class] + + +# [DEF:ClarificationAnswerCommand:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Typed answer command for clarification state mutation. +@dataclass +class ClarificationAnswerCommand: + session: DatasetReviewSession + question_id: str + answer_kind: AnswerKind + answer_value: Optional[str] + user: User +# [/DEF:ClarificationAnswerCommand:Class] + + +# [DEF:ClarificationEngine:Class] +# @COMPLEXITY: 4 +# @PURPOSE: Provide deterministic one-question-at-a-time clarification selection and answer persistence. +# @RELATION: [DEPENDS_ON] ->[DatasetReviewSessionRepository] +# @RELATION: [DEPENDS_ON] ->[ClarificationSession] +# @RELATION: [DEPENDS_ON] ->[ValidationFinding] +# @PRE: Repository is bound to the current request transaction scope. +# @POST: Returned clarification state is persistence-backed and aligned with session readiness/recommended action. +# @SIDE_EFFECT: Mutates clarification answers, session flags, and related clarification findings. +class ClarificationEngine: + # [DEF:ClarificationEngine.__init__:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Bind repository dependency for clarification persistence operations. + def __init__(self, repository: DatasetReviewSessionRepository) -> None: + self.repository = repository + # [/DEF:ClarificationEngine.__init__:Function] + + # [DEF:ClarificationEngine.build_question_payload:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Return the one active highest-priority clarification question payload with why-it-matters, current guess, and options. + # @RELATION: [DEPENDS_ON] ->[ClarificationQuestion] + # @RELATION: [DEPENDS_ON] ->[ClarificationOption] + # @PRE: Session contains unresolved clarification state or a resumable clarification session. + # @POST: Returns exactly one active/open question payload or None when no unresolved question remains. + # @SIDE_EFFECT: Normalizes the active-question pointer and clarification status in persistence. + # @DATA_CONTRACT: Input[DatasetReviewSession] -> Output[ClarificationQuestionPayload|None] + def build_question_payload( + self, + session: DatasetReviewSession, + ) -> Optional[ClarificationQuestionPayload]: + with belief_scope("ClarificationEngine.build_question_payload"): + clarification_session = self._get_latest_clarification_session(session) + if clarification_session is None: + logger.reason( + "Clarification payload requested without clarification session", + extra={"session_id": session.session_id}, + ) + return None + + active_questions = [ + question for question in clarification_session.questions + if question.state == QuestionState.OPEN + ] + active_questions.sort(key=lambda item: (-int(item.priority), item.created_at, item.question_id)) + + if not active_questions: + clarification_session.current_question_id = None + clarification_session.status = ClarificationStatus.COMPLETED + session.readiness_state = self._derive_readiness_state(session) + session.recommended_action = self._derive_recommended_action(session) + if session.current_phase == SessionPhase.CLARIFICATION: + session.current_phase = SessionPhase.REVIEW + self.repository.db.commit() + logger.reflect( + "No unresolved clarification question remains", + extra={"session_id": session.session_id}, + ) + return None + + selected_question = active_questions[0] + clarification_session.current_question_id = selected_question.question_id + clarification_session.status = ClarificationStatus.ACTIVE + session.readiness_state = ReadinessState.CLARIFICATION_ACTIVE + session.recommended_action = RecommendedAction.ANSWER_NEXT_QUESTION + session.current_phase = SessionPhase.CLARIFICATION + + logger.reason( + "Selected active clarification question", + extra={ + "session_id": session.session_id, + "clarification_session_id": clarification_session.clarification_session_id, + "question_id": selected_question.question_id, + "priority": selected_question.priority, + }, + ) + self.repository.db.commit() + + payload = ClarificationQuestionPayload( + question_id=selected_question.question_id, + clarification_session_id=selected_question.clarification_session_id, + topic_ref=selected_question.topic_ref, + question_text=selected_question.question_text, + why_it_matters=selected_question.why_it_matters, + current_guess=selected_question.current_guess, + priority=selected_question.priority, + state=selected_question.state, + options=[ + { + "option_id": option.option_id, + "question_id": option.question_id, + "label": option.label, + "value": option.value, + "is_recommended": option.is_recommended, + "display_order": option.display_order, + } + for option in sorted( + selected_question.options, + key=lambda item: (item.display_order, item.label, item.option_id), + ) + ], + ) + logger.reflect( + "Clarification payload built", + extra={ + "session_id": session.session_id, + "question_id": payload.question_id, + "option_count": len(payload.options), + }, + ) + return payload + # [/DEF:ClarificationEngine.build_question_payload:Function] + + # [DEF:ClarificationEngine.record_answer:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Persist one clarification answer before any pointer/readiness mutation and compute deterministic state impact. + # @RELATION: [DEPENDS_ON] ->[ClarificationAnswer] + # @RELATION: [DEPENDS_ON] ->[ValidationFinding] + # @PRE: Target question belongs to the session's active clarification session and is still open. + # @POST: Answer row is persisted before current-question pointer advances; skipped/expert-review items remain unresolved and visible. + # @SIDE_EFFECT: Inserts answer row, mutates question/session states, updates clarification findings, and commits. + # @DATA_CONTRACT: Input[ClarificationAnswerCommand] -> Output[ClarificationStateResult] + def record_answer(self, command: ClarificationAnswerCommand) -> ClarificationStateResult: + with belief_scope("ClarificationEngine.record_answer"): + session = command.session + clarification_session = self._get_latest_clarification_session(session) + if clarification_session is None: + logger.explore( + "Cannot record clarification answer because no clarification session exists", + extra={"session_id": session.session_id}, + ) + raise ValueError("Clarification session not found") + + question = self._find_question(clarification_session, command.question_id) + if question is None: + logger.explore( + "Cannot record clarification answer for foreign or missing question", + extra={"session_id": session.session_id, "question_id": command.question_id}, + ) + raise ValueError("Clarification question not found") + + if question.answer is not None: + logger.explore( + "Rejected duplicate clarification answer submission", + extra={"session_id": session.session_id, "question_id": command.question_id}, + ) + raise ValueError("Clarification question already answered") + + if clarification_session.current_question_id and clarification_session.current_question_id != question.question_id: + logger.explore( + "Rejected answer for non-active clarification question", + extra={ + "session_id": session.session_id, + "question_id": question.question_id, + "current_question_id": clarification_session.current_question_id, + }, + ) + raise ValueError("Only the active clarification question can be answered") + + normalized_answer_value = self._normalize_answer_value(command.answer_kind, command.answer_value, question) + + logger.reason( + "Persisting clarification answer before state advancement", + extra={ + "session_id": session.session_id, + "question_id": question.question_id, + "answer_kind": command.answer_kind.value, + }, + ) + persisted_answer = ClarificationAnswer( + question_id=question.question_id, + answer_kind=command.answer_kind, + answer_value=normalized_answer_value, + answered_by_user_id=command.user.id, + impact_summary=self._build_impact_summary(question, command.answer_kind, normalized_answer_value), + ) + self.repository.db.add(persisted_answer) + self.repository.db.flush() + + changed_finding = self._upsert_clarification_finding( + session=session, + question=question, + answer_kind=command.answer_kind, + answer_value=normalized_answer_value, + ) + + if command.answer_kind == AnswerKind.SELECTED: + question.state = QuestionState.ANSWERED + elif command.answer_kind == AnswerKind.CUSTOM: + question.state = QuestionState.ANSWERED + elif command.answer_kind == AnswerKind.SKIPPED: + question.state = QuestionState.SKIPPED + elif command.answer_kind == AnswerKind.EXPERT_REVIEW: + question.state = QuestionState.EXPERT_REVIEW + + question.updated_at = datetime.utcnow() + self.repository.db.flush() + + clarification_session.resolved_count = self._count_resolved_questions(clarification_session) + clarification_session.remaining_count = self._count_remaining_questions(clarification_session) + clarification_session.summary_delta = self.summarize_progress(clarification_session) + clarification_session.updated_at = datetime.utcnow() + + next_question = self._select_next_open_question(clarification_session) + clarification_session.current_question_id = next_question.question_id if next_question else None + clarification_session.status = ( + ClarificationStatus.ACTIVE if next_question else ClarificationStatus.COMPLETED + ) + if clarification_session.status == ClarificationStatus.COMPLETED: + clarification_session.completed_at = datetime.utcnow() + + session.readiness_state = self._derive_readiness_state(session) + session.recommended_action = self._derive_recommended_action(session) + session.current_phase = ( + SessionPhase.CLARIFICATION + if clarification_session.current_question_id + else SessionPhase.REVIEW + ) + session.last_activity_at = datetime.utcnow() + + self.repository.db.commit() + self.repository.db.refresh(session) + + logger.reflect( + "Clarification answer recorded and session advanced", + extra={ + "session_id": session.session_id, + "question_id": question.question_id, + "next_question_id": clarification_session.current_question_id, + "readiness_state": session.readiness_state.value, + "remaining_count": clarification_session.remaining_count, + }, + ) + + return ClarificationStateResult( + clarification_session=clarification_session, + current_question=self.build_question_payload(session), + session=session, + changed_findings=[changed_finding] if changed_finding else [], + ) + # [/DEF:ClarificationEngine.record_answer:Function] + + # [DEF:ClarificationEngine.summarize_progress:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Produce a compact progress summary for pause/resume and completion UX. + # @RELATION: [DEPENDS_ON] ->[ClarificationSession] + def summarize_progress(self, clarification_session: ClarificationSession) -> str: + resolved = self._count_resolved_questions(clarification_session) + remaining = self._count_remaining_questions(clarification_session) + return f"{resolved} resolved, {remaining} unresolved" + # [/DEF:ClarificationEngine.summarize_progress:Function] + + # [DEF:ClarificationEngine._get_latest_clarification_session:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Select the latest clarification session for the current dataset review aggregate. + def _get_latest_clarification_session( + self, + session: DatasetReviewSession, + ) -> Optional[ClarificationSession]: + if not session.clarification_sessions: + return None + ordered_sessions = sorted( + session.clarification_sessions, + key=lambda item: (item.started_at, item.clarification_session_id), + reverse=True, + ) + return ordered_sessions[0] + # [/DEF:ClarificationEngine._get_latest_clarification_session:Function] + + # [DEF:ClarificationEngine._find_question:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Resolve a clarification question from the active clarification aggregate. + def _find_question( + self, + clarification_session: ClarificationSession, + question_id: str, + ) -> Optional[ClarificationQuestion]: + for question in clarification_session.questions: + if question.question_id == question_id: + return question + return None + # [/DEF:ClarificationEngine._find_question:Function] + + # [DEF:ClarificationEngine._select_next_open_question:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Select the next unresolved question in deterministic priority order. + def _select_next_open_question( + self, + clarification_session: ClarificationSession, + ) -> Optional[ClarificationQuestion]: + open_questions = [ + question for question in clarification_session.questions + if question.state == QuestionState.OPEN + ] + if not open_questions: + return None + open_questions.sort(key=lambda item: (-int(item.priority), item.created_at, item.question_id)) + return open_questions[0] + # [/DEF:ClarificationEngine._select_next_open_question:Function] + + # [DEF:ClarificationEngine._count_resolved_questions:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Count questions whose answers fully resolved the ambiguity. + def _count_resolved_questions(self, clarification_session: ClarificationSession) -> int: + return sum( + 1 + for question in clarification_session.questions + if question.state == QuestionState.ANSWERED + ) + # [/DEF:ClarificationEngine._count_resolved_questions:Function] + + # [DEF:ClarificationEngine._count_remaining_questions:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Count questions still unresolved or deferred after clarification interaction. + def _count_remaining_questions(self, clarification_session: ClarificationSession) -> int: + return sum( + 1 + for question in clarification_session.questions + if question.state in {QuestionState.OPEN, QuestionState.SKIPPED, QuestionState.EXPERT_REVIEW} + ) + # [/DEF:ClarificationEngine._count_remaining_questions:Function] + + # [DEF:ClarificationEngine._normalize_answer_value:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Validate and normalize answer payload based on answer kind and active question options. + def _normalize_answer_value( + self, + answer_kind: AnswerKind, + answer_value: Optional[str], + question: ClarificationQuestion, + ) -> Optional[str]: + normalized_answer_value = str(answer_value).strip() if answer_value is not None else None + if answer_kind in {AnswerKind.SELECTED, AnswerKind.CUSTOM} and not normalized_answer_value: + raise ValueError("answer_value is required for selected or custom clarification answers") + if answer_kind == AnswerKind.SELECTED: + allowed_values = {option.value for option in question.options} + if normalized_answer_value not in allowed_values: + raise ValueError("answer_value must match one of the current clarification options") + if answer_kind == AnswerKind.SKIPPED: + return normalized_answer_value or "skipped" + if answer_kind == AnswerKind.EXPERT_REVIEW: + return normalized_answer_value or "expert_review" + return normalized_answer_value + # [/DEF:ClarificationEngine._normalize_answer_value:Function] + + # [DEF:ClarificationEngine._build_impact_summary:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Build a compact audit note describing how the clarification answer affects session state. + def _build_impact_summary( + self, + question: ClarificationQuestion, + answer_kind: AnswerKind, + answer_value: Optional[str], + ) -> str: + if answer_kind == AnswerKind.SKIPPED: + return f"Clarification for {question.topic_ref} was skipped and remains unresolved." + if answer_kind == AnswerKind.EXPERT_REVIEW: + return f"Clarification for {question.topic_ref} was deferred for expert review." + return f"Clarification for {question.topic_ref} recorded as '{answer_value}'." + # [/DEF:ClarificationEngine._build_impact_summary:Function] + + # [DEF:ClarificationEngine._upsert_clarification_finding:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Keep one finding per clarification topic aligned with answer outcome and unresolved visibility rules. + # @RELATION: [DEPENDS_ON] ->[ValidationFinding] + def _upsert_clarification_finding( + self, + session: DatasetReviewSession, + question: ClarificationQuestion, + answer_kind: AnswerKind, + answer_value: Optional[str], + ) -> ValidationFinding: + caused_by_ref = f"clarification:{question.question_id}" + existing = next( + ( + finding for finding in session.findings + if finding.area == FindingArea.CLARIFICATION and finding.caused_by_ref == caused_by_ref + ), + None, + ) + + if answer_kind in {AnswerKind.SELECTED, AnswerKind.CUSTOM}: + resolution_state = ResolutionState.RESOLVED + resolved_at = datetime.utcnow() + message = f"Clarified '{question.topic_ref}' with answer '{answer_value}'." + elif answer_kind == AnswerKind.SKIPPED: + resolution_state = ResolutionState.SKIPPED + resolved_at = None + message = f"Clarification for '{question.topic_ref}' was skipped and still needs review." + else: + resolution_state = ResolutionState.EXPERT_REVIEW + resolved_at = None + message = f"Clarification for '{question.topic_ref}' requires expert review." + + if existing is None: + existing = ValidationFinding( + finding_id=str(uuid.uuid4()), + session_id=session.session_id, + area=FindingArea.CLARIFICATION, + severity=FindingSeverity.WARNING, + code="CLARIFICATION_PENDING", + title="Clarification pending", + message=message, + resolution_state=resolution_state, + resolution_note=None, + caused_by_ref=caused_by_ref, + created_at=datetime.utcnow(), + resolved_at=resolved_at, + ) + self.repository.db.add(existing) + session.findings.append(existing) + else: + existing.message = message + existing.resolution_state = resolution_state + existing.resolved_at = resolved_at + + if answer_kind in {AnswerKind.SELECTED, AnswerKind.CUSTOM}: + existing.code = "CLARIFICATION_RESOLVED" + existing.title = "Clarification resolved" + elif answer_kind == AnswerKind.SKIPPED: + existing.code = "CLARIFICATION_SKIPPED" + existing.title = "Clarification skipped" + else: + existing.code = "CLARIFICATION_EXPERT_REVIEW" + existing.title = "Clarification requires expert review" + + return existing + # [/DEF:ClarificationEngine._upsert_clarification_finding:Function] + + # [DEF:ClarificationEngine._derive_readiness_state:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Recompute readiness after clarification mutation while preserving unresolved visibility semantics. + # @RELATION: [DEPENDS_ON] ->[ClarificationSession] + # @RELATION: [DEPENDS_ON] ->[DatasetReviewSession] + def _derive_readiness_state(self, session: DatasetReviewSession) -> ReadinessState: + clarification_session = self._get_latest_clarification_session(session) + if clarification_session is None: + return session.readiness_state + + if clarification_session.current_question_id: + return ReadinessState.CLARIFICATION_ACTIVE + + if clarification_session.remaining_count > 0: + return ReadinessState.CLARIFICATION_NEEDED + + return ReadinessState.REVIEW_READY + # [/DEF:ClarificationEngine._derive_readiness_state:Function] + + # [DEF:ClarificationEngine._derive_recommended_action:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Recompute next-action guidance after clarification mutations. + def _derive_recommended_action(self, session: DatasetReviewSession) -> RecommendedAction: + clarification_session = self._get_latest_clarification_session(session) + if clarification_session is None: + return session.recommended_action + if clarification_session.current_question_id: + return RecommendedAction.ANSWER_NEXT_QUESTION + if clarification_session.remaining_count > 0: + return RecommendedAction.START_CLARIFICATION + return RecommendedAction.REVIEW_DOCUMENTATION + # [/DEF:ClarificationEngine._derive_recommended_action:Function] +# [/DEF:ClarificationEngine:Class] + +# [/DEF:ClarificationEngine:Module] \ No newline at end of file diff --git a/backend/src/services/dataset_review/event_logger.py b/backend/src/services/dataset_review/event_logger.py new file mode 100644 index 00000000..53063261 --- /dev/null +++ b/backend/src/services/dataset_review/event_logger.py @@ -0,0 +1,156 @@ +# [DEF:SessionEventLogger:Module] +# @COMPLEXITY: 4 +# @SEMANTICS: dataset_review, audit, session_events, persistence, observability +# @PURPOSE: Persist explicit session mutation events for dataset-review audit trails without weakening ownership or approval invariants. +# @LAYER: Domain +# @RELATION: [DEPENDS_ON] ->[SessionEvent] +# @RELATION: [DEPENDS_ON] ->[DatasetReviewSession] +# @PRE: Caller provides an owned session scope and an authenticated actor identifier for each persisted mutation event. +# @POST: Every logged event is committed as an explicit, queryable audit record with deterministic event metadata. +# @SIDE_EFFECT: Inserts persisted session event rows and emits runtime belief-state logs for audit-sensitive mutations. + +from __future__ import annotations + +# [DEF:SessionEventLogger.imports:Block] +from dataclasses import dataclass, field +from typing import Any, Dict, Optional + +from sqlalchemy.orm import Session + +from src.core.logger import belief_scope, logger +from src.models.dataset_review import DatasetReviewSession, SessionEvent +# [/DEF:SessionEventLogger.imports:Block] + + +# [DEF:SessionEventPayload:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Typed input contract for one persisted dataset-review session audit event. +@dataclass(frozen=True) +class SessionEventPayload: + session_id: str + actor_user_id: str + event_type: str + event_summary: str + current_phase: Optional[str] = None + readiness_state: Optional[str] = None + event_details: Dict[str, Any] = field(default_factory=dict) +# [/DEF:SessionEventPayload:Class] + + +# [DEF:SessionEventLogger:Class] +# @COMPLEXITY: 4 +# @PURPOSE: Persist explicit dataset-review session audit events with meaningful runtime reasoning logs. +# @RELATION: [DEPENDS_ON] ->[SessionEvent] +# @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session] +# @PRE: The database session is live and payload identifiers are non-empty. +# @POST: Returns the committed session event row with a stable identifier and stored detail payload. +# @SIDE_EFFECT: Writes one audit row to persistence and emits logger.reason/logger.reflect traces. +class SessionEventLogger: + # [DEF:SessionEventLogger.__init__:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Bind a live SQLAlchemy session to the session-event logger. + def __init__(self, db: Session) -> None: + self.db = db + # [/DEF:SessionEventLogger.__init__:Function] + + # [DEF:SessionEventLogger.log_event:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Persist one explicit session event row for an owned dataset-review mutation. + # @RELATION: [DEPENDS_ON] ->[SessionEvent] + # @PRE: session_id, actor_user_id, event_type, and event_summary are non-empty. + # @POST: Returns the committed SessionEvent record with normalized detail payload. + # @SIDE_EFFECT: Inserts and commits one session_events row. + # @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent] + def log_event(self, payload: SessionEventPayload) -> SessionEvent: + with belief_scope("SessionEventLogger.log_event"): + session_id = str(payload.session_id or "").strip() + actor_user_id = str(payload.actor_user_id or "").strip() + event_type = str(payload.event_type or "").strip() + event_summary = str(payload.event_summary or "").strip() + + if not session_id: + logger.explore("Session event logging rejected because session_id is empty") + raise ValueError("session_id must be non-empty") + if not actor_user_id: + logger.explore( + "Session event logging rejected because actor_user_id is empty", + extra={"session_id": session_id}, + ) + raise ValueError("actor_user_id must be non-empty") + if not event_type: + logger.explore( + "Session event logging rejected because event_type is empty", + extra={"session_id": session_id, "actor_user_id": actor_user_id}, + ) + raise ValueError("event_type must be non-empty") + if not event_summary: + logger.explore( + "Session event logging rejected because event_summary is empty", + extra={"session_id": session_id, "event_type": event_type}, + ) + raise ValueError("event_summary must be non-empty") + + normalized_details = dict(payload.event_details or {}) + logger.reason( + "Persisting explicit dataset-review session audit event", + extra={ + "session_id": session_id, + "actor_user_id": actor_user_id, + "event_type": event_type, + "current_phase": payload.current_phase, + "readiness_state": payload.readiness_state, + }, + ) + + event = SessionEvent( + session_id=session_id, + actor_user_id=actor_user_id, + event_type=event_type, + event_summary=event_summary, + current_phase=payload.current_phase, + readiness_state=payload.readiness_state, + event_details=normalized_details, + ) + self.db.add(event) + self.db.commit() + self.db.refresh(event) + + logger.reflect( + "Dataset-review session audit event persisted", + extra={ + "session_id": session_id, + "session_event_id": event.session_event_id, + "event_type": event.event_type, + }, + ) + return event + # [/DEF:SessionEventLogger.log_event:Function] + + # [DEF:SessionEventLogger.log_for_session:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Convenience wrapper for logging an event directly from a session aggregate root. + # @RELATION: [CALLS] ->[SessionEventLogger.log_event] + def log_for_session( + self, + session: DatasetReviewSession, + *, + actor_user_id: str, + event_type: str, + event_summary: str, + event_details: Optional[Dict[str, Any]] = None, + ) -> SessionEvent: + return self.log_event( + SessionEventPayload( + session_id=session.session_id, + actor_user_id=actor_user_id, + event_type=event_type, + event_summary=event_summary, + current_phase=session.current_phase.value if session.current_phase else None, + readiness_state=session.readiness_state.value if session.readiness_state else None, + event_details=dict(event_details or {}), + ) + ) + # [/DEF:SessionEventLogger.log_for_session:Function] +# [/DEF:SessionEventLogger:Class] + +# [/DEF:SessionEventLogger:Module] \ No newline at end of file diff --git a/backend/src/services/dataset_review/orchestrator.py b/backend/src/services/dataset_review/orchestrator.py index 54abcffc..0dfcec48 100644 --- a/backend/src/services/dataset_review/orchestrator.py +++ b/backend/src/services/dataset_review/orchestrator.py @@ -19,23 +19,36 @@ from __future__ import annotations # [DEF:DatasetReviewOrchestrator.imports:Block] from dataclasses import dataclass, field +from datetime import datetime +import hashlib +import json from typing import Any, Dict, List, Optional from src.core.config_manager import ConfigManager from src.core.logger import belief_scope, logger from src.core.task_manager import TaskManager +from src.core.utils.superset_compilation_adapter import ( + PreviewCompilationPayload, + SqlLabLaunchPayload, + SupersetCompilationAdapter, +) from src.core.utils.superset_context_extractor import ( SupersetContextExtractor, SupersetParsedContext, ) from src.models.auth import User from src.models.dataset_review import ( + ApprovalState, BusinessSummarySource, + CompiledPreview, ConfidenceState, DatasetProfile, DatasetReviewSession, + DatasetRunContext, FindingArea, FindingSeverity, + LaunchStatus, + PreviewStatus, RecommendedAction, ReadinessState, ResolutionState, @@ -47,6 +60,7 @@ from src.services.dataset_review.repositories.session_repository import ( DatasetReviewSessionRepository, ) from src.services.dataset_review.semantic_resolver import SemanticSourceResolver +from src.services.dataset_review.event_logger import SessionEventPayload # [/DEF:DatasetReviewOrchestrator.imports:Block] @@ -73,6 +87,48 @@ class StartSessionResult: # [/DEF:StartSessionResult:Class] +# [DEF:PreparePreviewCommand:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Typed input contract for compiling one Superset-backed session preview. +@dataclass +class PreparePreviewCommand: + user: User + session_id: str +# [/DEF:PreparePreviewCommand:Class] + + +# [DEF:PreparePreviewResult:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Result contract for one persisted compiled preview attempt. +@dataclass +class PreparePreviewResult: + session: DatasetReviewSession + preview: CompiledPreview + blocked_reasons: List[str] = field(default_factory=list) +# [/DEF:PreparePreviewResult:Class] + + +# [DEF:LaunchDatasetCommand:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Typed input contract for launching one dataset-review session into SQL Lab. +@dataclass +class LaunchDatasetCommand: + user: User + session_id: str +# [/DEF:LaunchDatasetCommand:Class] + + +# [DEF:LaunchDatasetResult:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Launch result carrying immutable run context and any gate blockers surfaced before launch. +@dataclass +class LaunchDatasetResult: + session: DatasetReviewSession + run_context: DatasetRunContext + blocked_reasons: List[str] = field(default_factory=list) +# [/DEF:LaunchDatasetResult:Class] + + # [DEF:DatasetReviewOrchestrator:Class] # @COMPLEXITY: 5 # @PURPOSE: Coordinate safe session startup while preserving cross-user isolation and explicit partial recovery. @@ -197,6 +253,23 @@ class DatasetReviewOrchestrator: parsed_context=parsed_context, dataset_ref=dataset_ref, ) + self.repository.event_logger.log_event( + SessionEventPayload( + session_id=persisted_session.session_id, + actor_user_id=command.user.id, + event_type="session_started", + event_summary="Dataset review session shell created", + current_phase=persisted_session.current_phase.value, + readiness_state=persisted_session.readiness_state.value, + event_details={ + "source_kind": persisted_session.source_kind, + "dataset_ref": persisted_session.dataset_ref, + "dataset_id": persisted_session.dataset_id, + "dashboard_id": persisted_session.dashboard_id, + "partial_recovery": bool(parsed_context and parsed_context.partial_recovery), + }, + ) + ) persisted_session = self.repository.save_profile_and_findings( persisted_session.session_id, command.user.id, @@ -213,6 +286,17 @@ class DatasetReviewOrchestrator: persisted_session.active_task_id = active_task_id self.repository.db.commit() self.repository.db.refresh(persisted_session) + self.repository.event_logger.log_event( + SessionEventPayload( + session_id=persisted_session.session_id, + actor_user_id=command.user.id, + event_type="recovery_task_linked", + event_summary="Recovery task linked to dataset review session", + current_phase=persisted_session.current_phase.value, + readiness_state=persisted_session.readiness_state.value, + event_details={"task_id": active_task_id}, + ) + ) logger.reason( "Linked recovery task to started dataset review session", extra={"session_id": persisted_session.session_id, "task_id": active_task_id}, @@ -237,6 +321,238 @@ class DatasetReviewOrchestrator: ) # [/DEF:DatasetReviewOrchestrator.start_session:Function] + # [DEF:DatasetReviewOrchestrator.prepare_launch_preview:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Assemble effective execution inputs and trigger Superset-side preview compilation. + # @RELATION: [CALLS] ->[SupersetCompilationAdapter.compile_preview] + # @PRE: all required variables have candidate values or explicitly accepted defaults. + # @POST: returns preview artifact in pending, ready, failed, or stale state. + # @SIDE_EFFECT: persists preview attempt and upstream compilation diagnostics. + # @DATA_CONTRACT: Input[PreparePreviewCommand] -> Output[PreparePreviewResult] + def prepare_launch_preview(self, command: PreparePreviewCommand) -> PreparePreviewResult: + with belief_scope("DatasetReviewOrchestrator.prepare_launch_preview"): + session = self.repository.load_session_detail(command.session_id, command.user.id) + if session is None or session.user_id != command.user.id: + logger.explore( + "Preview preparation rejected because owned session was not found", + extra={"session_id": command.session_id, "user_id": command.user.id}, + ) + raise ValueError("Session not found") + + if session.dataset_id is None: + raise ValueError("Preview requires a resolved dataset_id") + + environment = self.config_manager.get_environment(session.environment_id) + if environment is None: + raise ValueError("Environment not found") + + execution_snapshot = self._build_execution_snapshot(session) + preview_blockers = execution_snapshot["preview_blockers"] + if preview_blockers: + logger.explore( + "Preview preparation blocked by incomplete execution context", + extra={ + "session_id": session.session_id, + "blocked_reasons": preview_blockers, + }, + ) + raise ValueError("Preview blocked: " + "; ".join(preview_blockers)) + + adapter = SupersetCompilationAdapter(environment) + preview = adapter.compile_preview( + PreviewCompilationPayload( + session_id=session.session_id, + dataset_id=session.dataset_id, + preview_fingerprint=execution_snapshot["preview_fingerprint"], + template_params=execution_snapshot["template_params"], + effective_filters=execution_snapshot["effective_filters"], + ) + ) + persisted_preview = self.repository.save_preview( + session.session_id, + command.user.id, + preview, + ) + + session.current_phase = SessionPhase.PREVIEW + session.last_activity_at = datetime.utcnow() + if persisted_preview.preview_status == PreviewStatus.READY: + launch_blockers = self._build_launch_blockers( + session=session, + execution_snapshot=execution_snapshot, + preview=persisted_preview, + ) + if launch_blockers: + session.readiness_state = ReadinessState.COMPILED_PREVIEW_READY + session.recommended_action = RecommendedAction.APPROVE_MAPPING + else: + session.readiness_state = ReadinessState.RUN_READY + session.recommended_action = RecommendedAction.LAUNCH_DATASET + else: + session.readiness_state = ReadinessState.PARTIALLY_READY + session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW + self.repository.db.commit() + self.repository.db.refresh(session) + self.repository.event_logger.log_event( + SessionEventPayload( + session_id=session.session_id, + actor_user_id=command.user.id, + event_type="preview_generated", + event_summary="Superset preview generation persisted", + current_phase=session.current_phase.value, + readiness_state=session.readiness_state.value, + event_details={ + "preview_id": persisted_preview.preview_id, + "preview_status": persisted_preview.preview_status.value, + "preview_fingerprint": persisted_preview.preview_fingerprint, + }, + ) + ) + + logger.reflect( + "Superset preview preparation completed", + extra={ + "session_id": session.session_id, + "preview_id": persisted_preview.preview_id, + "preview_status": persisted_preview.preview_status.value, + "preview_fingerprint": persisted_preview.preview_fingerprint, + }, + ) + return PreparePreviewResult( + session=session, + preview=persisted_preview, + blocked_reasons=[], + ) + # [/DEF:DatasetReviewOrchestrator.prepare_launch_preview:Function] + + # [DEF:DatasetReviewOrchestrator.launch_dataset:Function] + # @COMPLEXITY: 5 + # @PURPOSE: Start the approved dataset execution through SQL Lab and persist run context for audit/replay. + # @RELATION: [CALLS] ->[SupersetCompilationAdapter.create_sql_lab_session] + # @PRE: session is run-ready and compiled preview is current. + # @POST: returns persisted run context with SQL Lab session reference and launch outcome. + # @SIDE_EFFECT: creates SQL Lab execution session and audit snapshot. + # @DATA_CONTRACT: Input[LaunchDatasetCommand] -> Output[LaunchDatasetResult] + # @INVARIANT: launch remains blocked unless blocking findings are closed, approvals are satisfied, and the latest Superset preview fingerprint matches current execution inputs. + def launch_dataset(self, command: LaunchDatasetCommand) -> LaunchDatasetResult: + with belief_scope("DatasetReviewOrchestrator.launch_dataset"): + session = self.repository.load_session_detail(command.session_id, command.user.id) + if session is None or session.user_id != command.user.id: + logger.explore( + "Launch rejected because owned session was not found", + extra={"session_id": command.session_id, "user_id": command.user.id}, + ) + raise ValueError("Session not found") + + if session.dataset_id is None: + raise ValueError("Launch requires a resolved dataset_id") + + environment = self.config_manager.get_environment(session.environment_id) + if environment is None: + raise ValueError("Environment not found") + + execution_snapshot = self._build_execution_snapshot(session) + current_preview = self._get_latest_preview(session) + launch_blockers = self._build_launch_blockers( + session=session, + execution_snapshot=execution_snapshot, + preview=current_preview, + ) + if launch_blockers: + logger.explore( + "Launch gate blocked dataset execution", + extra={ + "session_id": session.session_id, + "blocked_reasons": launch_blockers, + }, + ) + raise ValueError("Launch blocked: " + "; ".join(launch_blockers)) + + adapter = SupersetCompilationAdapter(environment) + try: + sql_lab_session_ref = adapter.create_sql_lab_session( + SqlLabLaunchPayload( + session_id=session.session_id, + dataset_id=session.dataset_id, + preview_id=current_preview.preview_id, + compiled_sql=str(current_preview.compiled_sql or ""), + template_params=execution_snapshot["template_params"], + ) + ) + launch_status = LaunchStatus.STARTED + launch_error = None + except Exception as exc: + logger.explore( + "SQL Lab launch failed after passing gates", + extra={"session_id": session.session_id, "error": str(exc)}, + ) + sql_lab_session_ref = "unavailable" + launch_status = LaunchStatus.FAILED + launch_error = str(exc) + + run_context = DatasetRunContext( + session_id=session.session_id, + dataset_ref=session.dataset_ref, + environment_id=session.environment_id, + preview_id=current_preview.preview_id, + sql_lab_session_ref=sql_lab_session_ref, + effective_filters=execution_snapshot["effective_filters"], + template_params=execution_snapshot["template_params"], + approved_mapping_ids=execution_snapshot["approved_mapping_ids"], + semantic_decision_refs=execution_snapshot["semantic_decision_refs"], + open_warning_refs=execution_snapshot["open_warning_refs"], + launch_status=launch_status, + launch_error=launch_error, + ) + persisted_run_context = self.repository.save_run_context( + session.session_id, + command.user.id, + run_context, + ) + + session.current_phase = SessionPhase.LAUNCH + session.last_activity_at = datetime.utcnow() + if launch_status == LaunchStatus.FAILED: + session.readiness_state = ReadinessState.COMPILED_PREVIEW_READY + session.recommended_action = RecommendedAction.LAUNCH_DATASET + else: + session.readiness_state = ReadinessState.RUN_IN_PROGRESS + session.recommended_action = RecommendedAction.EXPORT_OUTPUTS + self.repository.db.commit() + self.repository.db.refresh(session) + self.repository.event_logger.log_event( + SessionEventPayload( + session_id=session.session_id, + actor_user_id=command.user.id, + event_type="dataset_launch_requested", + event_summary="Dataset launch handoff persisted", + current_phase=session.current_phase.value, + readiness_state=session.readiness_state.value, + event_details={ + "run_context_id": persisted_run_context.run_context_id, + "launch_status": persisted_run_context.launch_status.value, + "preview_id": persisted_run_context.preview_id, + "sql_lab_session_ref": persisted_run_context.sql_lab_session_ref, + }, + ) + ) + + logger.reflect( + "Dataset launch orchestration completed with audited run context", + extra={ + "session_id": session.session_id, + "run_context_id": persisted_run_context.run_context_id, + "launch_status": persisted_run_context.launch_status.value, + "preview_id": persisted_run_context.preview_id, + }, + ) + return LaunchDatasetResult( + session=session, + run_context=persisted_run_context, + blocked_reasons=[], + ) + # [/DEF:DatasetReviewOrchestrator.launch_dataset:Function] + # [DEF:DatasetReviewOrchestrator._parse_dataset_selection:Function] # @COMPLEXITY: 3 # @PURPOSE: Normalize dataset-selection payload into canonical session references. @@ -328,6 +644,158 @@ class DatasetReviewOrchestrator: return findings # [/DEF:DatasetReviewOrchestrator._build_partial_recovery_findings:Function] + # [DEF:DatasetReviewOrchestrator._build_execution_snapshot:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Build effective filters, template params, approvals, and fingerprint for preview and launch gating. + # @RELATION: [DEPENDS_ON] ->[DatasetReviewSession] + # @PRE: Session aggregate includes imported filters, template variables, and current execution mappings. + # @POST: returns deterministic execution snapshot for current session state without mutating persistence. + # @SIDE_EFFECT: none. + # @DATA_CONTRACT: Input[DatasetReviewSession] -> Output[Dict[str,Any]] + def _build_execution_snapshot(self, session: DatasetReviewSession) -> Dict[str, Any]: + filter_lookup = {item.filter_id: item for item in session.imported_filters} + variable_lookup = {item.variable_id: item for item in session.template_variables} + + effective_filters: List[Dict[str, Any]] = [] + template_params: Dict[str, Any] = {} + approved_mapping_ids: List[str] = [] + open_warning_refs: List[str] = [] + preview_blockers: List[str] = [] + + for mapping in session.execution_mappings: + imported_filter = filter_lookup.get(mapping.filter_id) + template_variable = variable_lookup.get(mapping.variable_id) + if imported_filter is None: + preview_blockers.append(f"mapping:{mapping.mapping_id}:missing_filter") + continue + if template_variable is None: + preview_blockers.append(f"mapping:{mapping.mapping_id}:missing_variable") + continue + + effective_value = mapping.effective_value + if effective_value is None: + effective_value = imported_filter.normalized_value + if effective_value is None: + effective_value = imported_filter.raw_value + if effective_value is None: + effective_value = template_variable.default_value + + if effective_value is None and template_variable.is_required: + preview_blockers.append(f"variable:{template_variable.variable_name}:missing_required_value") + continue + + effective_filters.append( + { + "mapping_id": mapping.mapping_id, + "filter_id": imported_filter.filter_id, + "filter_name": imported_filter.filter_name, + "variable_id": template_variable.variable_id, + "variable_name": template_variable.variable_name, + "effective_value": effective_value, + "raw_input_value": mapping.raw_input_value, + } + ) + template_params[template_variable.variable_name] = effective_value + if mapping.approval_state == ApprovalState.APPROVED: + approved_mapping_ids.append(mapping.mapping_id) + if mapping.requires_explicit_approval and mapping.approval_state != ApprovalState.APPROVED: + open_warning_refs.append(mapping.mapping_id) + + mapped_variable_ids = {mapping.variable_id for mapping in session.execution_mappings} + for variable in session.template_variables: + if variable.variable_id in mapped_variable_ids: + continue + if variable.default_value is not None: + template_params[variable.variable_name] = variable.default_value + continue + if variable.is_required: + preview_blockers.append(f"variable:{variable.variable_name}:unmapped") + + semantic_decision_refs = [ + field.field_id + for field in session.semantic_fields + if field.is_locked or not field.needs_review or field.provenance.value != "unresolved" + ] + preview_fingerprint = self._compute_preview_fingerprint( + { + "dataset_id": session.dataset_id, + "template_params": template_params, + "effective_filters": effective_filters, + } + ) + return { + "effective_filters": effective_filters, + "template_params": template_params, + "approved_mapping_ids": sorted(approved_mapping_ids), + "semantic_decision_refs": sorted(semantic_decision_refs), + "open_warning_refs": sorted(open_warning_refs), + "preview_blockers": sorted(set(preview_blockers)), + "preview_fingerprint": preview_fingerprint, + } + # [/DEF:DatasetReviewOrchestrator._build_execution_snapshot:Function] + + # [DEF:DatasetReviewOrchestrator._build_launch_blockers:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Enforce launch gates from findings, approvals, and current preview truth. + # @RELATION: [DEPENDS_ON] ->[CompiledPreview] + # @PRE: execution_snapshot was computed from current session state and preview is the latest persisted preview or None. + # @POST: returns explicit blocker codes for every unmet launch invariant. + # @SIDE_EFFECT: none. + # @DATA_CONTRACT: Input[DatasetReviewSession,Dict[str,Any],CompiledPreview|None] -> Output[List[str]] + def _build_launch_blockers( + self, + session: DatasetReviewSession, + execution_snapshot: Dict[str, Any], + preview: Optional[CompiledPreview], + ) -> List[str]: + blockers = list(execution_snapshot["preview_blockers"]) + + for finding in session.findings: + if ( + finding.severity == FindingSeverity.BLOCKING + and finding.resolution_state not in {ResolutionState.RESOLVED, ResolutionState.APPROVED} + ): + blockers.append(f"finding:{finding.code}:blocking") + for mapping in session.execution_mappings: + if mapping.requires_explicit_approval and mapping.approval_state != ApprovalState.APPROVED: + blockers.append(f"mapping:{mapping.mapping_id}:approval_required") + + if preview is None: + blockers.append("preview:missing") + else: + if preview.preview_status != PreviewStatus.READY: + blockers.append(f"preview:{preview.preview_status.value}") + if preview.preview_fingerprint != execution_snapshot["preview_fingerprint"]: + blockers.append("preview:fingerprint_mismatch") + + return sorted(set(blockers)) + # [/DEF:DatasetReviewOrchestrator._build_launch_blockers:Function] + + # [DEF:DatasetReviewOrchestrator._get_latest_preview:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Resolve the current latest preview snapshot for one session aggregate. + def _get_latest_preview(self, session: DatasetReviewSession) -> Optional[CompiledPreview]: + if not session.previews: + return None + if session.last_preview_id: + for preview in session.previews: + if preview.preview_id == session.last_preview_id: + return preview + return sorted( + session.previews, + key=lambda item: (item.created_at or datetime.min, item.preview_id), + reverse=True, + )[0] + # [/DEF:DatasetReviewOrchestrator._get_latest_preview:Function] + + # [DEF:DatasetReviewOrchestrator._compute_preview_fingerprint:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Produce deterministic execution fingerprint for preview truth and staleness checks. + def _compute_preview_fingerprint(self, payload: Dict[str, Any]) -> str: + serialized = json.dumps(payload, sort_keys=True, default=str) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest() + # [/DEF:DatasetReviewOrchestrator._compute_preview_fingerprint:Function] + # [DEF:DatasetReviewOrchestrator._enqueue_recovery_task:Function] # @COMPLEXITY: 4 # @PURPOSE: Link session start to observable async recovery when task infrastructure is available. diff --git a/backend/src/services/dataset_review/repositories/session_repository.py b/backend/src/services/dataset_review/repositories/session_repository.py index 706afc6c..63e932de 100644 --- a/backend/src/services/dataset_review/repositories/session_repository.py +++ b/backend/src/services/dataset_review/repositories/session_repository.py @@ -16,14 +16,19 @@ from typing import Optional, List from sqlalchemy import or_ from sqlalchemy.orm import Session, joinedload from src.models.dataset_review import ( + ClarificationQuestion, + ClarificationSession, DatasetReviewSession, DatasetProfile, ValidationFinding, CompiledPreview, DatasetRunContext, - SessionCollaborator + SemanticFieldEntry, + SessionCollaborator, + SessionEvent, ) -from src.core.logger import belief_scope +from src.core.logger import belief_scope, logger +from src.services.dataset_review.event_logger import SessionEventLogger # [DEF:SessionRepo:Class] # @COMPLEXITY: 4 @@ -37,16 +42,46 @@ from src.core.logger import belief_scope # @SIDE_EFFECT: mutates and queries the persistence layer through the injected database session. # @DATA_CONTRACT: Input[OwnedSessionQuery|SessionMutation] -> Output[PersistedSessionAggregate|PersistedChildRecord] class DatasetReviewSessionRepository: - """ - @PURPOSE: Persist and retrieve dataset review session aggregates. - @INVARIANT: ownership_scope -> All operations must respect the session owner's user_id. - """ # [DEF:init_repo:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Bind one live SQLAlchemy session to the repository instance. def __init__(self, db: Session): self.db = db + self.event_logger = SessionEventLogger(db) # [/DEF:init_repo:Function] + # [DEF:get_owned_session:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Resolve one owner-scoped dataset review session for mutation paths without leaking foreign-session state. + # @RELATION: [DEPENDS_ON] -> [DatasetReviewSession] + # @PRE: session_id and user_id are non-empty identifiers from the authenticated ownership scope. + # @POST: returns the owned session or raises a deterministic access error. + # @SIDE_EFFECT: reads one session row from the current database transaction. + # @DATA_CONTRACT: Input[OwnedSessionQuery] -> Output[DatasetReviewSession|ValueError] + def _get_owned_session(self, session_id: str, user_id: str) -> DatasetReviewSession: + with belief_scope("DatasetReviewSessionRepository.get_owned_session"): + logger.reason( + "Resolving owner-scoped dataset review session for mutation path", + extra={"session_id": session_id, "user_id": user_id}, + ) + session = self.db.query(DatasetReviewSession).filter( + DatasetReviewSession.session_id == session_id, + DatasetReviewSession.user_id == user_id, + ).first() + if not session: + logger.explore( + "Owner-scoped dataset review session lookup failed", + extra={"session_id": session_id, "user_id": user_id}, + ) + raise ValueError("Session not found or access denied") + logger.reflect( + "Owner-scoped dataset review session resolved", + extra={"session_id": session.session_id, "user_id": session.user_id}, + ) + return session + # [/DEF:get_owned_session:Function] + # [DEF:create_sess:Function] # @COMPLEXITY: 4 # @PURPOSE: Persist an initial dataset review session shell. @@ -57,9 +92,17 @@ class DatasetReviewSessionRepository: # @DATA_CONTRACT: Input[DatasetReviewSession] -> Output[DatasetReviewSession] def create_session(self, session: DatasetReviewSession) -> DatasetReviewSession: with belief_scope("DatasetReviewSessionRepository.create_session"): + logger.reason( + "Persisting dataset review session shell", + extra={"user_id": session.user_id, "environment_id": session.environment_id}, + ) self.db.add(session) self.db.commit() self.db.refresh(session) + logger.reflect( + "Dataset review session shell persisted with stable identifier", + extra={"session_id": session.session_id, "user_id": session.user_id}, + ) return session # [/DEF:create_sess:Function] @@ -69,25 +112,27 @@ class DatasetReviewSessionRepository: # @RELATION: [DEPENDS_ON] -> [DatasetReviewSession] # @RELATION: [DEPENDS_ON] -> [SessionCollaborator] def load_session_detail(self, session_id: str, user_id: str) -> Optional[DatasetReviewSession]: - """ - @PRE: user_id must match session owner or authorized collaborator. - """ with belief_scope("DatasetReviewSessionRepository.load_session_detail"): - # Check if user is owner or collaborator - return self.db.query(DatasetReviewSession)\ + logger.reason( + "Loading dataset review session detail for owner-or-collaborator scope", + extra={"session_id": session_id, "user_id": user_id}, + ) + session = self.db.query(DatasetReviewSession)\ .outerjoin(SessionCollaborator, DatasetReviewSession.session_id == SessionCollaborator.session_id)\ .options( joinedload(DatasetReviewSession.profile), joinedload(DatasetReviewSession.findings), joinedload(DatasetReviewSession.collaborators), joinedload(DatasetReviewSession.semantic_sources), - joinedload(DatasetReviewSession.semantic_fields), + joinedload(DatasetReviewSession.semantic_fields).joinedload(SemanticFieldEntry.candidates), joinedload(DatasetReviewSession.imported_filters), joinedload(DatasetReviewSession.template_variables), joinedload(DatasetReviewSession.execution_mappings), - joinedload(DatasetReviewSession.clarification_sessions), + joinedload(DatasetReviewSession.clarification_sessions).joinedload(ClarificationSession.questions).joinedload(ClarificationQuestion.options), + joinedload(DatasetReviewSession.clarification_sessions).joinedload(ClarificationSession.questions).joinedload(ClarificationQuestion.answer), joinedload(DatasetReviewSession.previews), - joinedload(DatasetReviewSession.run_contexts) + joinedload(DatasetReviewSession.run_contexts), + joinedload(DatasetReviewSession.events) )\ .filter(DatasetReviewSession.session_id == session_id)\ .filter( @@ -97,6 +142,15 @@ class DatasetReviewSessionRepository: ) )\ .first() + logger.reflect( + "Dataset review session detail lookup completed", + extra={ + "session_id": session_id, + "user_id": user_id, + "found": bool(session), + }, + ) + return session # [/DEF:load_detail:Function] # [DEF:save_prof_find:Function] @@ -111,32 +165,40 @@ class DatasetReviewSessionRepository: # @DATA_CONTRACT: Input[ProfileAndFindingsMutation] -> Output[DatasetReviewSession] def save_profile_and_findings(self, session_id: str, user_id: str, profile: DatasetProfile, findings: List[ValidationFinding]) -> DatasetReviewSession: with belief_scope("DatasetReviewSessionRepository.save_profile_and_findings"): - session = self.db.query(DatasetReviewSession).filter( - DatasetReviewSession.session_id == session_id, - DatasetReviewSession.user_id == user_id - ).first() - - if not session: - raise ValueError("Session not found or access denied") + session = self._get_owned_session(session_id, user_id) + logger.reason( + "Persisting dataset profile and replacing validation findings", + extra={ + "session_id": session_id, + "user_id": user_id, + "has_profile": bool(profile), + "findings_count": len(findings), + }, + ) if profile: - # Ensure we update existing profile by session_id if it exists existing_profile = self.db.query(DatasetProfile).filter_by(session_id=session_id).first() if existing_profile: profile.profile_id = existing_profile.profile_id self.db.merge(profile) - # Remove old findings for this session to avoid stale data self.db.query(ValidationFinding).filter( ValidationFinding.session_id == session_id ).delete() - # Add new findings for finding in findings: finding.session_id = session_id self.db.add(finding) self.db.commit() + logger.reflect( + "Dataset profile and validation findings committed", + extra={ + "session_id": session.session_id, + "user_id": user_id, + "findings_count": len(findings), + }, + ) return self.load_session_detail(session_id, user_id) # [/DEF:save_prof_find:Function] @@ -151,15 +213,12 @@ class DatasetReviewSessionRepository: # @DATA_CONTRACT: Input[PreviewMutation] -> Output[CompiledPreview] def save_preview(self, session_id: str, user_id: str, preview: CompiledPreview) -> CompiledPreview: with belief_scope("DatasetReviewSessionRepository.save_preview"): - session = self.db.query(DatasetReviewSession).filter( - DatasetReviewSession.session_id == session_id, - DatasetReviewSession.user_id == user_id - ).first() + session = self._get_owned_session(session_id, user_id) + logger.reason( + "Persisting compiled preview and staling previous preview snapshots", + extra={"session_id": session_id, "user_id": user_id}, + ) - if not session: - raise ValueError("Session not found or access denied") - - # Mark existing previews for this session as stale if they are not the new one self.db.query(CompiledPreview).filter( CompiledPreview.session_id == session_id ).update({"preview_status": "stale"}) @@ -170,6 +229,14 @@ class DatasetReviewSessionRepository: self.db.commit() self.db.refresh(preview) + logger.reflect( + "Compiled preview committed as latest session preview", + extra={ + "session_id": session.session_id, + "preview_id": preview.preview_id, + "user_id": user_id, + }, + ) return preview # [/DEF:save_prev:Function] @@ -184,13 +251,11 @@ class DatasetReviewSessionRepository: # @DATA_CONTRACT: Input[RunContextMutation] -> Output[DatasetRunContext] def save_run_context(self, session_id: str, user_id: str, run_context: DatasetRunContext) -> DatasetRunContext: with belief_scope("DatasetReviewSessionRepository.save_run_context"): - session = self.db.query(DatasetReviewSession).filter( - DatasetReviewSession.session_id == session_id, - DatasetReviewSession.user_id == user_id - ).first() - - if not session: - raise ValueError("Session not found or access denied") + session = self._get_owned_session(session_id, user_id) + logger.reason( + "Persisting dataset run context audit snapshot", + extra={"session_id": session_id, "user_id": user_id}, + ) self.db.add(run_context) self.db.flush() @@ -198,6 +263,14 @@ class DatasetReviewSessionRepository: self.db.commit() self.db.refresh(run_context) + logger.reflect( + "Dataset run context committed as latest launch snapshot", + extra={ + "session_id": session.session_id, + "run_context_id": run_context.run_context_id, + "user_id": user_id, + }, + ) return run_context # [/DEF:save_run_ctx:Function] @@ -207,9 +280,18 @@ class DatasetReviewSessionRepository: # @RELATION: [DEPENDS_ON] -> [DatasetReviewSession] def list_sessions_for_user(self, user_id: str) -> List[DatasetReviewSession]: with belief_scope("DatasetReviewSessionRepository.list_sessions_for_user"): - return self.db.query(DatasetReviewSession).filter( + logger.reason( + "Listing dataset review sessions for owner scope", + extra={"user_id": user_id}, + ) + sessions = self.db.query(DatasetReviewSession).filter( DatasetReviewSession.user_id == user_id ).order_by(DatasetReviewSession.updated_at.desc()).all() + logger.reflect( + "Dataset review session list assembled", + extra={"user_id": user_id, "session_count": len(sessions)}, + ) + return sessions # [/DEF:list_user_sess:Function] # [/DEF:SessionRepo:Class] diff --git a/backend/src/services/dataset_review/semantic_resolver.py b/backend/src/services/dataset_review/semantic_resolver.py index 9bc06287..d09ff49e 100644 --- a/backend/src/services/dataset_review/semantic_resolver.py +++ b/backend/src/services/dataset_review/semantic_resolver.py @@ -24,6 +24,7 @@ from src.models.dataset_review import ( CandidateMatchType, CandidateStatus, FieldProvenance, + SemanticSource, ) # [/DEF:SemanticSourceResolver.imports:Block] @@ -259,6 +260,63 @@ class SemanticSourceResolver: return merged # [/DEF:SemanticSourceResolver.apply_field_decision:Function] + # [DEF:SemanticSourceResolver.propagate_source_version_update:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Propagate a semantic source version change to unlocked field entries without silently overwriting manual or locked values. + # @RELATION: [DEPENDS_ON] ->[SemanticSource] + # @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry] + # @PRE: source is persisted and fields belong to the same session aggregate. + # @POST: unlocked fields linked to the source carry the new source version and are marked reviewable; manual or locked fields keep their active values untouched. + # @SIDE_EFFECT: mutates in-memory field state for the caller to persist. + # @DATA_CONTRACT: Input[SemanticSource,List[SemanticFieldEntry]] -> Output[Dict[str,int]] + def propagate_source_version_update( + self, + source: SemanticSource, + fields: Iterable[Any], + ) -> Dict[str, int]: + with belief_scope("SemanticSourceResolver.propagate_source_version_update"): + source_id = str(source.source_id or "").strip() + source_version = str(source.source_version or "").strip() + if not source_id or not source_version: + logger.explore( + "Semantic source version propagation rejected due to incomplete source metadata", + extra={"source_id": source_id, "source_version": source_version}, + ) + raise ValueError("Semantic source must provide source_id and source_version") + + propagated = 0 + preserved_locked = 0 + untouched = 0 + for field in fields: + if str(getattr(field, "source_id", "") or "").strip() != source_id: + untouched += 1 + continue + if bool(getattr(field, "is_locked", False)) or getattr(field, "provenance", None) == FieldProvenance.MANUAL_OVERRIDE: + preserved_locked += 1 + continue + + field.source_version = source_version + field.needs_review = True + field.has_conflict = bool(getattr(field, "has_conflict", False)) + propagated += 1 + + logger.reflect( + "Semantic source version propagation completed", + extra={ + "source_id": source_id, + "source_version": source_version, + "propagated": propagated, + "preserved_locked": preserved_locked, + "untouched": untouched, + }, + ) + return { + "propagated": propagated, + "preserved_locked": preserved_locked, + "untouched": untouched, + } + # [/DEF:SemanticSourceResolver.propagate_source_version_update:Function] + # [DEF:SemanticSourceResolver._normalize_dictionary_row:Function] # @COMPLEXITY: 2 # @PURPOSE: Normalize one dictionary row into a consistent lookup structure. diff --git a/backend/tests/services/dataset_review/test_superset_matrix.py b/backend/tests/services/dataset_review/test_superset_matrix.py new file mode 100644 index 00000000..3fddabe4 --- /dev/null +++ b/backend/tests/services/dataset_review/test_superset_matrix.py @@ -0,0 +1,126 @@ +# [DEF:SupersetCompatibilityMatrixTests:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: dataset_review, superset, compatibility_matrix, preview, sql_lab, tests +# @PURPOSE: Verifies Superset preview and SQL Lab endpoint fallback strategy used by dataset-review orchestration. +# @LAYER: Tests +# @RELATION: [DEPENDS_ON] ->[SupersetCompilationAdapter] +# @RELATION: [DEPENDS_ON] ->[SupersetClient] + +from types import SimpleNamespace +from unittest.mock import MagicMock + +from src.core.utils.superset_compilation_adapter import ( + PreviewCompilationPayload, + SqlLabLaunchPayload, + SupersetCompilationAdapter, +) + + +# [DEF:make_adapter:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Build an adapter with a mock Superset client and deterministic environment for compatibility tests. +def make_adapter(): + environment = SimpleNamespace( + id="env-1", + name="Test Env", + url="http://superset.example", + username="user", + password="pass", + verify_ssl=True, + timeout=30, + ) + client = MagicMock() + client.network = MagicMock() + return SupersetCompilationAdapter(environment=environment, client=client), client +# [/DEF:make_adapter:Function] + + +# [DEF:test_preview_prefers_supported_client_method_before_network_fallback:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Confirms preview compilation uses a supported client method first when the capability exists. +def test_preview_prefers_supported_client_method_before_network_fallback(): + adapter, client = make_adapter() + client.compile_preview = MagicMock(return_value={"compiled_sql": "SELECT 1"}) + payload = PreviewCompilationPayload( + session_id="sess-1", + dataset_id=42, + preview_fingerprint="fp-1", + template_params={"country": "RU"}, + effective_filters=[{"name": "country", "value": "RU"}], + ) + + preview = adapter.compile_preview(payload) + + assert preview.preview_status.value == "ready" + assert preview.compiled_sql == "SELECT 1" + client.compile_preview.assert_called_once() + client.network.request.assert_not_called() +# [/DEF:test_preview_prefers_supported_client_method_before_network_fallback:Function] + + +# [DEF:test_preview_falls_back_across_matrix_until_supported_endpoint_returns_sql:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Confirms preview fallback walks the compatibility matrix from preferred to legacy endpoints until one returns compiled SQL. +def test_preview_falls_back_across_matrix_until_supported_endpoint_returns_sql(): + adapter, client = make_adapter() + payload = PreviewCompilationPayload( + session_id="sess-2", + dataset_id=77, + preview_fingerprint="fp-2", + template_params={"region": "emea"}, + effective_filters=[], + ) + + client.network.request.side_effect = [ + RuntimeError("preview endpoint unavailable"), + {"result": {"sql": "SELECT * FROM dataset_77"}}, + ] + + preview = adapter.compile_preview(payload) + + assert preview.preview_status.value == "ready" + assert preview.compiled_sql == "SELECT * FROM dataset_77" + assert client.network.request.call_count == 2 + first_call = client.network.request.call_args_list[0].kwargs + second_call = client.network.request.call_args_list[1].kwargs + assert first_call["endpoint"] == "/dataset/77/preview" + assert second_call["endpoint"] == "/dataset/77/sql" +# [/DEF:test_preview_falls_back_across_matrix_until_supported_endpoint_returns_sql:Function] + + +# [DEF:test_sql_lab_launch_falls_back_to_legacy_execute_endpoint:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Confirms SQL Lab launch falls back from modern to legacy execute endpoint and preserves canonical session reference extraction. +def test_sql_lab_launch_falls_back_to_legacy_execute_endpoint(): + adapter, client = make_adapter() + client.get_dataset.return_value = { + "result": { + "id": 55, + "schema": "public", + "database": {"id": 9}, + } + } + client.network.request.side_effect = [ + RuntimeError("sqllab execute unavailable"), + {"result": {"id": "query-123"}}, + ] + payload = SqlLabLaunchPayload( + session_id="sess-3", + dataset_id=55, + preview_id="preview-9", + compiled_sql="SELECT * FROM sales", + template_params={"limit": 10}, + ) + + sql_lab_ref = adapter.create_sql_lab_session(payload) + + assert sql_lab_ref == "query-123" + assert client.network.request.call_count == 2 + first_call = client.network.request.call_args_list[0].kwargs + second_call = client.network.request.call_args_list[1].kwargs + assert first_call["endpoint"] == "/sqllab/execute/" + assert second_call["endpoint"] == "/sql_lab/execute/" +# [/DEF:test_sql_lab_launch_falls_back_to_legacy_execute_endpoint:Function] + + +# [/DEF:SupersetCompatibilityMatrixTests:Module] \ No newline at end of file diff --git a/frontend/src/lib/components/dataset-review/ClarificationDialog.svelte b/frontend/src/lib/components/dataset-review/ClarificationDialog.svelte new file mode 100644 index 00000000..5d06543a --- /dev/null +++ b/frontend/src/lib/components/dataset-review/ClarificationDialog.svelte @@ -0,0 +1,472 @@ + + + + + + + + + + + + + + + + + + + +
+
+
+

+ {$t.dataset_review?.clarification?.eyebrow} +

+

+ {$t.dataset_review?.clarification?.title} +

+

+ {$t.dataset_review?.clarification?.description} +

+
+ +
+
+ {$t.dataset_review?.clarification?.progress_label} +
+
{progressLabel}
+
+ {$t.dataset_review?.clarification?.state_label}: {dialogState} +
+
+
+ + {#if clarificationSession} +
+ + {$t.dataset_review?.clarification?.status_label}: {clarificationSession.status} + + + {$t.dataset_review?.clarification?.remaining_label}: {clarificationSession.remaining_count} + + {#if clarificationSession.summary_delta} + + {$t.dataset_review?.clarification?.summary_delta_label}: + {clarificationSession.summary_delta} + + {/if} +
+ {/if} + + {#if hasQuestion} +
+
+
+

+ {$t.dataset_review?.clarification?.active_question_label} +

+

+ {currentQuestion.question_text} +

+ +
+
+ {$t.dataset_review?.clarification?.why_it_matters_label} +
+

{currentQuestion.why_it_matters}

+
+ +
+
+
+ {$t.dataset_review?.clarification?.current_guess_label} +
+

+ {currentQuestion.current_guess || + $t.dataset_review?.clarification?.current_guess_empty} +

+
+ +
+
+ {$t.dataset_review?.clarification?.topic_label} +
+

{currentQuestion.topic_ref}

+
+
+ +
+

+ {$t.dataset_review?.clarification?.options_title} +

+ +
+ {#each currentQuestion.options || [] as option} + + {/each} +
+
+ + +
+ +
+ + + + + + + + + +
+
+
+ {:else} +
+ {$t.dataset_review?.clarification?.completed} +
+ +
+ + + +
+ {/if} + + {#if feedbackQuestionId} +
+
+
+
+ {$t.dataset_review?.clarification?.feedback_label} +
+

+ {feedbackQuestionText || + $t.dataset_review?.clarification?.feedback_prompt} +

+
+
+ + +
+
+
+ {/if} + + {#if localMessage || localStatus === "saving"} +
+ {localStatus === "saving" + ? $t.dataset_review?.clarification?.messages?.saving + : localMessage} +
+ {/if} +
+ + \ No newline at end of file diff --git a/frontend/src/lib/components/dataset-review/CompiledSQLPreview.svelte b/frontend/src/lib/components/dataset-review/CompiledSQLPreview.svelte new file mode 100644 index 00000000..6a3a42e3 --- /dev/null +++ b/frontend/src/lib/components/dataset-review/CompiledSQLPreview.svelte @@ -0,0 +1,247 @@ + + + + + + + + + + + + + + + + + + + + +
+
+
+

+ {$t.dataset_review?.preview?.eyebrow} +

+

+ {$t.dataset_review?.preview?.title} +

+

+ {$t.dataset_review?.preview?.description} +

+
+ +
+ + {$t.dataset_review?.preview?.state_label}: {getStateLabel(effectiveState)} + + +
+
+ +
+ {getPreviewBodyText()} +
+ + {#if preview} +
+ + {$t.dataset_review?.preview?.compiler_label}: {getCompilerLabel()} + + + {$t.dataset_review?.preview?.fingerprint_label}: {preview.preview_fingerprint} + + {#if previewTimestamp} + + {$t.dataset_review?.preview?.compiled_at_label}: {previewTimestamp} + + {/if} +
+ {/if} + + {#if effectiveState === "ready" && hasSql} +
+
+
+ {$t.dataset_review?.preview?.sql_block_title} +
+
+ {$t.dataset_review?.preview?.compiled_truth_note} +
+
+ 
{preview.compiled_sql}
+
+ {:else if effectiveState === "failed"} +
+ +
+ {:else if effectiveState === "stale"} +
+ +
+ {/if} + + {#if localMessage} +
+ {localMessage} +
+ {/if} +
+ + \ No newline at end of file diff --git a/frontend/src/lib/components/dataset-review/ExecutionMappingReview.svelte b/frontend/src/lib/components/dataset-review/ExecutionMappingReview.svelte new file mode 100644 index 00000000..51fe2958 --- /dev/null +++ b/frontend/src/lib/components/dataset-review/ExecutionMappingReview.svelte @@ -0,0 +1,578 @@ + + + + + + + + + + + + + + + + + + +
+
+
+

+ {$t.dataset_review?.mapping?.eyebrow} +

+

+ {$t.dataset_review?.mapping?.title} +

+

+ {$t.dataset_review?.mapping?.description} +

+
+ +
+ + {$t.dataset_review?.mapping?.state_label}: {$t.dataset_review?.mapping?.state?.[reviewState]} + + + {$t.dataset_review?.mapping?.pending_approvals_label}: {pendingApprovalCount} + + + {$t.dataset_review?.mapping?.required_values_label}: {missingRequiredCount} + + +
+
+ + {#if sortedMappings.length === 0} +
+ {$t.dataset_review?.mapping?.empty} +
+ {:else} + {#if missingRequiredCount > 0} +
+ {$t.dataset_review?.mapping?.required_blockers_notice.replace( + "{count}", + String(missingRequiredCount), + )} +
+ {/if} + + {#if pendingApprovalCount > 0} +
+ {$t.dataset_review?.mapping?.approval_notice.replace( + "{count}", + String(pendingApprovalCount), + )} +
+ {/if} + +
+ {#each sortedMappings as mapping} + {@const rowState = getRowState(mapping)} + {@const templateVariable = rowState.templateVariable} + {@const importedFilter = rowState.importedFilter} + {@const isEditing = editingMappingId === mapping.mapping_id} + {@const isSaving = savingMappingId === mapping.mapping_id} + {@const rowMessage = rowMessages[mapping.mapping_id] || { status: "", text: "" }} + +
+
+
+
+

+ {importedFilter?.display_name || importedFilter?.filter_name || mapping.mapping_id} +

+ + {$t.dataset_review?.mapping?.to_variable_label}: {templateVariable?.variable_name || + mapping.variable_id} + + {#if templateVariable?.is_required} + + {$t.dataset_review?.mapping?.required_badge} + + {/if} + {#if rowState.needsExplicitApproval} + + {$t.dataset_review?.mapping?.approval_required_badge} + + {/if} + {#if mapping.approval_state === "approved"} + + {$t.dataset_review?.mapping?.approved_badge} + + {/if} +
+ +
+
+
+ {$t.dataset_review?.mapping?.source_filter_label} +
+
+ {importedFilter?.filter_name || mapping.filter_id} +
+
+ + {$t.dataset_review?.mapping?.source_value_label}: + {stringifyValue(importedFilter?.raw_value)} + + + {$t.dataset_review?.mapping?.confidence_label}: + {importedFilter?.confidence_state || ($t.common?.unknown || "unknown")} + + + {$t.dataset_review?.mapping?.recovery_label}: + {importedFilter?.recovery_status || ($t.common?.unknown || "unknown")} + +
+
+ +
+
+ {$t.dataset_review?.mapping?.effective_value_label} +
+
+ {stringifyValue(rowState.effectiveValue)} +
+
+ + {$t.dataset_review?.mapping?.method_label}: {getMethodLabel(mapping.mapping_method)} + + + {$t.dataset_review?.mapping?.approval_label}: {getApprovalLabel(mapping.approval_state)} + + {#if mapping.warning_level} + + {$t.dataset_review?.mapping?.warning_label}: {getWarningLabel(mapping.warning_level)} + + {/if} +
+
+
+ + {#if mapping.transformation_note} +
+ + {$t.dataset_review?.mapping?.transformation_note_label}: + + {mapping.transformation_note} +
+ {/if} + + {#if rowState.missingRequiredValue} +
+ {$t.dataset_review?.mapping?.missing_required_value} +
+ {/if} + + {#if isEditing} +
+ + + + +
+ + +
+
+ {/if} + + {#if rowMessage.text || rowMessage.status === "saving"} +
+ {rowMessage.status === "saving" + ? $t.dataset_review?.mapping?.messages?.saving + : rowMessage.text} +
+ {/if} +
+ +
+ + + +
+
+
+ {/each} +
+ {/if} +
+ + \ No newline at end of file diff --git a/frontend/src/lib/components/dataset-review/LaunchConfirmationPanel.svelte b/frontend/src/lib/components/dataset-review/LaunchConfirmationPanel.svelte new file mode 100644 index 00000000..e190e89b --- /dev/null +++ b/frontend/src/lib/components/dataset-review/LaunchConfirmationPanel.svelte @@ -0,0 +1,372 @@ + + + + + + + + + + + + + + + + + + +
+
+
+

+ {$t.dataset_review?.launch?.eyebrow} +

+

+ {$t.dataset_review?.launch?.title} +

+

+ {$t.dataset_review?.launch?.description} +

+
+ + + {$t.dataset_review?.launch?.state_label}: {getStateLabel(panelState)} + +
+ + {#if panelState === "Blocked"} +
+

+ {$t.dataset_review?.launch?.blocked_title} +

+

+ {$t.dataset_review?.launch?.blocked_body} +

+ +
+ {#each launchBlockers as blocker} +
+
{blocker.label}
+ {#if blocker.detail} +
{blocker.detail}
+ {/if} + +
+ {/each} +
+
+ {/if} + + {#if panelState === "Ready"} +
+
+
+
+
+ {$t.dataset_review?.launch?.dataset_ref_label} +
+
+ {session?.dataset_ref || ($t.common?.unknown || "unknown")} +
+
+ +
+
+ {$t.dataset_review?.launch?.readiness_label} +
+
+ {session?.readiness_state || ($t.common?.unknown || "unknown")} +
+
+ +
+
+ {$t.dataset_review?.launch?.approved_mappings_label} +
+
+ {approvedMappingsCount} +
+
+ +
+
+ {$t.dataset_review?.launch?.preview_fingerprint_label} +
+
+ {preview?.preview_fingerprint || ($t.common?.not_available || "N/A")} +
+
+
+
+ +
+

+ {$t.dataset_review?.launch?.sql_lab_target_title} +

+

+ {$t.dataset_review?.launch?.sql_lab_target_body} +

+
+ + {$t.dataset_review?.launch?.preview_status_label}: {preview?.preview_status} + + + {$t.dataset_review?.launch?.compiled_by_label}: {preview?.compiled_by} + +
+
+ + +
+ {/if} + + {#if panelState === "Submitted"} +
+
+

+ {$t.dataset_review?.launch?.submitted_title} +

+

+ {$t.dataset_review?.launch?.submitted_body} +

+
+ +
+
+
+ {$t.dataset_review?.launch?.run_context_label} +
+
+ {activeRunContext?.run_context_id} +
+
+ +
+
+ {$t.dataset_review?.launch?.sql_lab_session_label} +
+
+ {activeRunContext?.sql_lab_session_ref} +
+
+ +
+
+ {$t.dataset_review?.launch?.launch_status_label} +
+
+ {activeRunContext?.launch_status} +
+
+ +
+
+ {$t.dataset_review?.launch?.preview_ref_label} +
+
+ {activeRunContext?.preview_id} +
+
+
+
+ {/if} + + {#if launchMessage} +
+ {launchMessage} +
+ {/if} +
+ + \ No newline at end of file diff --git a/frontend/src/lib/components/dataset-review/SemanticLayerReview.svelte b/frontend/src/lib/components/dataset-review/SemanticLayerReview.svelte new file mode 100644 index 00000000..c8756ca3 --- /dev/null +++ b/frontend/src/lib/components/dataset-review/SemanticLayerReview.svelte @@ -0,0 +1,598 @@ + + + + + + + + + + + + + + + + + +
+
+
+

+ {$t.dataset_review?.semantics?.eyebrow} +

+

+ {$t.dataset_review?.semantics?.title} +

+

+ {$t.dataset_review?.semantics?.description} +

+
+ +
+
+
+ {$t.dataset_review?.semantics?.field_count_label} +
+
{activeFieldCount}
+
+ +
+
+ + {#if activeFieldCount === 0} +
+ {$t.dataset_review?.semantics?.empty} +
+ {:else} +
+ {#each sortedFields as field} + {@const message = fieldMessages[field.field_id] || { status: "", text: "" }} + {@const isEditing = editingFieldId === field.field_id} + {@const isSaving = savingFieldId === field.field_id} + +
+
+
+
+

{field.field_name}

+ + {field.field_kind} + + + {field.is_locked + ? $t.dataset_review?.semantics?.locked_badge + : $t.dataset_review?.semantics?.unlocked_badge} + + {#if field.has_conflict} + + {$t.dataset_review?.semantics?.conflict_badge} + + {/if} + {#if field.needs_review} + + {$t.dataset_review?.semantics?.needs_review_badge} + + {/if} +
+ +
+
+ {$t.dataset_review?.semantics?.active_value_label} +
+
+ {getCurrentValueSummary(field)} +
+
+ + {$t.dataset_review?.semantics?.provenance_label}: + {getProvenanceLabel(field.provenance)} + + + {$t.dataset_review?.semantics?.confidence_label}: + {getConfidenceLabel(field.confidence_rank)} + + {#if field.source_id} + + {$t.dataset_review?.semantics?.source_label}: {getSourceLabel(field.source_id)} + + {/if} + + {$t.dataset_review?.semantics?.changed_by_label}: {field.last_changed_by} + +
+
+ + {#if isEditing} +
+ + + + + + +
+ + +
+
+ {/if} + +
+
+

+ {$t.dataset_review?.semantics?.candidates_title} +

+ + {field.candidates?.length || 0} + +
+ + {#if !field.candidates?.length} +

+ {$t.dataset_review?.semantics?.candidates_empty} +

+ {:else} +
+ {#each field.candidates as candidate} +
+
+
+
+
+ {candidate.proposed_verbose_name || + $t.dataset_review?.semantics?.empty_value} +
+ + {candidate.match_type} + + + {$t.dataset_review?.semantics?.score_label}: + {candidate.confidence_score} + + + {getCandidateStatusLabel(candidate.status)} + +
+ +

+ {candidate.proposed_description || + $t.dataset_review?.semantics?.candidate_description_empty} +

+ + {#if candidate.proposed_display_format} +

+ {$t.dataset_review?.semantics?.display_format_label}: + {candidate.proposed_display_format} +

+ {/if} +
+ +
+ + +
+
+
+ {/each} +
+ {/if} +
+ + {#if message.text || message.status === "saving"} +
+ {message.status === "saving" + ? $t.dataset_review?.semantics?.messages?.saving + : message.text} +
+ {/if} +
+ +
+ + + + +
+ + +
+
+
+
+ {/each} +
+ {/if} +
+ + \ No newline at end of file diff --git a/frontend/src/lib/components/dataset-review/__tests__/us2_semantic_clarification.ux.test.js b/frontend/src/lib/components/dataset-review/__tests__/us2_semantic_clarification.ux.test.js new file mode 100644 index 00000000..9bb3025c --- /dev/null +++ b/frontend/src/lib/components/dataset-review/__tests__/us2_semantic_clarification.ux.test.js @@ -0,0 +1,412 @@ +/** + * @vitest-environment jsdom + */ +// @ts-nocheck +// [DEF:DatasetReviewUs2UxTests:Module] +// @COMPLEXITY: 3 +// @SEMANTICS: dataset-review, semantics, clarification, ux-tests, field-review +// @PURPOSE: Verify US2 semantic field review and one-question clarification UX contracts. +// @LAYER: UI +// @RELATION: [VERIFIES] ->[SemanticLayerReview] +// @RELATION: [VERIFIES] ->[ClarificationDialog] +// @UX_STATE: Conflicted -> Semantic candidates and manual override affordances remain visible. +// @UX_STATE: Question -> Clarification shows one active question, why-it-matters context, and resumable actions. +// @TEST_CONTRACT: Us2DatasetReviewProps -> ObservableSemanticAndClarificationUx +// @TEST_SCENARIO: semantic_review_surfaces_active_value_and_candidates -> Field-level semantic review shows provenance, candidates, and mutation callbacks. +// @TEST_SCENARIO: clarification_dialog_shows_active_question_and_actions -> Clarification dialog renders one active question with answer and escalation actions. +// @TEST_EDGE: missing_field -> Missing optional values still keep semantic summary readable. +// @TEST_EDGE: external_fail -> Failed request surfaces inline recovery message. +// @TEST_INVARIANT: us2_components_remain_actionable -> VERIFIED_BY: [semantic_review_surfaces_active_value_and_candidates, clarification_dialog_shows_active_question_and_actions] + +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { render, screen, fireEvent, waitFor } from "@testing-library/svelte"; +import SemanticLayerReview from "../SemanticLayerReview.svelte"; +import ClarificationDialog from "../ClarificationDialog.svelte"; +import { requestApi } from "$lib/api.js"; +import { openAssistantChat } from "$lib/stores/assistantChat.js"; + +const mockedRequestApi = /** @type {any} */ (requestApi); +const mockedOpenAssistantChat = /** @type {any} */ (openAssistantChat); + +vi.mock("$lib/api.js", () => ({ + requestApi: vi.fn(), +})); + +vi.mock("$lib/stores/assistantChat.js", () => ({ + openAssistantChat: vi.fn(), +})); + +vi.mock("$lib/i18n", () => ({ + t: { + subscribe: (fn) => { + fn({ + common: { + error: "Common error", + cancel: "Cancel", + }, + dataset_review: { + semantics: { + eyebrow: "Semantic layer", + title: "Review semantic field values", + description: "Review semantic candidates", + field_count_label: "Fields in review", + empty: "No semantic fields", + unknown_source: "Unknown source", + active_value_label: "Active semantic value", + provenance_label: "Provenance", + confidence_label: "Confidence", + confidence_unset: "No confidence rank", + confidence_rank_label: "Rank", + source_label: "Source", + changed_by_label: "Changed by", + locked_badge: "Locked", + unlocked_badge: "Unlocked", + conflict_badge: "Conflict", + needs_review_badge: "Needs review", + manual_verbose_name_label: "Manual verbose name", + manual_description_label: "Manual description", + manual_display_format_label: "Manual display format", + manual_override_action: "Manual override", + save_manual_action: "Save manual value", + lock_action: "Lock field", + unlock_action: "Unlock field", + feedback_up_action: "Thumbs up", + feedback_down_action: "Thumbs down", + candidates_title: "Candidate options", + candidates_empty: "No candidates", + candidate_description_empty: "No candidate description", + display_format_label: "Display format", + score_label: "Score", + apply_candidate_action: "Apply", + apply_and_lock_action: "Apply and lock", + empty_value: "No value", + provenance: { + unresolved: "Unresolved", + dictionary_exact: "Dictionary exact match", + manual_override: "Manual override", + }, + candidate_status: { + pending: "Pending", + accepted: "Accepted", + superseded: "Superseded", + }, + messages: { + saving: "Saving semantic decision...", + save_failed: "Failed to save semantic decision.", + manual_override_required: "Manual value required", + manual_saved: "Manual semantic value saved and locked.", + candidate_applied: "Candidate value applied.", + candidate_locked: "Candidate value applied and locked.", + locked: "Field locked against silent overwrite.", + unlocked: "Field unlocked for renewed review.", + feedback_up: "Positive feedback recorded.", + feedback_down: "Negative feedback recorded.", + }, + }, + clarification: { + eyebrow: "Clarification", + title: "Resolve one ambiguity at a time", + description: "One question at a time", + progress_label: "Clarification progress", + state_label: "Dialog state", + status_label: "Status", + remaining_label: "Remaining questions", + summary_delta_label: "Summary delta", + active_question_label: "Active question", + why_it_matters_label: "Why it matters", + current_guess_label: "Current guess", + current_guess_empty: "No current guess", + topic_label: "Topic", + options_title: "Available options", + recommended_badge: "Recommended", + custom_answer_label: "Custom answer", + answer_action: "Answer with selected option", + custom_answer_action: "Submit custom answer", + skip_action: "Skip for now", + assistant_action: "Open assistant guidance", + expert_review_action: "Send to expert review", + resume_action: "Resume clarification", + completed: "No active clarification question remains.", + feedback_label: "Clarification feedback", + feedback_prompt: "Record whether the clarification result was useful.", + feedback_up_action: "Helpful", + feedback_down_action: "Not helpful", + messages: { + saving: "Saving clarification answer...", + saved: "Clarification answer saved.", + skipped: "Question skipped and left resumable.", + expert_review_requested: "Question escalated to expert review.", + assistant_opened: "Assistant guidance opened.", + resumed: "Clarification resumed.", + resume_failed: "Failed to resume clarification.", + save_failed: "Failed to save clarification answer.", + feedback_up: "Positive clarification feedback recorded.", + feedback_down: "Negative clarification feedback recorded.", + feedback_failed: "Failed to save clarification feedback.", + option_required: "Option required", + custom_required: "Custom answer required", + }, + }, + }, + }); + return () => {}; + }, + }, +})); + +describe("Dataset review US2 component UX", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("semantic_review_surfaces_active_value_and_candidates", async () => { + requestApi.mockResolvedValue({ + field_id: "field-1", + field_name: "customer_name", + field_kind: "dimension", + verbose_name: "Customer name", + description: "Customer display name", + display_format: "text", + provenance: "dictionary_exact", + source_id: "source-1", + confidence_rank: 1, + is_locked: false, + has_conflict: false, + needs_review: false, + last_changed_by: "user", + candidates: [], + }); + + const onupdated = vi.fn(); + + render(SemanticLayerReview, { + sessionId: "session-1", + semanticSources: [ + { + source_id: "source-1", + display_name: "Trusted dictionary", + source_version: "v1", + }, + ], + fields: [ + { + field_id: "field-1", + field_name: "customer_name", + field_kind: "dimension", + verbose_name: "Customer name", + description: "Resolved from trusted dictionary", + display_format: "text", + provenance: "dictionary_exact", + source_id: "source-1", + confidence_rank: 1, + is_locked: false, + has_conflict: true, + needs_review: true, + last_changed_by: "system", + candidates: [ + { + candidate_id: "candidate-1", + match_type: "exact", + confidence_score: 0.98, + proposed_verbose_name: "Customer name", + proposed_description: "Customer display name", + proposed_display_format: "text", + status: "pending", + }, + ], + }, + ], + onupdated, + }); + + expect(screen.getByText("Review semantic field values")).toBeDefined(); + expect(screen.getByText("customer_name")).toBeDefined(); + expect(screen.getByText("Conflict")).toBeDefined(); + expect(screen.getByText("Needs review")).toBeDefined(); + expect(screen.getByText("Candidate options")).toBeDefined(); + expect(screen.getByText(/Trusted dictionary/)).toBeDefined(); + expect(screen.getByText(/v1/)).toBeDefined(); + + await fireEvent.click(screen.getByRole("button", { name: "Apply" })); + + await waitFor(() => { + expect(requestApi).toHaveBeenCalledWith( + "/dataset-orchestration/sessions/session-1/fields/field-1/semantic", + "PATCH", + { + candidate_id: "candidate-1", + lock_field: false, + }, + ); + }); + + expect(onupdated).toHaveBeenCalled(); + expect(screen.getByText("Candidate value applied.")).toBeDefined(); + }); + + it("clarification_dialog_shows_active_question_and_actions", async () => { + requestApi.mockResolvedValue({ + clarification_state: { + clarification_session: { + clarification_session_id: "clarification-1", + session_id: "session-1", + status: "active", + current_question_id: "question-2", + resolved_count: 2, + remaining_count: 1, + summary_delta: "Updated summary", + }, + current_question: { + question_id: "question-2", + clarification_session_id: "clarification-1", + topic_ref: "profile.summary", + question_text: "Which label should reviewers see?", + why_it_matters: "This label is reused in the workspace summary.", + current_guess: "Customer name", + options: [ + { + option_id: "option-1", + value: "Customer name", + label: "Customer name", + is_recommended: true, + }, + ], + }, + }, + session: { + session_id: "session-1", + readiness_state: "clarification_active", + recommended_action: "answer_next_question", + }, + changed_findings: [], + }); + + const onupdated = vi.fn(); + + render(ClarificationDialog, { + sessionId: "session-1", + clarificationState: { + clarification_session: { + clarification_session_id: "clarification-1", + session_id: "session-1", + status: "active", + current_question_id: "question-1", + resolved_count: 1, + remaining_count: 2, + summary_delta: "Awaiting confirmation", + }, + current_question: { + question_id: "question-1", + clarification_session_id: "clarification-1", + topic_ref: "profile.summary", + question_text: "Which customer label should be used?", + why_it_matters: "This label is shown to reviewers.", + current_guess: "Customer name", + options: [ + { + option_id: "option-1", + value: "Customer name", + label: "Customer name", + is_recommended: true, + }, + ], + }, + }, + onupdated, + }); + + expect(screen.getByText("Resolve one ambiguity at a time")).toBeDefined(); + expect(screen.getByText("Which customer label should be used?")).toBeDefined(); + expect(screen.getByText("This label is shown to reviewers.")).toBeDefined(); + expect(screen.getAllByText("Customer name").length).toBeGreaterThan(0); + expect(screen.getByText("Recommended")).toBeDefined(); + + await fireEvent.click( + screen.getByRole("button", { name: "Open assistant guidance" }), + ); + + expect(openAssistantChat).toHaveBeenCalledTimes(1); + expect(screen.getByText("Assistant guidance opened.")).toBeDefined(); + + await fireEvent.click( + screen.getByRole("button", { name: "Answer with selected option" }), + ); + + await waitFor(() => { + expect(requestApi).toHaveBeenCalledWith( + "/dataset-orchestration/sessions/session-1/clarification/answers", + "POST", + { + question_id: "question-1", + answer_kind: "selected", + answer_value: "Customer name", + }, + ); + }); + + expect(onupdated).toHaveBeenCalled(); + expect(screen.getByText("Clarification answer saved.")).toBeDefined(); + + mockedRequestApi.mockResolvedValueOnce({ + target_id: "question-1", + feedback: "up", + }); + + await fireEvent.click(screen.getByRole("button", { name: "Helpful" })); + + await waitFor(() => { + expect(requestApi).toHaveBeenCalledWith( + "/dataset-orchestration/sessions/session-1/clarification/questions/question-1/feedback", + "POST", + { feedback: "up" }, + ); + }); + + expect( + screen.getByText("Positive clarification feedback recorded."), + ).toBeDefined(); + }); + + it("clarification_dialog_external_fail_surfaces_inline_error", async () => { + requestApi.mockRejectedValue(new Error("Clarification backend unavailable")); + + render(ClarificationDialog, { + sessionId: "session-1", + clarificationState: { + clarification_session: { + clarification_session_id: "clarification-1", + session_id: "session-1", + status: "active", + current_question_id: "question-1", + resolved_count: 1, + remaining_count: 2, + }, + current_question: { + question_id: "question-1", + clarification_session_id: "clarification-1", + topic_ref: "profile.summary", + question_text: "Which customer label should be used?", + why_it_matters: "This label is shown to reviewers.", + current_guess: "Customer name", + options: [ + { + option_id: "option-1", + value: "Customer name", + label: "Customer name", + is_recommended: true, + }, + ], + }, + }, + }); + + await fireEvent.click( + screen.getByRole("button", { name: "Answer with selected option" }), + ); + + await waitFor(() => { + expect(screen.getByText("Clarification backend unavailable")).toBeDefined(); + }); + }); +}); +// [/DEF:DatasetReviewUs2UxTests:Module] \ No newline at end of file diff --git a/frontend/src/lib/components/dataset-review/__tests__/us3_execution_batch.ux.test.js b/frontend/src/lib/components/dataset-review/__tests__/us3_execution_batch.ux.test.js new file mode 100644 index 00000000..cba31f68 --- /dev/null +++ b/frontend/src/lib/components/dataset-review/__tests__/us3_execution_batch.ux.test.js @@ -0,0 +1,499 @@ +/** + * @vitest-environment jsdom + */ +// @ts-nocheck +// [DEF:DatasetReviewUs3UxTests:Module] +// @COMPLEXITY: 3 +// @SEMANTICS: dataset-review, execution, mapping, preview, launch, ux-tests +// @PURPOSE: Verify US3 mapping review, Superset preview, and launch confirmation UX contracts. +// @LAYER: UI +// @RELATION: [VERIFIES] ->[ExecutionMappingReview] +// @RELATION: [VERIFIES] ->[CompiledSQLPreview] +// @RELATION: [VERIFIES] ->[LaunchConfirmationPanel] +// @UX_STATE: WarningApproval -> Mapping review requires explicit approval or manual override. +// @UX_STATE: Ready -> Preview and launch panels expose reviewed Superset-backed execution context. +// @UX_STATE: Blocked -> Launch panel lists blockers instead of allowing hidden bypass. +// @TEST_CONTRACT: Us3ExecutionProps -> ObservableExecutionUx +// @TEST_SCENARIO: mapping_review_approves_warning_sensitive_row -> Approval persists and keeps blockers visible until cleared. +// @TEST_SCENARIO: preview_panel_requests_superset_compilation_and_renders_sql -> Preview stays explicitly Superset-derived. +// @TEST_SCENARIO: launch_panel_blocks_then_submits_sql_lab_launch -> Launch lists gates first and shows audited handoff after success. +// @TEST_EDGE: missing_field -> Missing preview or run context remains readable. +// @TEST_EDGE: external_fail -> Failed preview request surfaces inline error. +// @TEST_EDGE: invalid_type -> Mixed values stringify without crashing. +// @TEST_INVARIANT: execution_gates_remain_visible -> VERIFIED_BY: [mapping_review_approves_warning_sensitive_row, preview_panel_requests_superset_compilation_and_renders_sql, launch_panel_blocks_then_submits_sql_lab_launch] + +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { fireEvent, render, screen, waitFor } from "@testing-library/svelte"; +import { requestApi } from "$lib/api.js"; +import ExecutionMappingReview from "../ExecutionMappingReview.svelte"; +import CompiledSQLPreview from "../CompiledSQLPreview.svelte"; +import LaunchConfirmationPanel from "../LaunchConfirmationPanel.svelte"; + +vi.mock("$lib/api.js", () => ({ + requestApi: vi.fn(), +})); + +vi.mock("$lib/i18n", () => ({ + t: { + subscribe: (fn) => { + fn({ + common: { + error: "Common error", + cancel: "Cancel", + not_available: "N/A", + unknown: "Unknown", + }, + dataset_review: { + mapping: { + eyebrow: "Template mapping", + title: "Review filter-to-template mappings", + description: "Verify imported filter values, effective execution values, and warning-sensitive transformations before preview or launch.", + state_label: "Mapping state", + state: { + Incomplete: "Incomplete", + WarningApproval: "Approval required", + Approved: "Approved", + }, + pending_approvals_label: "Pending approvals", + required_values_label: "Missing required values", + empty: "No execution mappings are available yet.", + required_blockers_notice: "{count} required values still need attention before preview or launch can proceed.", + approval_notice: "{count} mapping transformations still need explicit approval.", + to_variable_label: "To variable", + required_badge: "Required", + approval_required_badge: "Approval required", + approved_badge: "Approved", + source_filter_label: "Source filter", + source_value_label: "Source value", + confidence_label: "Confidence", + recovery_label: "Recovery", + effective_value_label: "Effective value", + method_label: "Method", + approval_label: "Approval", + warning_label: "Warning", + transformation_note_label: "Transformation note", + missing_required_value: "This mapping still lacks a required effective value.", + manual_value_label: "Manual effective value", + manual_note_label: "Manual override note", + save_override_action: "Save manual override", + manual_override_action: "Manual override", + approve_action: "Approve mapping", + approve_all_action: "Approve all pending mappings", + approval_state: { + pending: "Pending", + approved: "Approved", + rejected: "Rejected", + not_required: "Not required", + }, + warning_level: { + none: "None", + low: "Low", + medium: "Medium", + high: "High", + }, + method: { + direct_match: "Direct match", + heuristic_match: "Heuristic match", + semantic_match: "Semantic match", + manual_override: "Manual override", + }, + messages: { + saving: "Saving mapping decision...", + approval_saved: "Mapping approval recorded.", + approval_failed: "Failed to save mapping approval.", + override_saved: "Manual mapping override saved. Preview should be refreshed.", + override_failed: "Failed to save manual mapping override.", + required_value_missing: "Provide a required value before saving the manual override.", + }, + }, + preview: { + eyebrow: "Compiled SQL preview", + title: "Review Superset-compiled SQL", + description: "Preview truth comes only from Superset. Regenerate the preview whenever mappings or required inputs change.", + state_label: "Preview state", + state: { + missing: "Missing", + pending: "Pending", + ready: "Ready", + stale: "Stale", + failed: "Error", + }, + generate_action: "Generate SQL preview", + generate_loading: "Generating preview...", + missing_body: "No Superset preview is available yet. Generate one before attempting launch.", + pending_body: "Superset is compiling the current execution context. Launch stays blocked until preview completes.", + ready_body: "This SQL preview was compiled by Superset for the current execution inputs.", + stale_body: "Mappings or effective values changed after the last successful preview. Regenerate before launch.", + error_body: "Superset could not compile the current execution context.", + compiler_label: "Compiled source", + compiled_by_superset: "Compiled by Superset", + compiled_source_unknown: "Compilation source unavailable", + fingerprint_label: "Preview fingerprint", + compiled_at_label: "Compiled at", + sql_block_title: "Superset SQL", + compiled_truth_note: "Exact SQL returned by Superset", + go_to_mapping_action: "Review mapping inputs", + review_inputs_action: "Review changed inputs", + messages: { + generated: "Superset preview refreshed.", + generate_failed: "Failed to generate Superset preview.", + }, + }, + launch: { + eyebrow: "Launch confirmation", + title: "Confirm SQL Lab launch", + description: "Launch remains blocked until preview truth, approvals, and readiness gates all match the reviewed execution context.", + state_label: "Launch state", + state: { + Blocked: "Blocked", + Ready: "Ready", + Submitted: "Submitted", + }, + blocked_title: "Launch blockers", + blocked_body: "Resolve the items below before sending this dataset run to SQL Lab.", + resolve_action: "Open related area", + dataset_ref_label: "Dataset reference", + readiness_label: "Readiness", + approved_mappings_label: "Approved mappings", + preview_fingerprint_label: "Preview fingerprint", + sql_lab_target_title: "Launch target", + sql_lab_target_body: "The canonical launch target is a Superset SQL Lab session using the reviewed preview and effective execution inputs.", + preview_status_label: "Preview status", + compiled_by_label: "Compiled by", + launch_action: "Launch dataset", + launch_loading: "Launching dataset...", + submitted_title: "Launch submitted", + submitted_body: "SQL Lab handoff and audited run context were recorded for this launch request.", + run_context_label: "Run context", + sql_lab_session_label: "SQL Lab session", + launch_status_label: "Launch status", + preview_ref_label: "Preview reference", + blockers: { + blocking_finding: "Blocking findings remain unresolved", + mapping_approval_required: "Mapping approval is still required", + preview_missing: "Superset preview is required before launch", + preview_pending: "Preview generation is still in progress", + preview_stale: "Preview no longer matches the current execution inputs", + preview_failed: "Preview failed and launch remains blocked", + readiness_not_run_ready: "Session is not yet in run-ready state", + preview_fingerprint_missing: "Preview fingerprint is missing, so launch cannot be trusted", + }, + messages: { + launch_started: "Dataset launch request sent to SQL Lab.", + launch_failed: "Failed to launch dataset in SQL Lab.", + }, + }, + }, + }); + return () => {}; + }, + }, +})); + +describe("Dataset review US3 execution UX", () => { + beforeEach(() => { + requestApi.mockReset(); + }); + + it("mapping_review_approves_warning_sensitive_row", async () => { + requestApi.mockResolvedValueOnce({ + mapping_id: "map-1", + filter_id: "filter-1", + variable_id: "var-1", + mapping_method: "direct_match", + raw_input_value: "DE", + effective_value: "DE", + transformation_note: "Trimmed imported value", + warning_level: "medium", + requires_explicit_approval: true, + approval_state: "approved", + }); + + const onupdated = vi.fn(); + + render(ExecutionMappingReview, { + sessionId: "session-1", + mappings: [ + { + mapping_id: "map-1", + filter_id: "filter-1", + variable_id: "var-1", + mapping_method: "direct_match", + raw_input_value: "DE", + effective_value: "DE", + transformation_note: "Trimmed imported value", + warning_level: "medium", + requires_explicit_approval: true, + approval_state: "pending", + }, + ], + importedFilters: [ + { + filter_id: "filter-1", + filter_name: "country", + display_name: "Country", + raw_value: "DE", + normalized_value: "DE", + confidence_state: "imported", + recovery_status: "recovered", + }, + ], + templateVariables: [ + { + variable_id: "var-1", + variable_name: "country", + is_required: true, + default_value: null, + }, + ], + onupdated, + }); + + expect(screen.getByText("Review filter-to-template mappings")).toBeDefined(); + expect(screen.getByText("Approval required")).toBeDefined(); + expect(screen.getByText("Country")).toBeDefined(); + expect(screen.getByText(/Trimmed imported value/)).toBeDefined(); + + await fireEvent.click(screen.getByRole("button", { name: "Approve mapping" })); + + await waitFor(() => { + expect(requestApi).toHaveBeenCalledWith( + "/dataset-orchestration/sessions/session-1/mappings/map-1/approve", + "POST", + { approval_note: "Trimmed imported value" }, + ); + }); + + expect(onupdated).toHaveBeenCalledWith({ + mapping: expect.objectContaining({ + mapping_id: "map-1", + approval_state: "approved", + }), + preview_state: "unchanged", + }); + expect(screen.getByText("Mapping approval recorded.")).toBeDefined(); + }); + + it("mapping_review_batch_approves_pending_rows", async () => { + requestApi.mockResolvedValueOnce([ + { + mapping_id: "map-1", + filter_id: "filter-1", + variable_id: "var-1", + mapping_method: "direct_match", + raw_input_value: "DE", + effective_value: "DE", + transformation_note: "Trimmed imported value", + warning_level: "medium", + requires_explicit_approval: true, + approval_state: "approved", + }, + ]); + + const onupdated = vi.fn(); + + render(ExecutionMappingReview, { + sessionId: "session-1", + mappings: [ + { + mapping_id: "map-1", + filter_id: "filter-1", + variable_id: "var-1", + mapping_method: "direct_match", + raw_input_value: "DE", + effective_value: "DE", + transformation_note: "Trimmed imported value", + warning_level: "medium", + requires_explicit_approval: true, + approval_state: "pending", + }, + ], + importedFilters: [ + { + filter_id: "filter-1", + filter_name: "country", + display_name: "Country", + raw_value: "DE", + normalized_value: "DE", + confidence_state: "imported", + recovery_status: "recovered", + }, + ], + templateVariables: [ + { + variable_id: "var-1", + variable_name: "country", + is_required: true, + default_value: null, + }, + ], + onupdated, + }); + + await fireEvent.click(screen.getByRole("button", { name: "Approve all pending mappings" })); + + await waitFor(() => { + expect(requestApi).toHaveBeenCalledWith( + "/dataset-orchestration/sessions/session-1/mappings/approve-batch", + "POST", + { mapping_ids: ["map-1"] }, + ); + }); + + expect(onupdated).toHaveBeenCalledWith({ + mappings: expect.any(Array), + preview_state: "unchanged", + }); + }); + + it("preview_panel_requests_superset_compilation_and_renders_sql", async () => { + requestApi.mockResolvedValueOnce({ + preview_id: "preview-1", + session_id: "session-1", + preview_status: "ready", + compiled_sql: "SELECT * FROM sales WHERE country = 'DE'", + preview_fingerprint: "fingerprint-1", + compiled_by: "superset", + compiled_at: "2026-03-17T09:00:00Z", + }); + + const onupdated = vi.fn(); + + render(CompiledSQLPreview, { + sessionId: "session-1", + preview: null, + previewState: "missing", + onupdated, + }); + + expect(screen.getByText("Review Superset-compiled SQL")).toBeDefined(); + expect(screen.getByText("No Superset preview is available yet. Generate one before attempting launch.")).toBeDefined(); + + await fireEvent.click(screen.getByRole("button", { name: "Generate SQL preview" })); + + await waitFor(() => { + expect(requestApi).toHaveBeenCalledWith( + "/dataset-orchestration/sessions/session-1/preview", + "POST", + ); + }); + + expect(onupdated).toHaveBeenCalledWith({ + preview: expect.objectContaining({ + preview_id: "preview-1", + compiled_by: "superset", + }), + preview_state: "ready", + }); + + requestApi.mockRejectedValueOnce(new Error("Superset compile failed")); + + const onjump = vi.fn(); + render(CompiledSQLPreview, { + sessionId: "session-1", + preview: { + preview_id: "preview-err", + preview_status: "failed", + preview_fingerprint: "fingerprint-err", + compiled_by: "superset", + error_details: "Variable country is invalid", + }, + previewState: "failed", + onjump, + }); + + expect(screen.getByText("Variable country is invalid")).toBeDefined(); + + await fireEvent.click(screen.getByRole("button", { name: "Review mapping inputs" })); + expect(onjump).toHaveBeenCalledWith({ target: "mapping" }); + }); + + it("launch_panel_blocks_then_submits_sql_lab_launch", async () => { + const onjump = vi.fn(); + const { rerender } = render(LaunchConfirmationPanel, { + sessionId: "session-1", + session: { + dataset_ref: "public.sales", + readiness_state: "mapping_review_needed", + }, + findings: [ + { + severity: "blocking", + resolution_state: "open", + title: "Missing required value", + message: "country is required", + code: "REQ_COUNTRY", + }, + ], + mappings: [ + { + mapping_id: "map-1", + requires_explicit_approval: true, + approval_state: "pending", + transformation_note: "Europe -> EU", + }, + ], + preview: null, + previewState: "missing", + onjump, + }); + + expect(screen.getByText("Launch blockers")).toBeDefined(); + expect(screen.getByText("Blocking findings remain unresolved")).toBeDefined(); + expect(screen.getByText("Mapping approval is still required")).toBeDefined(); + expect(screen.getByText("Superset preview is required before launch")).toBeDefined(); + + await fireEvent.click(screen.getAllByRole("button", { name: "Open related area" })[1]); + expect(onjump).toHaveBeenCalledWith({ target: "mapping" }); + + requestApi.mockResolvedValueOnce({ + run_context: { + run_context_id: "run-1", + sql_lab_session_ref: "sql-lab-77", + launch_status: "started", + preview_id: "preview-1", + }, + }); + + await rerender({ + sessionId: "session-1", + session: { + dataset_ref: "public.sales", + readiness_state: "run_ready", + }, + findings: [], + mappings: [ + { + mapping_id: "map-1", + requires_explicit_approval: true, + approval_state: "approved", + transformation_note: "Europe -> EU", + }, + ], + preview: { + preview_id: "preview-1", + preview_status: "ready", + preview_fingerprint: "fingerprint-1", + compiled_by: "superset", + }, + previewState: "ready", + onupdated: vi.fn(), + onjump, + }); + + expect(screen.getByText("Confirm SQL Lab launch")).toBeDefined(); + expect(screen.getByText("Launch state: Ready")).toBeDefined(); + + await fireEvent.click(screen.getByRole("button", { name: "Launch dataset" })); + + await waitFor(() => { + expect(requestApi).toHaveBeenCalledWith( + "/dataset-orchestration/sessions/session-1/launch", + "POST", + ); + }); + + expect(screen.getByText("Dataset launch request sent to SQL Lab.")).toBeDefined(); + }); +}); +// [/DEF:DatasetReviewUs3UxTests:Module] \ No newline at end of file diff --git a/frontend/src/lib/i18n/locales/en.json b/frontend/src/lib/i18n/locales/en.json index 40d8ea76..655f8498 100644 --- a/frontend/src/lib/i18n/locales/en.json +++ b/frontend/src/lib/i18n/locales/en.json @@ -664,6 +664,11 @@ "export_failed": "Failed to export the requested review artifact.", "empty_state_title": "No session loaded", "state_label": "Workspace", + "state": { + "empty": "Empty", + "importing": "Importing", + "review": "Review" + }, "readiness_label": "Readiness", "source_badge_fallback": "review session", "save_session_action": "Save session", @@ -721,6 +726,8 @@ "preview_compiler_label": "Compiled by", "preview_pending_note": "Compiled preview is not part of this US1 batch yet, but the workspace keeps the state visible when present.", "jump_target_label": "Focused area", + "semantic_review_visible": "Semantic review is active for field-level source decisions.", + "clarification_visible": "Clarification is available with one active question at a time.", "resume_action": "Resume session", "pause_action": "Pause session", "actions": { @@ -765,6 +772,245 @@ "low_confidence": "Low confidence", "unresolved": "Unresolved" } + }, + "semantics": { + "eyebrow": "Semantic layer", + "title": "Review semantic field values", + "description": "Compare the active semantic value with ranked candidates, preserve provenance, and lock or unlock manual decisions explicitly.", + "field_count_label": "Fields in review", + "empty": "No semantic fields need review right now.", + "unknown_source": "Unknown source", + "active_value_label": "Active semantic value", + "provenance_label": "Provenance", + "confidence_label": "Confidence", + "confidence_unset": "No confidence rank", + "confidence_rank_label": "Rank", + "source_label": "Source", + "changed_by_label": "Changed by", + "locked_badge": "Locked", + "unlocked_badge": "Unlocked", + "conflict_badge": "Conflict", + "needs_review_badge": "Needs review", + "manual_verbose_name_label": "Manual verbose name", + "manual_description_label": "Manual description", + "manual_display_format_label": "Manual display format", + "manual_override_action": "Manual override", + "approve_all_action": "Approve visible candidates", + "save_manual_action": "Save manual value", + "lock_action": "Lock field", + "unlock_action": "Unlock field", + "feedback_up_action": "👍", + "feedback_down_action": "👎", + "candidates_title": "Candidate options", + "candidates_empty": "No ranked candidate options were returned for this field.", + "candidate_description_empty": "No description proposed for this candidate.", + "display_format_label": "Display format", + "score_label": "Score", + "apply_candidate_action": "Apply", + "apply_and_lock_action": "Apply & lock", + "empty_value": "No current value", + "provenance": { + "unresolved": "Unresolved", + "dictionary_exact": "Dictionary exact match", + "reference_imported": "Imported reference", + "ai_generated": "AI generated", + "fuzzy_inferred": "Fuzzy inferred", + "manual_override": "Manual override" + }, + "candidate_status": { + "pending": "Pending", + "accepted": "Accepted", + "superseded": "Superseded", + "rejected": "Rejected" + }, + "messages": { + "saving": "Saving semantic decision...", + "save_failed": "Failed to save semantic decision.", + "manual_override_required": "Provide at least one manual semantic value before saving.", + "manual_saved": "Manual semantic value saved and locked.", + "candidate_applied": "Candidate value applied.", + "candidate_locked": "Candidate value applied and locked.", + "locked": "Field locked against silent overwrite.", + "unlocked": "Field unlocked for renewed review.", + "feedback_up": "Positive feedback recorded for this semantic value.", + "feedback_down": "Negative feedback recorded for this semantic value." + } + }, + "clarification": { + "eyebrow": "Clarification", + "title": "Resolve one ambiguity at a time", + "description": "Answer the active clarification question, skip it, or send it to expert review while progress stays resumable.", + "progress_label": "Clarification progress", + "state_label": "Dialog state", + "status_label": "Status", + "remaining_label": "Remaining questions", + "summary_delta_label": "Summary delta", + "active_question_label": "Active question", + "why_it_matters_label": "Why it matters", + "current_guess_label": "Current guess", + "current_guess_empty": "No current guess is available.", + "topic_label": "Topic reference", + "options_title": "Available options", + "recommended_badge": "Recommended", + "custom_answer_label": "Custom answer", + "answer_action": "Answer with selected option", + "custom_answer_action": "Submit custom answer", + "skip_action": "Skip for now", + "assistant_action": "Open assistant guidance", + "expert_review_action": "Send to expert review", + "resume_action": "Resume clarification", + "completed": "No active clarification question remains. The session can be resumed later if new ambiguity appears.", + "messages": { + "saving": "Saving clarification answer...", + "saved": "Clarification answer saved.", + "skipped": "Question skipped and left resumable.", + "expert_review_requested": "Question escalated to expert review.", + "assistant_opened": "Assistant guidance opened for this clarification step.", + "resumed": "Clarification resumed from the highest-priority unresolved question.", + "resume_failed": "Failed to resume clarification.", + "save_failed": "Failed to save clarification answer.", + "option_required": "Select an option before submitting the answer.", + "custom_required": "Enter a custom answer before submitting." + } + }, + "mapping": { + "eyebrow": "Template mapping", + "title": "Review filter-to-template mappings", + "description": "Verify imported filter values, effective execution values, and warning-sensitive transformations before preview or launch.", + "state_label": "Mapping state", + "state": { + "Incomplete": "Incomplete", + "WarningApproval": "Approval required", + "Approved": "Approved" + }, + "pending_approvals_label": "Pending approvals", + "required_values_label": "Missing required values", + "empty": "No execution mappings are available yet.", + "required_blockers_notice": "{count} required values still need attention before preview or launch can proceed.", + "approval_notice": "{count} mapping transformations still need explicit approval.", + "to_variable_label": "To variable", + "required_badge": "Required", + "approval_required_badge": "Approval required", + "approved_badge": "Approved", + "source_filter_label": "Source filter", + "source_value_label": "Source value", + "confidence_label": "Confidence", + "recovery_label": "Recovery", + "effective_value_label": "Effective value", + "method_label": "Method", + "approval_label": "Approval", + "warning_label": "Warning", + "transformation_note_label": "Transformation note", + "missing_required_value": "This mapping still lacks a required effective value.", + "manual_value_label": "Manual effective value", + "manual_note_label": "Manual override note", + "save_override_action": "Save manual override", + "manual_override_action": "Manual override", + "approve_action": "Approve mapping", + "approve_all_action": "Approve all pending mappings", + "approval_state": { + "pending": "Pending", + "approved": "Approved", + "rejected": "Rejected", + "not_required": "Not required" + }, + "warning_level": { + "none": "None", + "low": "Low", + "medium": "Medium", + "high": "High" + }, + "method": { + "direct_match": "Direct match", + "heuristic_match": "Heuristic match", + "semantic_match": "Semantic match", + "manual_override": "Manual override" + }, + "messages": { + "saving": "Saving mapping decision...", + "approval_saved": "Mapping approval recorded.", + "approval_failed": "Failed to save mapping approval.", + "override_saved": "Manual mapping override saved. Preview should be refreshed.", + "override_failed": "Failed to save manual mapping override.", + "required_value_missing": "Provide a required value before saving the manual override." + } + }, + "preview": { + "eyebrow": "Compiled SQL preview", + "title": "Review Superset-compiled SQL", + "description": "Preview truth comes only from Superset. Regenerate the preview whenever mappings or required inputs change.", + "state_label": "Preview state", + "state": { + "missing": "Missing", + "pending": "Pending", + "ready": "Ready", + "stale": "Stale", + "failed": "Error" + }, + "generate_action": "Generate SQL preview", + "generate_loading": "Generating preview...", + "missing_body": "No Superset preview is available yet. Generate one before attempting launch.", + "pending_body": "Superset is compiling the current execution context. Launch stays blocked until preview completes.", + "ready_body": "This SQL preview was compiled by Superset for the current execution inputs.", + "stale_body": "Mappings or effective values changed after the last successful preview. Regenerate before launch.", + "error_body": "Superset could not compile the current execution context.", + "compiler_label": "Compiled source", + "compiled_by_superset": "Compiled by Superset", + "compiled_source_unknown": "Compilation source unavailable", + "fingerprint_label": "Preview fingerprint", + "compiled_at_label": "Compiled at", + "sql_block_title": "Superset SQL", + "compiled_truth_note": "Exact SQL returned by Superset", + "go_to_mapping_action": "Review mapping inputs", + "review_inputs_action": "Review changed inputs", + "messages": { + "generated": "Superset preview refreshed.", + "generate_failed": "Failed to generate Superset preview." + } + }, + "launch": { + "eyebrow": "Launch confirmation", + "title": "Confirm SQL Lab launch", + "description": "Launch remains blocked until preview truth, approvals, and readiness gates all match the reviewed execution context.", + "state_label": "Launch state", + "state": { + "Blocked": "Blocked", + "Ready": "Ready", + "Submitted": "Submitted" + }, + "blocked_title": "Launch blockers", + "blocked_body": "Resolve the items below before sending this dataset run to SQL Lab.", + "resolve_action": "Open related area", + "dataset_ref_label": "Dataset reference", + "readiness_label": "Readiness", + "approved_mappings_label": "Approved mappings", + "preview_fingerprint_label": "Preview fingerprint", + "sql_lab_target_title": "Launch target", + "sql_lab_target_body": "The canonical launch target is a Superset SQL Lab session using the reviewed preview and effective execution inputs.", + "preview_status_label": "Preview status", + "compiled_by_label": "Compiled by", + "launch_action": "Launch dataset", + "launch_loading": "Launching dataset...", + "submitted_title": "Launch submitted", + "submitted_body": "SQL Lab handoff and audited run context were recorded for this launch request.", + "run_context_label": "Run context", + "sql_lab_session_label": "SQL Lab session", + "launch_status_label": "Launch status", + "preview_ref_label": "Preview reference", + "blockers": { + "blocking_finding": "Blocking findings remain unresolved", + "mapping_approval_required": "Mapping approval is still required", + "preview_missing": "Superset preview is required before launch", + "preview_pending": "Preview generation is still in progress", + "preview_stale": "Preview no longer matches the current execution inputs", + "preview_failed": "Preview failed and launch remains blocked", + "readiness_not_run_ready": "Session is not yet in run-ready state", + "preview_fingerprint_missing": "Preview fingerprint is missing, so launch cannot be trusted" + }, + "messages": { + "launch_started": "Dataset launch request sent to SQL Lab.", + "launch_failed": "Failed to launch dataset in SQL Lab." + } } }, "tasks": { diff --git a/frontend/src/lib/i18n/locales/ru.json b/frontend/src/lib/i18n/locales/ru.json index 7a869a4e..230a4f96 100644 --- a/frontend/src/lib/i18n/locales/ru.json +++ b/frontend/src/lib/i18n/locales/ru.json @@ -662,6 +662,11 @@ "export_failed": "Не удалось экспортировать выбранный артефакт review.", "empty_state_title": "Сессия не загружена", "state_label": "Workspace", + "state": { + "empty": "Пусто", + "importing": "Импорт", + "review": "Проверка" + }, "readiness_label": "Готовность", "source_badge_fallback": "review-сессия", "save_session_action": "Сохранить сессию", @@ -719,6 +724,8 @@ "preview_compiler_label": "Скомпилировано", "preview_pending_note": "Скомпилированный preview не входит в этот пакет US1, но workspace сохраняет его видимым, если он уже существует.", "jump_target_label": "Выбранная зона", + "semantic_review_visible": "Семантический review активен для полей с решениями по источникам.", + "clarification_visible": "Уточнение доступно в режиме одного активного вопроса.", "resume_action": "Возобновить сессию", "pause_action": "Поставить сессию на паузу", "actions": { @@ -763,6 +770,245 @@ "low_confidence": "Низкая уверенность", "unresolved": "Не разрешено" } + }, + "semantics": { + "eyebrow": "Семантический слой", + "title": "Проверьте семантические значения полей", + "description": "Сравнивайте активное семантическое значение с ранжированными кандидатами, сохраняйте provenance и явно блокируйте или разблокируйте ручные решения.", + "field_count_label": "Поля в review", + "empty": "Семантические поля сейчас не требуют review.", + "unknown_source": "Неизвестный источник", + "active_value_label": "Активное семантическое значение", + "provenance_label": "Provenance", + "confidence_label": "Уверенность", + "confidence_unset": "Ранг уверенности отсутствует", + "confidence_rank_label": "Ранг", + "source_label": "Источник", + "changed_by_label": "Изменил", + "locked_badge": "Заблокировано", + "unlocked_badge": "Разблокировано", + "conflict_badge": "Конфликт", + "needs_review_badge": "Нужен review", + "manual_verbose_name_label": "Ручное verbose name", + "manual_description_label": "Ручное описание", + "manual_display_format_label": "Ручной display format", + "manual_override_action": "Ручное переопределение", + "approve_all_action": "Подтвердить видимые кандидаты", + "save_manual_action": "Сохранить ручное значение", + "lock_action": "Заблокировать поле", + "unlock_action": "Разблокировать поле", + "feedback_up_action": "👍", + "feedback_down_action": "👎", + "candidates_title": "Варианты кандидатов", + "candidates_empty": "Для этого поля не возвращены ранжированные кандидаты.", + "candidate_description_empty": "Для этого кандидата описание не предложено.", + "display_format_label": "Формат отображения", + "score_label": "Оценка", + "apply_candidate_action": "Применить", + "apply_and_lock_action": "Применить и заблокировать", + "empty_value": "Текущее значение отсутствует", + "provenance": { + "unresolved": "Не разрешено", + "dictionary_exact": "Точное совпадение словаря", + "reference_imported": "Импортированная ссылка", + "ai_generated": "Сгенерировано AI", + "fuzzy_inferred": "Нечеткий вывод", + "manual_override": "Ручное переопределение" + }, + "candidate_status": { + "pending": "В ожидании", + "accepted": "Принято", + "superseded": "Заменено", + "rejected": "Отклонено" + }, + "messages": { + "saving": "Сохранение семантического решения...", + "save_failed": "Не удалось сохранить семантическое решение.", + "manual_override_required": "Перед сохранением укажите хотя бы одно ручное семантическое значение.", + "manual_saved": "Ручное семантическое значение сохранено и заблокировано.", + "candidate_applied": "Значение кандидата применено.", + "candidate_locked": "Значение кандидата применено и заблокировано.", + "locked": "Поле заблокировано от тихой перезаписи.", + "unlocked": "Поле разблокировано для повторного review.", + "feedback_up": "Позитивный feedback для этого семантического значения сохранен.", + "feedback_down": "Негативный feedback для этого семантического значения сохранен." + } + }, + "clarification": { + "eyebrow": "Уточнение", + "title": "Разрешайте неоднозначности по одному вопросу", + "description": "Ответьте на активный вопрос уточнения, пропустите его или отправьте на экспертный review, сохраняя возможность возобновления.", + "progress_label": "Прогресс уточнения", + "state_label": "Состояние диалога", + "status_label": "Статус", + "remaining_label": "Оставшиеся вопросы", + "summary_delta_label": "Изменение summary", + "active_question_label": "Активный вопрос", + "why_it_matters_label": "Почему это важно", + "current_guess_label": "Текущая гипотеза", + "current_guess_empty": "Текущая гипотеза отсутствует.", + "topic_label": "Ссылка на тему", + "options_title": "Доступные варианты", + "recommended_badge": "Рекомендовано", + "custom_answer_label": "Свой ответ", + "answer_action": "Ответить выбранным вариантом", + "custom_answer_action": "Отправить свой ответ", + "skip_action": "Пропустить пока", + "assistant_action": "Открыть помощь ассистента", + "expert_review_action": "Передать на экспертный review", + "resume_action": "Возобновить уточнение", + "completed": "Активных вопросов уточнения не осталось. Сессию можно возобновить позже, если появится новая неоднозначность.", + "messages": { + "saving": "Сохранение ответа на уточнение...", + "saved": "Ответ на уточнение сохранен.", + "skipped": "Вопрос пропущен и сохранен как возобновляемый.", + "expert_review_requested": "Вопрос отправлен на экспертный review.", + "assistant_opened": "Для этого шага уточнения открыта помощь ассистента.", + "resumed": "Уточнение возобновлено с самого приоритетного неразрешенного вопроса.", + "resume_failed": "Не удалось возобновить уточнение.", + "save_failed": "Не удалось сохранить ответ на уточнение.", + "option_required": "Перед отправкой выберите один вариант.", + "custom_required": "Перед отправкой введите свой ответ." + } + }, + "mapping": { + "eyebrow": "Маппинг шаблонов", + "title": "Проверьте маппинг фильтров и шаблонов", + "description": "Проверьте импортированные значения фильтров, итоговые execution values и warning-преобразования перед preview или запуском.", + "state_label": "Состояние маппинга", + "state": { + "Incomplete": "Неполный", + "WarningApproval": "Требуется подтверждение", + "Approved": "Подтверждено" + }, + "pending_approvals_label": "Ожидают подтверждения", + "required_values_label": "Отсутствуют обязательные значения", + "empty": "Execution mappings пока отсутствуют.", + "required_blockers_notice": "Еще {count} обязательных значений требуют внимания перед preview или запуском.", + "approval_notice": "Еще {count} преобразований маппинга требуют явного подтверждения.", + "to_variable_label": "В переменную", + "required_badge": "Обязательно", + "approval_required_badge": "Нужно подтверждение", + "approved_badge": "Подтверждено", + "source_filter_label": "Исходный фильтр", + "source_value_label": "Исходное значение", + "confidence_label": "Уверенность", + "recovery_label": "Восстановление", + "effective_value_label": "Итоговое значение", + "method_label": "Метод", + "approval_label": "Подтверждение", + "warning_label": "Предупреждение", + "transformation_note_label": "Комментарий к преобразованию", + "missing_required_value": "Для этого маппинга все еще отсутствует обязательное итоговое значение.", + "manual_value_label": "Ручное итоговое значение", + "manual_note_label": "Комментарий к ручному переопределению", + "save_override_action": "Сохранить ручное переопределение", + "manual_override_action": "Ручное переопределение", + "approve_action": "Подтвердить маппинг", + "approve_all_action": "Подтвердить все ожидающие маппинги", + "approval_state": { + "pending": "Ожидает", + "approved": "Подтверждено", + "rejected": "Отклонено", + "not_required": "Не требуется" + }, + "warning_level": { + "none": "Нет", + "low": "Низкий", + "medium": "Средний", + "high": "Высокий" + }, + "method": { + "direct_match": "Прямое совпадение", + "heuristic_match": "Эвристическое совпадение", + "semantic_match": "Семантическое совпадение", + "manual_override": "Ручное переопределение" + }, + "messages": { + "saving": "Сохранение решения по маппингу...", + "approval_saved": "Подтверждение маппинга сохранено.", + "approval_failed": "Не удалось сохранить подтверждение маппинга.", + "override_saved": "Ручное переопределение маппинга сохранено. Preview нужно обновить.", + "override_failed": "Не удалось сохранить ручное переопределение маппинга.", + "required_value_missing": "Перед сохранением ручного переопределения укажите обязательное значение." + } + }, + "preview": { + "eyebrow": "Скомпилированный SQL preview", + "title": "Проверьте SQL, скомпилированный Superset", + "description": "Истина preview приходит только от Superset. Перегенерируйте preview после изменений маппинга или обязательных входных значений.", + "state_label": "Состояние preview", + "state": { + "missing": "Отсутствует", + "pending": "Генерируется", + "ready": "Готов", + "stale": "Устарел", + "failed": "Ошибка" + }, + "generate_action": "Сгенерировать SQL preview", + "generate_loading": "Генерация preview...", + "missing_body": "Preview от Superset пока отсутствует. Сгенерируйте его перед запуском.", + "pending_body": "Superset компилирует текущий execution context. Запуск остается заблокированным до завершения preview.", + "ready_body": "Этот SQL preview был скомпилирован Superset для текущих execution inputs.", + "stale_body": "После последнего успешного preview были изменены маппинги или итоговые значения. Перед запуском нужно перегенерировать preview.", + "error_body": "Superset не смог скомпилировать текущий execution context.", + "compiler_label": "Источник компиляции", + "compiled_by_superset": "Скомпилировано Superset", + "compiled_source_unknown": "Источник компиляции недоступен", + "fingerprint_label": "Fingerprint preview", + "compiled_at_label": "Время компиляции", + "sql_block_title": "SQL от Superset", + "compiled_truth_note": "Точный SQL, возвращенный Superset", + "go_to_mapping_action": "Проверить mapping inputs", + "review_inputs_action": "Проверить измененные inputs", + "messages": { + "generated": "Preview от Superset обновлен.", + "generate_failed": "Не удалось сгенерировать preview от Superset." + } + }, + "launch": { + "eyebrow": "Подтверждение запуска", + "title": "Подтвердите запуск в SQL Lab", + "description": "Запуск остается заблокированным, пока preview, approvals и readiness gates не соответствуют проверенному execution context.", + "state_label": "Состояние запуска", + "state": { + "Blocked": "Заблокирован", + "Ready": "Готов", + "Submitted": "Отправлен" + }, + "blocked_title": "Блокеры запуска", + "blocked_body": "Разрешите следующие пункты перед отправкой запуска датасета в SQL Lab.", + "resolve_action": "Открыть связанный раздел", + "dataset_ref_label": "Ссылка на датасет", + "readiness_label": "Готовность", + "approved_mappings_label": "Подтвержденные маппинги", + "preview_fingerprint_label": "Fingerprint preview", + "sql_lab_target_title": "Цель запуска", + "sql_lab_target_body": "Каноническая цель запуска — сессия Superset SQL Lab с использованием проверенного preview и итоговых execution inputs.", + "preview_status_label": "Статус preview", + "compiled_by_label": "Скомпилировано", + "launch_action": "Запустить датасет", + "launch_loading": "Запуск датасета...", + "submitted_title": "Запуск отправлен", + "submitted_body": "Передача в SQL Lab и аудитируемый run context были записаны для этого запроса запуска.", + "run_context_label": "Run context", + "sql_lab_session_label": "Сессия SQL Lab", + "launch_status_label": "Статус запуска", + "preview_ref_label": "Ссылка на preview", + "blockers": { + "blocking_finding": "Есть неразрешенные блокирующие findings", + "mapping_approval_required": "Маппинг все еще требует подтверждения", + "preview_missing": "Перед запуском требуется preview от Superset", + "preview_pending": "Preview все еще генерируется", + "preview_stale": "Preview больше не соответствует текущим execution inputs", + "preview_failed": "Preview завершился ошибкой, запуск заблокирован", + "readiness_not_run_ready": "Сессия еще не находится в состоянии run-ready", + "preview_fingerprint_missing": "Fingerprint preview отсутствует, поэтому запуску нельзя доверять" + }, + "messages": { + "launch_started": "Запрос запуска датасета отправлен в SQL Lab.", + "launch_failed": "Не удалось запустить датасет в SQL Lab." + } } }, "tasks": { diff --git a/frontend/src/routes/datasets/review/[id]/+page.svelte b/frontend/src/routes/datasets/review/[id]/+page.svelte index 49997698..ddd9efd8 100644 --- a/frontend/src/routes/datasets/review/[id]/+page.svelte +++ b/frontend/src/routes/datasets/review/[id]/+page.svelte @@ -14,8 +14,8 @@ - - + + @@ -26,6 +26,11 @@ import { api } from "$lib/api.js"; import SourceIntakePanel from "$lib/components/dataset-review/SourceIntakePanel.svelte"; import ValidationFindingsPanel from "$lib/components/dataset-review/ValidationFindingsPanel.svelte"; + import SemanticLayerReview from "$lib/components/dataset-review/SemanticLayerReview.svelte"; + import ClarificationDialog from "$lib/components/dataset-review/ClarificationDialog.svelte"; + import ExecutionMappingReview from "$lib/components/dataset-review/ExecutionMappingReview.svelte"; + import CompiledSQLPreview from "$lib/components/dataset-review/CompiledSQLPreview.svelte"; + import LaunchConfirmationPanel from "$lib/components/dataset-review/LaunchConfirmationPanel.svelte"; import { setError, setLoading, @@ -52,6 +57,9 @@ let summaryDraft = $state(""); let jumpTarget = $state(""); let intakeAcknowledgment = $state(""); + let clarificationState = $state(null); + let previewUiState = $state(""); + let launchResult = $state(null); const routeSessionId = $derived(page.params.id || ""); const environments = $derived(environmentContextState.current?.environments || []); @@ -62,10 +70,10 @@ ); const currentWorkspaceState = $derived( !session - ? (isSubmitting ? "Importing" : "Empty") + ? (isSubmitting ? "importing" : "empty") : session.readiness_state === "importing" - ? "Importing" - : "Review", + ? "importing" + : "review", ); const profile = $derived(session?.profile || null); const findings = $derived(session?.findings || []); @@ -73,8 +81,14 @@ const latestPreview = $derived( session?.previews?.length ? session.previews[session.previews.length - 1] : null, ); + const executionMappings = $derived(session?.execution_mappings || []); + const templateVariables = $derived(session?.template_variables || []); + const latestRunContext = $derived( + session?.run_contexts?.length ? session.run_contexts[session.run_contexts.length - 1] : null, + ); const exportDisabled = $derived(!session || isExporting); const saveDisabled = $derived(!session || isSubmitting); + const launchDisabled = $derived(!session || isSubmitting || isExporting); const blockingCount = $derived( findings.filter((finding) => finding.severity === "blocking").length, ); @@ -104,6 +118,14 @@ const importMilestones = $derived(buildImportMilestones()); const recentActions = $derived(buildRecentActions()); + function getWorkspaceStateLabel(state) { + return $t.dataset_review?.workspace?.state?.[state] || state; + } + + function getFallbackLabel(key, fallback = "") { + return $t.dataset_review?.workspace?.fallbacks?.[key] || fallback; + } + function getRecommendedActionLabel(action) { const normalized = String(action || "import_from_superset"); return $t.dataset_review?.workspace?.actions?.[normalized] || normalized; @@ -114,7 +136,7 @@ const filtersDone = importedFilters.length > 0; const variablesDone = Boolean(importedFilters.length || latestPreview); const semanticsDone = Boolean(profile?.business_summary || findings.length > 0); - const importState = currentWorkspaceState === "Importing"; + const importState = currentWorkspaceState === "importing"; return [ { @@ -154,7 +176,10 @@ { key: "source", title: $t.dataset_review?.workspace?.timeline?.source, - detail: session.source_input || session.dataset_ref || "—", + detail: + session.source_input || + session.dataset_ref || + getFallbackLabel("empty_value", "—"), }, { key: "status", @@ -197,6 +222,7 @@ async function loadSessionDetail(sessionId) { if (!sessionId) { session = null; + clarificationState = null; loadError = ""; summaryDraft = ""; resetSession(); @@ -210,8 +236,11 @@ try { const detail = await api.fetchApi(`/dataset-orchestration/sessions/${sessionId}`); session = detail; + previewUiState = ""; + launchResult = null; summaryDraft = detail?.profile?.business_summary || ""; setSession(detail); + await loadClarificationState(detail); } catch (error) { loadError = error?.message || @@ -223,6 +252,21 @@ } } + async function loadClarificationState(sessionDetail) { + if (!sessionDetail?.session_id || !sessionDetail?.clarification_sessions?.length) { + clarificationState = null; + return; + } + + try { + clarificationState = await api.fetchApi( + `/dataset-orchestration/sessions/${sessionDetail.session_id}/clarification`, + ); + } catch (error) { + clarificationState = null; + } + } + async function bootstrapWorkspace() { isBootstrapping = true; await initializeEnvironmentContext(); @@ -341,6 +385,121 @@ isEditingSummary = false; } + function handleSemanticUpdated(updatedField) { + if (!session || !updatedField) { + return; + } + + if (Array.isArray(updatedField.fields)) { + const updatedLookup = new Map(updatedField.fields.map((field) => [field.field_id, field])); + session = { + ...session, + semantic_fields: (session.semantic_fields || []).map((field) => + updatedLookup.get(field.field_id) || field, + ), + }; + return; + } + + if (!updatedField?.field_id) { + return; + } + + session = { + ...session, + semantic_fields: (session.semantic_fields || []).map((field) => + field.field_id === updatedField.field_id ? updatedField : field, + ), + }; + } + + function handleClarificationUpdated(result) { + if (!result) { + return; + } + + if (result.clarification_state) { + clarificationState = result.clarification_state; + } else if (result.clarification_session || result.current_question !== undefined) { + clarificationState = result; + } + + if (result.session && session) { + session = { + ...session, + ...result.session, + }; + } + } + + function handleExecutionMappingUpdated(result) { + if (!result || !session) { + return; + } + + if (Array.isArray(result.mappings)) { + const updatedLookup = new Map(result.mappings.map((mapping) => [mapping.mapping_id, mapping])); + session = { + ...session, + execution_mappings: executionMappings.map((mapping) => + updatedLookup.get(mapping.mapping_id) || mapping, + ), + }; + } else if (result.mapping) { + session = { + ...session, + execution_mappings: executionMappings.map((mapping) => + mapping.mapping_id === result.mapping.mapping_id ? result.mapping : mapping, + ), + }; + } else { + return; + } + + if (result.preview_state) { + previewUiState = result.preview_state; + } + } + + function handlePreviewUpdated(result) { + if (!session || !result?.preview) { + return; + } + + previewUiState = result.preview_state || result.preview.preview_status || ""; + const existingPreviews = session.previews || []; + const nextPreviews = existingPreviews.some( + (item) => item.preview_id === result.preview.preview_id, + ) + ? existingPreviews.map((item) => + item.preview_id === result.preview.preview_id ? result.preview : item, + ) + : [...existingPreviews, result.preview]; + + session = { + ...session, + previews: nextPreviews, + readiness_state: + result.preview.preview_status === "ready" + ? session.readiness_state === "run_ready" + ? "run_ready" + : "compiled_preview_ready" + : session.readiness_state, + }; + } + + function handleLaunchUpdated(result) { + if (!result) { + return; + } + + launchResult = result.launch_result || result; + } + + function handleSectionJump(event) { + jumpTarget = event?.target || ""; + } + function primaryActionHandler() { if (!session) { return; @@ -351,6 +510,21 @@ return; } + if (session.recommended_action === "generate_sql_preview") { + jumpTarget = "sql-preview"; + return; + } + + if (session.recommended_action === "approve_mapping") { + jumpTarget = "mapping"; + return; + } + + if (session.recommended_action === "launch_dataset") { + jumpTarget = "launch"; + return; + } + if (session.status === "active") { void updateSessionLifecycle("paused"); } @@ -383,7 +557,7 @@ {/if} - {$t.dataset_review?.workspace?.state_label}: {currentWorkspaceState} + {$t.dataset_review?.workspace?.state_label}: {getWorkspaceStateLabel(currentWorkspaceState)} {#if session} @@ -429,7 +603,7 @@ onsubmit={handleSourceSubmit} /> - {#if currentWorkspaceState === "Importing"} + {#if currentWorkspaceState === "importing"}
@@ -444,7 +618,7 @@

- {currentWorkspaceState} + {getWorkspaceStateLabel(currentWorkspaceState)}
@@ -499,7 +673,7 @@ {$t.dataset_review?.workspace?.import_status_title}
- {currentWorkspaceState} + {getWorkspaceStateLabel(currentWorkspaceState)}
@@ -571,7 +745,8 @@ {profile?.dataset_name || session.dataset_ref}

- {profile?.schema_name || "—"} • {profile?.database_name || "—"} + {profile?.schema_name || getFallbackLabel("empty_value", "—")} • {profile?.database_name || + getFallbackLabel("empty_value", "—")}

@@ -741,6 +916,32 @@ {/if} + + + {#if clarificationState || session.clarification_sessions?.length} + + {/if} + + + -
-

- {$t.dataset_review?.workspace?.preview_title} -

- {#if latestPreview} -
-

- {$t.dataset_review?.workspace?.preview_status_label}: - {latestPreview.preview_status} -

-

- {$t.dataset_review?.workspace?.preview_compiler_label}: - {latestPreview.compiled_by} -

-
- {:else} -

- {$t.dataset_review?.workspace?.preview_pending_note} -

- {/if} -
+ + + {/if} diff --git a/frontend/src/routes/datasets/review/[id]/__tests__/dataset_review_workspace.ux.test.js b/frontend/src/routes/datasets/review/[id]/__tests__/dataset_review_workspace.ux.test.js index 6a4e1678..31f211bc 100644 --- a/frontend/src/routes/datasets/review/[id]/__tests__/dataset_review_workspace.ux.test.js +++ b/frontend/src/routes/datasets/review/[id]/__tests__/dataset_review_workspace.ux.test.js @@ -5,15 +5,16 @@ // [DEF:DatasetReviewWorkspaceUxTests:Module] // @COMPLEXITY: 3 // @SEMANTICS: dataset-review, workspace, route, ux-tests, review-state, exports, recovery -// @PURPOSE: Verify US1 dataset review workspace flow evidence for empty, load, import, recovery, and export behaviors. +// @PURPOSE: Verify US1-safe and US2-integrated dataset review workspace flow evidence for empty, load, import, semantics, clarification, and export behaviors. // @LAYER: UI // @RELATION: [VERIFIES] ->[DatasetReviewWorkspace] // @UX_STATE: Empty -> Intake-first workspace renders clear starting actions. // @UX_STATE: Importing -> Progressive milestones remain visible while review context is assembled. -// @UX_STATE: Review -> Workspace exposes summary controls, timeline, findings, and export affordances. +// @UX_STATE: Review -> Workspace exposes summary controls, mounted semantic review, clarification dialog, findings, and export affordances. // @TEST_CONTRACT: DatasetReviewWorkspaceDependencies -> ObservableWorkspaceUX // @TEST_SCENARIO: empty_state_renders_source_intake -> No session route data shows intake-first workspace. // @TEST_SCENARIO: session_load_surfaces_partial_recovery_state -> Existing session route shows review summary, findings, and partial recovery evidence. +// @TEST_SCENARIO: session_load_mounts_us2_semantic_and_clarification_panels -> Existing session route mounts semantic review and clarification UI from real route integration. // @TEST_SCENARIO: import_submission_transitions_to_review_state -> Intake submit creates a session, navigates, and renders review state. // @TEST_SCENARIO: export_feedback_surfaces_success_and_error -> Export action exposes both artifact success detail and failure feedback. // @TEST_EDGE: missing_field -> Empty session context still renders stable workspace copy. @@ -44,6 +45,7 @@ function createSessionDetail(overrides = {}) { active_task_id: "task-77", findings: [ { + finding_id: "finding-1", title: "Missing semantic description", code: "SEM-1", area: "dataset_profile", @@ -80,6 +82,82 @@ function createSessionDetail(overrides = {}) { confidence_state: "mostly_confirmed", business_summary: "Imported business description", }, + semantic_sources: [ + { + source_id: "source-1", + display_name: "Trusted dictionary", + source_version: "v1", + }, + ], + semantic_fields: [ + { + field_id: "field-1", + field_name: "customer_name", + field_kind: "dimension", + verbose_name: "Customer name", + description: "Resolved from trusted dictionary", + display_format: "text", + provenance: "dictionary_exact", + source_id: "source-1", + confidence_rank: 1, + is_locked: false, + has_conflict: true, + needs_review: true, + last_changed_by: "system", + candidates: [ + { + candidate_id: "candidate-1", + match_type: "exact", + confidence_score: 0.98, + proposed_verbose_name: "Customer name", + proposed_description: "Customer display name", + proposed_display_format: "text", + status: "pending", + }, + ], + }, + ], + clarification_sessions: [ + { + clarification_session_id: "clarification-1", + status: "active", + current_question_id: "question-1", + resolved_count: 1, + remaining_count: 2, + summary_delta: "Awaiting user confirmation", + }, + ], + ...overrides, + }; +} + +function createClarificationState(overrides = {}) { + return { + clarification_session: { + clarification_session_id: "clarification-1", + session_id: "session-1", + status: "active", + current_question_id: "question-1", + resolved_count: 1, + remaining_count: 2, + summary_delta: "Awaiting user confirmation", + }, + current_question: { + question_id: "question-1", + clarification_session_id: "clarification-1", + topic_ref: "profile.summary", + question_text: "Which customer label should be used?", + why_it_matters: "This label is shown to reviewers.", + current_guess: "Customer name", + options: [ + { + option_id: "option-1", + value: "Customer name", + label: "Customer name", + is_recommended: true, + }, + ], + }, ...overrides, }; } @@ -239,6 +317,8 @@ vi.mock("$lib/i18n", () => ({ resume_action: "Resume session", pause_action: "Pause session", jump_target_label: "Jump target", + semantic_review_visible: "Semantic review visible", + clarification_visible: "Clarification visible", health_title: "Review health", open_findings_label: "Open findings", exports_title: "Exports", @@ -252,6 +332,108 @@ vi.mock("$lib/i18n", () => ({ preview_status_label: "Preview status", preview_compiler_label: "Compiled by", }, + semantics: { + eyebrow: "Semantic layer", + title: "Review semantic field values", + description: "Review semantic candidates", + field_count_label: "Fields in review", + empty: "No semantic fields", + unknown_source: "Unknown source", + active_value_label: "Active semantic value", + provenance_label: "Provenance", + confidence_label: "Confidence", + confidence_unset: "No confidence rank", + confidence_rank_label: "Rank", + source_label: "Source", + changed_by_label: "Changed by", + locked_badge: "Locked", + unlocked_badge: "Unlocked", + conflict_badge: "Conflict", + needs_review_badge: "Needs review", + manual_verbose_name_label: "Manual verbose name", + manual_description_label: "Manual description", + manual_display_format_label: "Manual display format", + manual_override_action: "Manual override", + save_manual_action: "Save manual value", + lock_action: "Lock field", + unlock_action: "Unlock field", + feedback_up_action: "Thumbs up", + feedback_down_action: "Thumbs down", + candidates_title: "Candidate options", + candidates_empty: "No candidates", + candidate_description_empty: "No candidate description", + display_format_label: "Display format", + score_label: "Score", + apply_candidate_action: "Apply", + apply_and_lock_action: "Apply and lock", + empty_value: "No value", + provenance: { + unresolved: "Unresolved", + dictionary_exact: "Dictionary exact match", + manual_override: "Manual override", + }, + candidate_status: { + pending: "Pending", + accepted: "Accepted", + superseded: "Superseded", + }, + messages: { + saving: "Saving semantic decision...", + save_failed: "Failed to save semantic decision.", + manual_override_required: "Manual value required", + manual_saved: "Manual semantic value saved and locked.", + candidate_applied: "Candidate value applied.", + candidate_locked: "Candidate value applied and locked.", + locked: "Field locked against silent overwrite.", + unlocked: "Field unlocked for renewed review.", + feedback_up: "Positive feedback recorded.", + feedback_down: "Negative feedback recorded.", + }, + }, + clarification: { + eyebrow: "Clarification", + title: "Resolve one ambiguity at a time", + description: "One question at a time", + progress_label: "Clarification progress", + state_label: "Dialog state", + status_label: "Status", + remaining_label: "Remaining questions", + summary_delta_label: "Summary delta", + active_question_label: "Active question", + why_it_matters_label: "Why it matters", + current_guess_label: "Current guess", + current_guess_empty: "No current guess", + topic_label: "Topic", + options_title: "Available options", + recommended_badge: "Recommended", + custom_answer_label: "Custom answer", + answer_action: "Answer with selected option", + custom_answer_action: "Submit custom answer", + skip_action: "Skip for now", + assistant_action: "Open assistant guidance", + expert_review_action: "Send to expert review", + resume_action: "Resume clarification", + completed: "No active clarification question remains.", + feedback_label: "Clarification feedback", + feedback_prompt: "Record whether the clarification result was useful.", + feedback_up_action: "Helpful", + feedback_down_action: "Not helpful", + messages: { + saving: "Saving clarification answer...", + saved: "Clarification answer saved.", + skipped: "Question skipped and left resumable.", + expert_review_requested: "Question escalated to expert review.", + assistant_opened: "Assistant guidance opened.", + resumed: "Clarification resumed.", + resume_failed: "Failed to resume clarification.", + save_failed: "Failed to save clarification answer.", + feedback_up: "Positive clarification feedback recorded.", + feedback_down: "Negative clarification feedback recorded.", + feedback_failed: "Failed to save clarification feedback.", + option_required: "Option required", + custom_required: "Custom answer required", + }, + }, }, }); return () => {}; @@ -311,7 +493,9 @@ describe("DatasetReviewWorkspace UX Contract", () => { it("session_load_surfaces_partial_recovery_state", async () => { routeState.id = "session-1"; - api.fetchApi.mockResolvedValue(createSessionDetail()); + api.fetchApi + .mockResolvedValueOnce(createSessionDetail()) + .mockResolvedValueOnce(createClarificationState()); render(DatasetReviewWorkspace); @@ -319,10 +503,13 @@ describe("DatasetReviewWorkspace UX Contract", () => { expect(api.fetchApi).toHaveBeenCalledWith("/dataset-orchestration/sessions/session-1"); }); + await waitFor(() => { + expect(screen.getByText("Business summary")).toBeDefined(); + }); + expect(screen.getByText("Workspace state: Review")).toBeDefined(); expect(screen.getByText("Readiness: Partially ready")).toBeDefined(); expect(screen.getAllByText("Sales Dataset").length).toBeGreaterThan(0); - expect(screen.getByText("Partial recovery")).toBeDefined(); expect(screen.getAllByText("Recovered filters").length).toBeGreaterThan(0); expect(screen.getByText("Imported business description")).toBeDefined(); expect(screen.getAllByRole("button", { name: "Save session" }).length).toBeGreaterThan(0); @@ -332,9 +519,29 @@ describe("DatasetReviewWorkspace UX Contract", () => { expect(screen.getByText("Source accepted")).toBeDefined(); }); + it("session_load_mounts_us2_semantic_and_clarification_panels", async () => { + routeState.id = "session-1"; + api.fetchApi + .mockResolvedValueOnce(createSessionDetail()) + .mockResolvedValueOnce(createClarificationState()); + + render(DatasetReviewWorkspace); + + await waitFor(() => { + expect(screen.getByText("Review semantic field values")).toBeDefined(); + }); + + expect(screen.getByText("Resolve one ambiguity at a time")).toBeDefined(); + expect(screen.getByText("customer_name")).toBeDefined(); + expect(screen.getByText("Which customer label should be used?")).toBeDefined(); + expect(screen.getAllByText("Customer name").length).toBeGreaterThan(0); + }); + it("summary_edit_affordance_updates_visible_draft", async () => { routeState.id = "session-1"; - api.fetchApi.mockResolvedValue(createSessionDetail()); + api.fetchApi + .mockResolvedValueOnce(createSessionDetail()) + .mockResolvedValueOnce(createClarificationState()); render(DatasetReviewWorkspace); @@ -344,7 +551,7 @@ describe("DatasetReviewWorkspace UX Contract", () => { await fireEvent.click(screen.getByRole("button", { name: "[Edit]" })); - const summaryEditor = screen.getByRole("textbox"); + const summaryEditor = screen.getAllByRole("textbox")[0]; await fireEvent.input(summaryEditor, { target: { value: "Updated analyst-facing summary" }, }); @@ -361,12 +568,17 @@ describe("DatasetReviewWorkspace UX Contract", () => { const createdSession = createSessionDetail(); let resolveSessionLoad; api.postApi.mockResolvedValue({ session_id: "session-1" }); - api.fetchApi.mockImplementation( - () => - new Promise((resolve) => { + api.fetchApi.mockImplementation((url) => { + if (url === "/dataset-orchestration/sessions/session-1") { + return new Promise((resolve) => { resolveSessionLoad = resolve; - }), - ); + }); + } + if (url === "/dataset-orchestration/sessions/session-1/clarification") { + return Promise.resolve(createClarificationState()); + } + return Promise.resolve(undefined); + }); render(DatasetReviewWorkspace); @@ -417,6 +629,7 @@ describe("DatasetReviewWorkspace UX Contract", () => { routeState.id = "session-1"; api.fetchApi .mockResolvedValueOnce(createSessionDetail()) + .mockResolvedValueOnce(createClarificationState()) .mockResolvedValueOnce({ artifact_type: "documentation", format: "json", diff --git a/specs/027-dataset-llm-orchestration/contracts/modules.md b/specs/027-dataset-llm-orchestration/contracts/modules.md index 24128f89..a87f7254 100644 --- a/specs/027-dataset-llm-orchestration/contracts/modules.md +++ b/specs/027-dataset-llm-orchestration/contracts/modules.md @@ -115,6 +115,7 @@ This document defines the semantic contracts for the core components of the Data # @PRE: selected source and target field set must be known. # @POST: candidate ranking follows the configured confidence hierarchy and unresolved fuzzy matches remain reviewable. # @SIDE_EFFECT: may create conflict findings and semantic candidate records. +# @DATA_CONTRACT: Input[SemanticSourceSelection | SemanticFieldSet | ManualFieldDecision] -> Output[SemanticCandidateSet | RankedSemanticResolution | ValidationFindingSet] # @INVARIANT: Manual overrides are never silently replaced by imported, inferred, or AI-generated values. # @TEST_CONTRACT: rank_candidates -> exact dictionary beats reference import beats fuzzy beats AI draft # @TEST_SCENARIO: manual_lock_survives_reimport -> locked field remains active after another source is applied @@ -156,6 +157,7 @@ This document defines the semantic contracts for the core components of the Data # @PRE: target session contains unresolved or contradictory review state. # @POST: every recorded answer updates the clarification session and associated session state deterministically. # @SIDE_EFFECT: creates clarification questions, persists answers, updates findings/profile state. +# @DATA_CONTRACT: Input[ClarificationSessionState | ClarificationAnswerCommand] -> Output[ClarificationQuestionPayload | ClarificationProgressSnapshot | SessionReadinessDelta] # @INVARIANT: Clarification answers are persisted before the current question pointer or readiness state is advanced. # @TEST_CONTRACT: next_question_selection -> returns only one highest-priority unresolved question at a time # @TEST_SCENARIO: save_and_resume_clarification -> reopening session restores current question and prior answers @@ -184,9 +186,10 @@ This document defines the semantic contracts for the core components of the Data # @COMPLEXITY: 4 # @PURPOSE: Recover dataset, dashboard, filter, and runtime-template context from Superset links and related API payloads. # @LAYER: Infra -# @RELATION: [CALLS] ->[SupersetClient] # @RELATION: [DEPENDS_ON] ->[ImportedFilter] # @RELATION: [DEPENDS_ON] ->[TemplateVariable] +# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_client.SupersetClient] +# @DATA_CONTRACT: Input[SupersetLink | DatasetReference | EnvironmentContext] -> Output[RecoveredSupersetContext | ImportedFilterSet | TemplateVariableSet | RecoverySummary] # @PRE: Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource. # @POST: returns the best available recovered context with explicit provenance and partial-recovery markers when necessary. # @SIDE_EFFECT: performs upstream Superset API reads. @@ -218,9 +221,10 @@ This document defines the semantic contracts for the core components of the Data # @COMPLEXITY: 4 # @PURPOSE: Interact with Superset preview compilation and SQL Lab execution endpoints using the current approved execution context. # @LAYER: Infra -# @RELATION: [CALLS] ->[SupersetClient] # @RELATION: [DEPENDS_ON] ->[CompiledPreview] # @RELATION: [DEPENDS_ON] ->[DatasetRunContext] +# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_client.SupersetClient] +# @DATA_CONTRACT: Input[ApprovedExecutionContext | PreviewFingerprint | LaunchRequest] -> Output[CompiledPreview | PreviewFailureArtifact | DatasetRunContext | LaunchFailureAudit] # @PRE: effective template params and dataset execution reference are available. # @POST: preview and launch calls return Superset-originated artifacts or explicit errors. # @SIDE_EFFECT: performs upstream Superset preview and SQL Lab calls. diff --git a/specs/027-dataset-llm-orchestration/tasks.md b/specs/027-dataset-llm-orchestration/tasks.md index 2055e130..38a55504 100644 --- a/specs/027-dataset-llm-orchestration/tasks.md +++ b/specs/027-dataset-llm-orchestration/tasks.md @@ -50,12 +50,12 @@ **Independent Test**: Open a session with unresolved findings; answer questions one by one and verify readiness state updates in real-time. -- [ ] T021 [P] [US2] Implement `ClarificationEngine.build_question_payload` (CRITICAL: C4, PRE: unresolved state, POST: prioritized question) in `backend/src/services/dataset_review/clarification_engine.py` -- [ ] T022 [US2] Implement `ClarificationEngine.record_answer` (CRITICAL: C4, PRE: question active, POST: answer persisted before state advance) in `backend/src/services/dataset_review/clarification_engine.py` -- [ ] T023 [P] [US2] Implement field-level semantic override and lock endpoints in `backend/src/api/routes/dataset_review.py` -- [ ] T024 [US2] Implement `SemanticLayerReview` component (C3, UX_STATE: Conflicted/Manual) in `frontend/src/lib/components/dataset-review/SemanticLayerReview.svelte` -- [ ] T025 [P] [US2] Implement `ClarificationDialog` (C3, UX_STATE: Question/Saving/Completed, REL: binds to `assistantChat`) in `frontend/src/lib/components/dataset-review/ClarificationDialog.svelte` -- [ ] T026 [US2] Implement LLM feedback (👍/👎) storage and UI handlers in `backend/src/api/routes/dataset_review.py` +- [X] T021 [P] [US2] Implement `ClarificationEngine.build_question_payload` (CRITICAL: C4, PRE: unresolved state, POST: prioritized question) in `backend/src/services/dataset_review/clarification_engine.py` +- [X] T022 [US2] Implement `ClarificationEngine.record_answer` (CRITICAL: C4, PRE: question active, POST: answer persisted before state advance) in `backend/src/services/dataset_review/clarification_engine.py` +- [X] T023 [P] [US2] Implement field-level semantic override and lock endpoints in `backend/src/api/routes/dataset_review.py` +- [X] T024 [US2] Implement `SemanticLayerReview` component (C3, UX_STATE: Conflicted/Manual) in `frontend/src/lib/components/dataset-review/SemanticLayerReview.svelte` +- [X] T025 [P] [US2] Implement `ClarificationDialog` (C3, UX_STATE: Question/Saving/Completed, REL: binds to `assistantChat`) in `frontend/src/lib/components/dataset-review/ClarificationDialog.svelte` +- [X] T026 [US2] Implement LLM feedback (👍/👎) storage and UI handlers in `backend/src/api/routes/dataset_review.py` - [ ] T027 [US2] Verify implementation matches ux_reference.md (Happy Path & Errors) - [ ] T028 [US2] Acceptance: Perform semantic audit & algorithm emulation by Tester @@ -67,13 +67,13 @@ **Independent Test**: Map filters to variables; trigger preview; verify launch blocked until preview succeeds; verify SQL Lab session creation. -- [ ] T029 [P] [US3] Implement `SupersetContextExtractor.recover_imported_filters` and variable discovery in `backend/src/core/utils/superset_context_extractor.py` -- [ ] T030 [US3] Implement `SupersetCompilationAdapter.compile_preview` (CRITICAL: C4, PRE: effective inputs available, POST: Superset-compiled SQL only) in `backend/src/core/utils/superset_compilation_adapter.py` -- [ ] T031 [US3] Implement `DatasetReviewOrchestrator.launch_dataset` (CRITICAL: C5, PRE: run-ready + preview match, POST: audited run context) in `backend/src/services/dataset_review/orchestrator.py` -- [ ] T032 [P] [US3] Implement mapping approval and preview trigger endpoints in `backend/src/api/routes/dataset_review.py` -- [ ] T033 [P] [US3] Implement `ExecutionMappingReview` component (C3, UX_STATE: WarningApproval/Approved) in `frontend/src/lib/components/dataset-review/ExecutionMappingReview.svelte` -- [ ] T034 [P] [US3] Implement `CompiledSQLPreview` component (C3, UX_STATE: Ready/Stale/Error) in `frontend/src/lib/components/dataset-review/CompiledSQLPreview.svelte` -- [ ] T035 [US3] Implement `LaunchConfirmationPanel` (C3, UX_STATE: Blocked/Ready/Submitted) in `frontend/src/lib/components/dataset-review/LaunchConfirmationPanel.svelte` +- [X] T029 [P] [US3] Implement `SupersetContextExtractor.recover_imported_filters` and variable discovery in `backend/src/core/utils/superset_context_extractor.py` +- [X] T030 [US3] Implement `SupersetCompilationAdapter.compile_preview` (CRITICAL: C4, PRE: effective inputs available, POST: Superset-compiled SQL only) in `backend/src/core/utils/superset_compilation_adapter.py` +- [X] T031 [US3] Implement `DatasetReviewOrchestrator.launch_dataset` (CRITICAL: C5, PRE: run-ready + preview match, POST: audited run context) in `backend/src/services/dataset_review/orchestrator.py` +- [X] T032 [P] [US3] Implement mapping approval and preview trigger endpoints in `backend/src/api/routes/dataset_review.py` +- [X] T033 [P] [US3] Implement `ExecutionMappingReview` component (C3, UX_STATE: WarningApproval/Approved) in `frontend/src/lib/components/dataset-review/ExecutionMappingReview.svelte` +- [X] T034 [P] [US3] Implement `CompiledSQLPreview` component (C3, UX_STATE: Ready/Stale/Error) in `frontend/src/lib/components/dataset-review/CompiledSQLPreview.svelte` +- [X] T035 [US3] Implement `LaunchConfirmationPanel` (C3, UX_STATE: Blocked/Ready/Submitted) in `frontend/src/lib/components/dataset-review/LaunchConfirmationPanel.svelte` - [ ] T036 [US3] Verify implementation matches ux_reference.md (Happy Path & Errors) - [ ] T037 [US3] Acceptance: Perform semantic audit & algorithm emulation by Tester