busya
306c5ae742
- Convert legacy [DEF🆔Type] anchors to #region/#endregion across 329 files
- Reinstate _normalize_timestamp_value in sql_generator.py
- Fix MarkerLogger→logger migration in events.py (molecular CoT markers)
- Fix dataset_review orchestrator dependencies (_build_execution_snapshot)
- Fix config_manager stale-record deletion (moved to save path only)
- Add 77 missing [/DEF:] closers in 5 unbalanced test files
- Update assistant_chat.integration.test.js for #region format
- Apply molecular-cot-logging markers (REASON/REFLECT/EXPLORE) via logger.* methods
291 lines
13 KiB
Python
291 lines
13 KiB
Python
# #region AssistantDatasetReview [C:4] [TYPE Module]
|
|
# @BRIEF Dataset review context loading and intent planning for the assistant API.
|
|
# @LAYER: API
|
|
# @RELATION DEPENDS_ON -> [DatasetReviewOrchestrator]
|
|
# @RELATION DEPENDS_ON -> [AssistantSchemas]
|
|
# @RELATION DISPATCHES -> [AssistantDatasetReviewDispatch]
|
|
# @INVARIANT: Dataset review operations are always scoped to the owner's session.
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from datetime import datetime
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
from fastapi import HTTPException, status
|
|
from sqlalchemy.orm import Session
|
|
|
|
from src.core.logger import belief_scope, logger
|
|
from src.core.utils.superset_context_extractor import (
|
|
sanitize_imported_filter_for_assistant,
|
|
)
|
|
from src.models.dataset_review import (
|
|
ApprovalState,
|
|
DatasetReviewSession,
|
|
ReadinessState,
|
|
RecommendedAction,
|
|
)
|
|
from src.services.dataset_review.repositories.session_repository import (
|
|
DatasetReviewSessionRepository,
|
|
DatasetReviewSessionVersionConflictError,
|
|
)
|
|
from src.schemas.auth import User
|
|
from ._schemas import (
|
|
AssistantAction,
|
|
)
|
|
|
|
|
|
# #region _serialize_dataset_review_context [C:4] [TYPE Function]
|
|
# @BRIEF Build assistant-safe dataset-review context snapshot with masked imported-filter payloads for session-scoped assistant routing.
|
|
# @RELATION DEPENDS_ON -> [DatasetReviewSession]
|
|
# @PRE: session_id is a valid active review session identifier.
|
|
# @POST: Returns a serializable dictionary containing the complete review context.
|
|
# @SIDE_EFFECT: Reads session data from the database.
|
|
def _serialize_dataset_review_context(session: DatasetReviewSession) -> Dict[str, Any]:
|
|
with belief_scope('_serialize_dataset_review_context'):
|
|
logger.reason('Belief protocol reasoning checkpoint for _serialize_dataset_review_context')
|
|
latest_preview = None
|
|
previews = getattr(session, 'previews', []) or []
|
|
if previews:
|
|
latest_preview = previews[-1]
|
|
logger.reflect('Belief protocol postcondition checkpoint for _serialize_dataset_review_context')
|
|
return {'session_id': session.session_id, 'version': int(getattr(session, 'version', 0) or 0), 'dataset_ref': session.dataset_ref, 'dataset_id': session.dataset_id, 'environment_id': session.environment_id, 'readiness_state': session.readiness_state.value, 'recommended_action': session.recommended_action.value, 'status': session.status.value, 'current_phase': session.current_phase.value, 'findings': [{'finding_id': item.finding_id, 'code': item.code, 'severity': item.severity.value, 'message': item.message, 'resolution_state': item.resolution_state.value} for item in getattr(session, 'findings', [])], 'imported_filters': [sanitize_imported_filter_for_assistant({'filter_id': item.filter_id, 'filter_name': item.filter_name, 'display_name': item.display_name, 'raw_value': item.raw_value, 'raw_value_masked': bool(getattr(item, 'raw_value_masked', False)), 'normalized_value': item.normalized_value, 'source': getattr(item.source, 'value', item.source), 'confidence_state': getattr(item.confidence_state, 'value', item.confidence_state), 'requires_confirmation': bool(item.requires_confirmation), 'recovery_status': getattr(item.recovery_status, 'value', item.recovery_status), 'notes': item.notes}) for item in getattr(session, 'imported_filters', [])], 'mappings': [{'mapping_id': item.mapping_id, 'filter_id': item.filter_id, 'variable_id': item.variable_id, 'mapping_method': getattr(item.mapping_method, 'value', item.mapping_method), 'effective_value': item.effective_value, 'approval_state': getattr(item.approval_state, 'value', item.approval_state), 'requires_explicit_approval': bool(item.requires_explicit_approval)} for item in getattr(session, 'execution_mappings', [])], 'semantic_fields': [{'field_id': item.field_id, 'field_name': item.field_name, 'verbose_name': item.verbose_name, 'description': item.description, 'display_format': item.display_format, 'provenance': getattr(item.provenance, 'value', item.provenance), 'is_locked': bool(item.is_locked), 'needs_review': bool(getattr(item, 'needs_review', False)), 'candidates': [{'candidate_id': c.candidate_id, 'field_id': c.field_id, 'verbose_name': c.proposed_verbose_name, 'description': c.proposed_description, 'display_format': c.proposed_display_format, 'status': getattr(c.status, 'value', c.status), 'source': c.source_id, 'score': c.confidence_score, 'created_at': c.created_at.isoformat() if c.created_at else None} for c in getattr(item, 'candidates', [])]} for item in getattr(session, 'semantic_fields', [])]}
|
|
|
|
|
|
# #endregion _serialize_dataset_review_context
|
|
|
|
|
|
# #region _load_dataset_review_context [C:4] [TYPE Function]
|
|
# @BRIEF Load owner-scoped dataset-review context for assistant planning and grounded response generation.
|
|
# @RELATION DEPENDS_ON -> [DatasetReviewSessionRepository]
|
|
# @PRE: session_id is a valid active review session identifier.
|
|
# @POST: Returns a loaded context object with session data and findings.
|
|
# @SIDE_EFFECT: Reads session data from the database.
|
|
def _load_dataset_review_context(dataset_review_session_id: Optional[str], current_user: User, db: Session) -> Optional[Dict[str, Any]]:
|
|
with belief_scope('_load_dataset_review_context'):
|
|
if not dataset_review_session_id:
|
|
return None
|
|
logger.reason('Belief protocol reasoning checkpoint for _load_dataset_review_context')
|
|
repository = DatasetReviewSessionRepository(db)
|
|
session = repository.load_session_detail(dataset_review_session_id, current_user.id)
|
|
if session is None or session.user_id != current_user.id:
|
|
raise HTTPException(status_code=404, detail='Dataset review session not found')
|
|
logger.reflect('Belief protocol postcondition checkpoint for _load_dataset_review_context')
|
|
return _serialize_dataset_review_context(session)
|
|
|
|
|
|
# #endregion _load_dataset_review_context
|
|
|
|
|
|
# #region _extract_dataset_review_target [C:2] [TYPE Function]
|
|
# @BRIEF Extract structured dataset-review focus target hints embedded in assistant prompts.
|
|
def _extract_dataset_review_target(message: str) -> Tuple[Optional[str], Optional[str]]:
|
|
match = re.search(
|
|
r"(?:target|focus)\s*[:=]\s*(field|mapping|finding|filter)[:=]([A-Za-z0-9._-]+)",
|
|
str(message or ""),
|
|
re.IGNORECASE,
|
|
)
|
|
if not match:
|
|
return None, None
|
|
return match.group(1).lower(), match.group(2)
|
|
|
|
|
|
# #endregion _extract_dataset_review_target
|
|
|
|
|
|
# #region _match_dataset_review_field [C:2] [TYPE Function]
|
|
# @BRIEF Resolve one semantic field from assistant-visible context by id or user-visible label.
|
|
def _match_dataset_review_field(
|
|
dataset_context: Dict[str, Any],
|
|
message: str,
|
|
) -> Optional[Dict[str, Any]]:
|
|
target_kind, target_id = _extract_dataset_review_target(message)
|
|
fields = dataset_context.get("semantic_fields", []) or []
|
|
if target_kind == "field" and target_id:
|
|
return next(
|
|
(item for item in fields if str(item.get("field_id")) == str(target_id)),
|
|
None,
|
|
)
|
|
|
|
normalized_message = str(message or "").lower()
|
|
for field in fields:
|
|
if str(field.get("field_id", "")).lower() in normalized_message:
|
|
return field
|
|
field_name = str(field.get("field_name", "")).lower()
|
|
if field_name and field_name in normalized_message:
|
|
return field
|
|
verbose_name = str(field.get("verbose_name", "")).lower()
|
|
if verbose_name and verbose_name in normalized_message:
|
|
return field
|
|
return None
|
|
|
|
|
|
# #endregion _match_dataset_review_field
|
|
|
|
|
|
# #region _extract_quoted_segment [C:2] [TYPE Function]
|
|
# @BRIEF Extract one quoted assistant command segment after a label token.
|
|
def _extract_quoted_segment(message: str, label: str) -> Optional[str]:
|
|
pattern = rf"{label}\s*[=:]?\s*[\"']([^\"']+)[\"']"
|
|
match = re.search(pattern, str(message or ""), re.IGNORECASE)
|
|
return match.group(1).strip() if match else None
|
|
|
|
|
|
# #endregion _extract_quoted_segment
|
|
|
|
|
|
# #region _plan_dataset_review_intent [C:3] [TYPE Function]
|
|
# @BRIEF Parse session-scoped dataset-review assistant commands before falling back to generic assistant tool routing.
|
|
# @RELATION CALLS -> DatasetReviewOrchestrator
|
|
def _plan_dataset_review_intent(
|
|
message: str,
|
|
dataset_context: Dict[str, Any],
|
|
) -> Optional[Dict[str, Any]]:
|
|
lower = message.strip().lower()
|
|
session_id = dataset_context["session_id"]
|
|
session_version = int(dataset_context.get("version", 0) or 0)
|
|
target_kind, target_id = _extract_dataset_review_target(message)
|
|
|
|
if any(
|
|
token in lower
|
|
for token in [
|
|
"approve mappings",
|
|
"approve mapping",
|
|
"подтверди мапп",
|
|
"одобри мапп",
|
|
]
|
|
):
|
|
pending_mapping_ids = [
|
|
item["mapping_id"]
|
|
for item in dataset_context.get("mappings", [])
|
|
if item.get("requires_explicit_approval")
|
|
and item.get("approval_state") != ApprovalState.APPROVED.value
|
|
]
|
|
return {
|
|
"domain": "dataset_review",
|
|
"operation": "dataset_review_approve_mappings",
|
|
"entities": {
|
|
"dataset_review_session_id": session_id,
|
|
"session_version": session_version,
|
|
"mapping_ids": pending_mapping_ids,
|
|
},
|
|
"confidence": 0.95,
|
|
"risk_level": "guarded",
|
|
"requires_confirmation": True,
|
|
}
|
|
|
|
if any(
|
|
token in lower
|
|
for token in [
|
|
"generate sql preview",
|
|
"generate preview",
|
|
"сгенерируй превью",
|
|
"собери превью",
|
|
]
|
|
):
|
|
return {
|
|
"domain": "dataset_review",
|
|
"operation": "dataset_review_generate_sql_preview",
|
|
"entities": {
|
|
"dataset_review_session_id": session_id,
|
|
"session_version": session_version,
|
|
},
|
|
"confidence": 0.94,
|
|
"risk_level": "guarded",
|
|
"requires_confirmation": True,
|
|
}
|
|
|
|
if any(
|
|
token in lower
|
|
for token in [
|
|
"set field semantics",
|
|
"apply field semantics",
|
|
"semantic override",
|
|
"update semantic field",
|
|
"установи семантик",
|
|
"обнови семантик",
|
|
]
|
|
):
|
|
field = _match_dataset_review_field(dataset_context, message)
|
|
if field is None:
|
|
return None
|
|
candidate_id = None
|
|
if any(
|
|
token in lower for token in ["accept candidate", "apply candidate", "прими кандидат"]
|
|
):
|
|
candidates = field.get("candidates") or []
|
|
if candidates:
|
|
candidate_id = candidates[0].get("candidate_id")
|
|
verbose_name = _extract_quoted_segment(
|
|
message, "verbose_name|verbose name|label"
|
|
)
|
|
description = _extract_quoted_segment(message, "description|desc")
|
|
display_format = _extract_quoted_segment(
|
|
message, "display_format|display format|format"
|
|
)
|
|
lock_field = any(
|
|
token in lower for token in [" lock", "lock it", "зафикс", "закреп"]
|
|
)
|
|
if not any([candidate_id, verbose_name, description, display_format]):
|
|
return None
|
|
return {
|
|
"domain": "dataset_review",
|
|
"operation": "dataset_review_set_field_semantics",
|
|
"entities": {
|
|
"dataset_review_session_id": session_id,
|
|
"session_version": session_version,
|
|
"field_id": field.get("field_id") or target_id,
|
|
"candidate_id": candidate_id,
|
|
"verbose_name": verbose_name,
|
|
"description": description,
|
|
"display_format": display_format,
|
|
"lock_field": lock_field,
|
|
},
|
|
"confidence": 0.9,
|
|
"risk_level": "guarded",
|
|
"requires_confirmation": True,
|
|
}
|
|
|
|
if any(
|
|
token in lower
|
|
for token in [
|
|
"filters",
|
|
"фильтр",
|
|
"mapping",
|
|
"маппинг",
|
|
"preview",
|
|
"превью",
|
|
"finding",
|
|
"ошиб",
|
|
]
|
|
):
|
|
findings_count = len(dataset_context.get("findings", []))
|
|
mappings_count = len(dataset_context.get("mappings", []))
|
|
filters_count = len(dataset_context.get("imported_filters", []))
|
|
preview = dataset_context.get("preview") or {}
|
|
preview_status = preview.get("preview_status") or "missing"
|
|
masked_filters = dataset_context.get("imported_filters", [])
|
|
return {
|
|
"domain": "dataset_review",
|
|
"operation": "dataset_review_answer_context",
|
|
"entities": {
|
|
"dataset_review_session_id": session_id,
|
|
"summary": (
|
|
f"Session {session_id}: readiness={dataset_context['readiness_state']}, "
|
|
f"recommended_action={dataset_context['recommended_action']}, "
|
|
f"filters={filters_count}, mappings={mappings_count}, findings={findings_count}, "
|
|
f"preview_status={preview_status}, imported_filters={masked_filters}"
|
|
),
|
|
},
|
|
"confidence": 0.8,
|
|
"risk_level": "safe",
|
|
"requires_confirmation": False,
|
|
}
|
|
return None
|
|
|
|
|
|
# #endregion _plan_dataset_review_intent
|
|
|
|
|
|
# #endregion AssistantDatasetReview
|