diff --git a/backend/src/api/routes/__init__.py b/backend/src/api/routes/__init__.py index 6364a93a..06778f95 100755 --- a/backend/src/api/routes/__init__.py +++ b/backend/src/api/routes/__init__.py @@ -37,6 +37,7 @@ __all__ = [ "dashboards", "datasets", "health", + "translate", ] # [/DEF:Route_Group_Contracts:Block] diff --git a/backend/src/api/routes/__tests__/test_dashboards.py b/backend/src/api/routes/__tests__/test_dashboards.py index 6bb6bdb3..e6a6231d 100644 --- a/backend/src/api/routes/__tests__/test_dashboards.py +++ b/backend/src/api/routes/__tests__/test_dashboards.py @@ -251,7 +251,7 @@ def test_get_dashboards_invalid_pagination(mock_deps): # @PURPOSE: Validate dashboard detail returns charts and datasets for an existing dashboard. # @TEST: GET /api/dashboards/{id} returns dashboard detail with charts and datasets def test_get_dashboard_detail_success(mock_deps): - with patch("src.api.routes.dashboards.SupersetClient") as mock_client_cls: + with patch("src.api.routes.dashboards._detail_routes.SupersetClient") as mock_client_cls: mock_env = MagicMock() mock_env.id = "prod" mock_deps["config"].get_environments.return_value = [mock_env] @@ -554,7 +554,7 @@ def test_get_dashboard_tasks_history_filters_success(mock_deps): # @PURPOSE: Validate dashboard thumbnail endpoint proxies image bytes and content type from Superset. # @TEST: GET /api/dashboards/{id}/thumbnail proxies image bytes from Superset def test_get_dashboard_thumbnail_success(mock_deps): - with patch("src.api.routes.dashboards.SupersetClient") as mock_client_cls: + with patch("src.api.routes.dashboards._detail_routes.SupersetClient") as mock_client_cls: mock_env = MagicMock() mock_env.id = "prod" mock_deps["config"].get_environments.return_value = [mock_env] @@ -666,7 +666,7 @@ def test_get_dashboards_profile_filter_contract_owners_or_modified_by(mock_deps) ] ) - with patch("src.api.routes.dashboards.ProfileService") as profile_service_cls: + with patch("src.api.routes.dashboards._listing_routes.ProfileService") as profile_service_cls: profile_service = MagicMock() profile_service.get_my_preference.return_value = _build_profile_preference_stub( username=" JOHN_DOE ", @@ -787,7 +787,7 @@ def test_get_dashboards_profile_filter_no_match_results_contract(mock_deps): ] ) - with patch("src.api.routes.dashboards.ProfileService") as profile_service_cls: + with patch("src.api.routes.dashboards._listing_routes.ProfileService") as profile_service_cls: profile_service = MagicMock() profile_service.get_my_preference.return_value = _build_profile_preference_stub( username="john_doe", @@ -916,10 +916,10 @@ def test_get_dashboards_profile_filter_matches_display_alias_without_detail_fano ) with ( - patch("src.api.routes.dashboards.ProfileService") as profile_service_cls, - patch("src.api.routes.dashboards.SupersetClient") as superset_client_cls, + patch("src.api.routes.dashboards._listing_routes.ProfileService") as profile_service_cls, + patch("src.api.routes.dashboards._projection.SupersetClient") as superset_client_cls, patch( - "src.api.routes.dashboards.SupersetAccountLookupAdapter" + "src.api.routes.dashboards._projection.SupersetAccountLookupAdapter" ) as lookup_adapter_cls, ): profile_service = MagicMock() @@ -1013,9 +1013,9 @@ def test_get_dashboards_profile_filter_matches_owner_object_payload_contract(moc ) with ( - patch("src.api.routes.dashboards.ProfileService") as profile_service_cls, + patch("src.api.routes.dashboards._listing_routes.ProfileService") as profile_service_cls, patch( - "src.api.routes.dashboards._resolve_profile_actor_aliases", + "src.api.routes.dashboards._projection._resolve_profile_actor_aliases", return_value=["user_1"], ), ): diff --git a/backend/src/api/routes/assistant.py b/backend/src/api/routes/assistant.py deleted file mode 100644 index 28f47632..00000000 --- a/backend/src/api/routes/assistant.py +++ /dev/null @@ -1,2683 +0,0 @@ -# [DEF:AssistantApi:Module] -# @COMPLEXITY: 5 -# @SEMANTICS: api, assistant, chat, command, confirmation -# @PURPOSE: API routes for LLM assistant command parsing and safe execution orchestration. -# @LAYER: API -# @RELATION: DEPENDS_ON -> [TaskManager] -# @RELATION: DEPENDS_ON -> [AssistantMessageRecord] -# @RELATION: DEPENDS_ON -> [AssistantConfirmationRecord] -# @RELATION: DEPENDS_ON -> [AssistantAuditRecord] -# @INVARIANT: Risky operations are never executed without valid confirmation token. - -from __future__ import annotations - -import json -import re -import uuid -from datetime import datetime, timedelta -from typing import Any, Dict, List, Optional, Tuple, cast - -from fastapi import APIRouter, Depends, HTTPException, Query, status -from pydantic import BaseModel, Field -from sqlalchemy.orm import Session -from sqlalchemy import desc - -from ...core.logger import belief_scope, logger -from ...core.task_manager import TaskManager -from ...dependencies import ( - get_current_user, - get_task_manager, - get_config_manager, - has_permission, -) -from ...core.config_manager import ConfigManager -from ...core.database import get_db -from ...services.git_service import GitService -from ...services.llm_provider import LLMProviderService -from ...services.llm_prompt_templates import ( - is_multimodal_model, - normalize_llm_settings, - resolve_bound_provider_id, -) -from ...core.superset_client import SupersetClient -from ...core.utils.superset_context_extractor import ( - sanitize_imported_filter_for_assistant, -) -from ...plugins.llm_analysis.service import LLMClient -from ...plugins.llm_analysis.models import LLMProviderType -from ...schemas.auth import User -from ...models.assistant import ( - AssistantAuditRecord, - AssistantConfirmationRecord, - AssistantMessageRecord, -) -from ...models.dataset_review import ( - ApprovalState, - DatasetReviewSession, - ReadinessState, - RecommendedAction, -) -from ...services.dataset_review.orchestrator import ( - DatasetReviewOrchestrator, - PreparePreviewCommand, -) -from ...services.dataset_review.repositories.session_repository import ( - DatasetReviewSessionRepository, - DatasetReviewSessionVersionConflictError, -) -from .dataset_review import FieldSemanticUpdateRequest, _update_semantic_field_state - -router = APIRouter(tags=["Assistant"]) -git_service = GitService() -logger = cast(Any, logger) - - -# [DEF:AssistantMessageRequest:Class] -# @COMPLEXITY: 1 -# @PURPOSE: Input payload for assistant message endpoint. -# @DATA_CONTRACT: Input[conversation_id?:str, message:str(1..4000)] -> Output[AssistantMessageRequest] -# @RELATION: USED_BY -> [send_message] -# @SIDE_EFFECT: None (schema declaration only). -# @PRE: message length is within accepted bounds. -# @POST: Request object provides message text and optional conversation binding. -# @INVARIANT: message is always non-empty and no longer than 4000 characters. -class AssistantMessageRequest(BaseModel): - conversation_id: Optional[str] = None - message: str = Field(..., min_length=1, max_length=4000) - dataset_review_session_id: Optional[str] = None - - -# [/DEF:AssistantMessageRequest:Class] - - -# [DEF:AssistantAction:Class] -# @COMPLEXITY: 1 -# @PURPOSE: UI action descriptor returned with assistant responses. -# @DATA_CONTRACT: Input[type:str, label:str, target?:str] -> Output[AssistantAction] -# @RELATION: USED_BY -> [AssistantMessageResponse] -# @SIDE_EFFECT: None (schema declaration only). -# @PRE: type and label are provided by orchestration logic. -# @POST: Action can be rendered as button on frontend. -# @INVARIANT: type and label are required for every UI action. -class AssistantAction(BaseModel): - type: str - label: str - target: Optional[str] = None - - -# [/DEF:AssistantAction:Class] - - -# [DEF:AssistantMessageResponse:Class] -# @COMPLEXITY: 1 -# @PURPOSE: Output payload contract for assistant interaction endpoints. -# @DATA_CONTRACT: Input[conversation_id,response_id,state,text,intent?,confirmation_id?,task_id?,actions[],created_at] -> Output[AssistantMessageResponse] -# @RELATION: RETURNED_BY -> [send_message] -# @RELATION: RETURNED_BY -> [confirm_operation] -# @RELATION: RETURNED_BY -> [cancel_operation] -# @SIDE_EFFECT: None (schema declaration only). -# @PRE: Response includes deterministic state and text. -# @POST: Payload may include task_id/confirmation_id/actions for UI follow-up. -# @INVARIANT: created_at and state are always present in endpoint responses. -class AssistantMessageResponse(BaseModel): - conversation_id: str - response_id: str - state: str - text: str - intent: Optional[Dict[str, Any]] = None - confirmation_id: Optional[str] = None - task_id: Optional[str] = None - actions: List[AssistantAction] = Field(default_factory=list) - created_at: datetime - - -# [/DEF:AssistantMessageResponse:Class] - - -# [DEF:ConfirmationRecord:Class] -# @COMPLEXITY: 1 -# @PURPOSE: In-memory confirmation token model for risky operation dispatch. -# @DATA_CONTRACT: Input[id,user_id,conversation_id,intent,dispatch,expires_at,state?,created_at] -> Output[ConfirmationRecord] -# @RELATION: USED_BY -> [send_message] -# @RELATION: USED_BY -> [confirm_operation] -# @RELATION: USED_BY -> [cancel_operation] -# @SIDE_EFFECT: None (schema declaration only). -# @PRE: intent/dispatch/user_id are populated at confirmation request time. -# @POST: Record tracks lifecycle state and expiry timestamp. -# @INVARIANT: state defaults to "pending" and expires_at bounds confirmation validity. -class ConfirmationRecord(BaseModel): - id: str - user_id: str - conversation_id: str - intent: Dict[str, Any] - dispatch: Dict[str, Any] - expires_at: datetime - state: str = "pending" - created_at: datetime - - -# [/DEF:ConfirmationRecord:Class] - - -CONVERSATIONS: Dict[Tuple[str, str], List[Dict[str, Any]]] = {} -USER_ACTIVE_CONVERSATION: Dict[str, str] = {} -CONFIRMATIONS: Dict[str, ConfirmationRecord] = {} -ASSISTANT_AUDIT: Dict[str, List[Dict[str, Any]]] = {} -ASSISTANT_ARCHIVE_AFTER_DAYS = 14 -ASSISTANT_MESSAGE_TTL_DAYS = 90 - -INTENT_PERMISSION_CHECKS: Dict[str, List[Tuple[str, str]]] = { - "get_task_status": [("tasks", "READ")], - "create_branch": [("plugin:git", "EXECUTE")], - "commit_changes": [("plugin:git", "EXECUTE")], - "deploy_dashboard": [("plugin:git", "EXECUTE")], - "execute_migration": [ - ("plugin:migration", "EXECUTE"), - ("plugin:superset-migration", "EXECUTE"), - ], - "run_backup": [("plugin:superset-backup", "EXECUTE"), ("plugin:backup", "EXECUTE")], - "run_llm_validation": [("plugin:llm_dashboard_validation", "EXECUTE")], - "run_llm_documentation": [("plugin:llm_documentation", "EXECUTE")], - "get_health_summary": [("plugin:migration", "READ")], - "dataset_review_answer_context": [("dataset:session", "READ")], - "dataset_review_approve_mappings": [("dataset:session", "MANAGE")], - "dataset_review_set_field_semantics": [("dataset:session", "MANAGE")], - "dataset_review_generate_sql_preview": [("dataset:session", "MANAGE")], -} - - -# [DEF:_append_history:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Append conversation message to in-memory history buffer. -# @DATA_CONTRACT: Input[user_id,conversation_id,role,text,state?,task_id?,confirmation_id?] -> Output[None] -# @RELATION: UPDATES -> [CONVERSATIONS] -# @SIDE_EFFECT: Mutates in-memory CONVERSATIONS store for user conversation history. -# @PRE: user_id and conversation_id identify target conversation bucket. -# @POST: Message entry is appended to CONVERSATIONS key list. -# @INVARIANT: every appended entry includes generated message_id and created_at timestamp. -def _append_history( - user_id: str, - conversation_id: str, - role: str, - text: str, - state: Optional[str] = None, - task_id: Optional[str] = None, - confirmation_id: Optional[str] = None, -): - key = (user_id, conversation_id) - if key not in CONVERSATIONS: - CONVERSATIONS[key] = [] - CONVERSATIONS[key].append( - { - "message_id": str(uuid.uuid4()), - "conversation_id": conversation_id, - "role": role, - "text": text, - "state": state, - "task_id": task_id, - "confirmation_id": confirmation_id, - "created_at": datetime.utcnow(), - } - ) - - -# [/DEF:_append_history:Function] - - -# [DEF:_persist_message:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Persist assistant/user message record to database. -# @DATA_CONTRACT: Input[Session,user_id,conversation_id,role,text,state?,task_id?,confirmation_id?,metadata?] -> Output[None] -# @RELATION: DEPENDS_ON -> [AssistantMessageRecord] -# @SIDE_EFFECT: Writes AssistantMessageRecord rows and commits or rollbacks the DB session. -# @PRE: db session is writable and message payload is serializable. -# @POST: Message row is committed or persistence failure is logged. -# @INVARIANT: failed persistence attempts always rollback before returning. -def _persist_message( - db: Session, - user_id: str, - conversation_id: str, - role: str, - text: str, - state: Optional[str] = None, - task_id: Optional[str] = None, - confirmation_id: Optional[str] = None, - metadata: Optional[Dict[str, Any]] = None, -): - try: - row = AssistantMessageRecord( - id=str(uuid.uuid4()), - user_id=user_id, - conversation_id=conversation_id, - role=role, - text=text, - state=state, - task_id=task_id, - confirmation_id=confirmation_id, - payload=metadata, - ) - db.add(row) - db.commit() - except Exception as exc: - db.rollback() - logger.warning(f"[assistant.message][persist_failed] {exc}") - - -# [/DEF:_persist_message:Function] - - -# [DEF:_audit:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Append in-memory audit record for assistant decision trace. -# @DATA_CONTRACT: Input[user_id,payload:Dict[str,Any]] -> Output[None] -# @RELATION: UPDATES -> [ASSISTANT_AUDIT] -# @SIDE_EFFECT: Mutates in-memory ASSISTANT_AUDIT store and emits structured log event. -# @PRE: payload describes decision/outcome fields. -# @POST: ASSISTANT_AUDIT list for user contains new timestamped entry. -# @INVARIANT: persisted in-memory audit entry always contains created_at in ISO format. -def _audit(user_id: str, payload: Dict[str, Any]): - if user_id not in ASSISTANT_AUDIT: - ASSISTANT_AUDIT[user_id] = [] - ASSISTANT_AUDIT[user_id].append( - {**payload, "created_at": datetime.utcnow().isoformat()} - ) - logger.info(f"[assistant.audit] {payload}") - - -# [/DEF:_audit:Function] - - -# [DEF:_persist_audit:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Persist structured assistant audit payload in database. -# @PRE: db session is writable and payload is JSON-serializable. -# @POST: Audit row is committed or failure is logged with rollback. -def _persist_audit( - db: Session, user_id: str, payload: Dict[str, Any], conversation_id: Optional[str] -): - try: - row = AssistantAuditRecord( - id=str(uuid.uuid4()), - user_id=user_id, - conversation_id=conversation_id, - decision=payload.get("decision"), - task_id=payload.get("task_id"), - message=payload.get("message"), - payload=payload, - ) - db.add(row) - db.commit() - except Exception as exc: - db.rollback() - logger.warning(f"[assistant.audit][persist_failed] {exc}") - - -# [/DEF:_persist_audit:Function] - - -# [DEF:_persist_confirmation:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Persist confirmation token record to database. -# @PRE: record contains id/user/intent/dispatch/expiry fields. -# @POST: Confirmation row exists in persistent storage. -def _persist_confirmation(db: Session, record: ConfirmationRecord): - try: - row = AssistantConfirmationRecord( - id=record.id, - user_id=record.user_id, - conversation_id=record.conversation_id, - state=record.state, - intent=record.intent, - dispatch=record.dispatch, - expires_at=record.expires_at, - created_at=record.created_at, - consumed_at=None, - ) - db.merge(row) - db.commit() - except Exception as exc: - db.rollback() - logger.warning(f"[assistant.confirmation][persist_failed] {exc}") - - -# [/DEF:_persist_confirmation:Function] - - -# [DEF:_update_confirmation_state:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Update persistent confirmation token lifecycle state. -# @PRE: confirmation_id references existing row. -# @POST: State and consumed_at fields are updated when applicable. -def _update_confirmation_state(db: Session, confirmation_id: str, state: str): - try: - row = ( - db.query(AssistantConfirmationRecord) - .filter(AssistantConfirmationRecord.id == confirmation_id) - .first() - ) - if not row: - return - row.state = state - if state in {"consumed", "expired", "cancelled"}: - row.consumed_at = datetime.utcnow() - db.commit() - except Exception as exc: - db.rollback() - logger.warning(f"[assistant.confirmation][update_failed] {exc}") - - -# [/DEF:_update_confirmation_state:Function] - - -# [DEF:_load_confirmation_from_db:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Load confirmation token from database into in-memory model. -# @PRE: confirmation_id may or may not exist in storage. -# @POST: Returns ConfirmationRecord when found, otherwise None. -def _load_confirmation_from_db( - db: Session, confirmation_id: str -) -> Optional[ConfirmationRecord]: - row = ( - db.query(AssistantConfirmationRecord) - .filter(AssistantConfirmationRecord.id == confirmation_id) - .first() - ) - if not row: - return None - return ConfirmationRecord( - id=row.id, - user_id=row.user_id, - conversation_id=row.conversation_id, - intent=row.intent or {}, - dispatch=row.dispatch or {}, - expires_at=row.expires_at, - state=row.state, - created_at=row.created_at, - ) - - -# [/DEF:_load_confirmation_from_db:Function] - - -# [DEF:_ensure_conversation:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve active conversation id in memory or create a new one. -# @PRE: user_id identifies current actor. -# @POST: Returns stable conversation id and updates USER_ACTIVE_CONVERSATION. -def _ensure_conversation(user_id: str, conversation_id: Optional[str]) -> str: - if conversation_id: - USER_ACTIVE_CONVERSATION[user_id] = conversation_id - return conversation_id - - active = USER_ACTIVE_CONVERSATION.get(user_id) - if active: - return active - - new_id = str(uuid.uuid4()) - USER_ACTIVE_CONVERSATION[user_id] = new_id - return new_id - - -# [/DEF:_ensure_conversation:Function] - - -# [DEF:_resolve_or_create_conversation:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve active conversation using explicit id, memory cache, or persisted history. -# @PRE: user_id and db session are available. -# @POST: Returns conversation id and updates USER_ACTIVE_CONVERSATION cache. -def _resolve_or_create_conversation( - user_id: str, conversation_id: Optional[str], db: Session -) -> str: - if conversation_id: - USER_ACTIVE_CONVERSATION[user_id] = conversation_id - return conversation_id - - active = USER_ACTIVE_CONVERSATION.get(user_id) - if active: - return active - - last_message = ( - db.query(AssistantMessageRecord) - .filter(AssistantMessageRecord.user_id == user_id) - .order_by(desc(AssistantMessageRecord.created_at)) - .first() - ) - if last_message: - USER_ACTIVE_CONVERSATION[user_id] = last_message.conversation_id - return last_message.conversation_id - - new_id = str(uuid.uuid4()) - USER_ACTIVE_CONVERSATION[user_id] = new_id - return new_id - - -# [/DEF:_resolve_or_create_conversation:Function] - - -# [DEF:_cleanup_history_ttl:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Enforce assistant message retention window by deleting expired rows and in-memory records. -# @PRE: db session is available and user_id references current actor scope. -# @POST: Messages older than ASSISTANT_MESSAGE_TTL_DAYS are removed from persistence and memory mirrors. -def _cleanup_history_ttl(db: Session, user_id: str): - cutoff = datetime.utcnow() - timedelta(days=ASSISTANT_MESSAGE_TTL_DAYS) - try: - query = db.query(AssistantMessageRecord).filter( - AssistantMessageRecord.user_id == user_id, - AssistantMessageRecord.created_at < cutoff, - ) - if hasattr(query, "delete"): - query.delete(synchronize_session=False) - db.commit() - except Exception as exc: - db.rollback() - logger.warning( - f"[assistant.history][ttl_cleanup_failed] user={user_id} error={exc}" - ) - - stale_keys: List[Tuple[str, str]] = [] - for key, items in CONVERSATIONS.items(): - if key[0] != user_id: - continue - kept = [] - for item in items: - created_at = item.get("created_at") - if isinstance(created_at, datetime) and created_at < cutoff: - continue - kept.append(item) - if kept: - CONVERSATIONS[key] = kept - else: - stale_keys.append(key) - for key in stale_keys: - CONVERSATIONS.pop(key, None) - - -# [/DEF:_cleanup_history_ttl:Function] - - -# [DEF:_is_conversation_archived:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Determine archived state for a conversation based on last update timestamp. -# @PRE: updated_at can be null for empty conversations. -# @POST: Returns True when conversation inactivity exceeds archive threshold. -def _is_conversation_archived(updated_at: Optional[datetime]) -> bool: - if not updated_at: - return False - cutoff = datetime.utcnow() - timedelta(days=ASSISTANT_ARCHIVE_AFTER_DAYS) - return updated_at < cutoff - - -# [/DEF:_is_conversation_archived:Function] - - -# [DEF:_coerce_query_bool:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Normalize bool-like query values for compatibility in direct handler invocations/tests. -# @PRE: value may be bool, string, or FastAPI Query metadata object. -# @POST: Returns deterministic boolean flag. -def _coerce_query_bool(value: Any) -> bool: - if isinstance(value, bool): - return value - if isinstance(value, str): - return value.strip().lower() in {"1", "true", "yes", "on"} - return False - - -# [/DEF:_coerce_query_bool:Function] - - -# [DEF:_extract_id:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Extract first regex match group from text by ordered pattern list. -# @PRE: patterns contain at least one capture group. -# @POST: Returns first matched token or None. -def _extract_id(text: str, patterns: List[str]) -> Optional[str]: - for p in patterns: - m = re.search(p, text, flags=re.IGNORECASE) - if m: - return m.group(1) - return None - - -# [/DEF:_extract_id:Function] - - -# [DEF:_resolve_env_id:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve environment identifier/name token to canonical environment id. -# @PRE: config_manager provides environment list. -# @POST: Returns matched environment id or None. -def _resolve_env_id( - token: Optional[str], config_manager: ConfigManager -) -> Optional[str]: - if not token: - return None - - normalized = token.strip().lower() - envs = config_manager.get_environments() - for env in envs: - if env.id.lower() == normalized or env.name.lower() == normalized: - return env.id - return None - - -# [/DEF:_resolve_env_id:Function] - - -# [DEF:_is_production_env:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Determine whether environment token resolves to production-like target. -# @PRE: config_manager provides environments or token text is provided. -# @POST: Returns True for production/prod synonyms, else False. -def _is_production_env(token: Optional[str], config_manager: ConfigManager) -> bool: - env_id = _resolve_env_id(token, config_manager) - if not env_id: - return (token or "").strip().lower() in {"prod", "production", "прод"} - - env = next((e for e in config_manager.get_environments() if e.id == env_id), None) - if not env: - return False - target = f"{env.id} {env.name}".lower() - return "prod" in target or "production" in target or "прод" in target - - -# [/DEF:_is_production_env:Function] - - -# [DEF:_resolve_provider_id:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve provider token to provider id with active/default fallback. -# @PRE: db session can load provider list through LLMProviderService. -# @POST: Returns provider id or None when no providers configured. -def _resolve_provider_id( - provider_token: Optional[str], - db: Session, - config_manager: Optional[ConfigManager] = None, - task_key: Optional[str] = None, -) -> Optional[str]: - service = LLMProviderService(db) - providers = service.get_all_providers() - if not providers: - return None - - if provider_token: - needle = provider_token.strip().lower() - for p in providers: - if p.id.lower() == needle or p.name.lower() == needle: - return p.id - - if config_manager and task_key: - try: - llm_settings = config_manager.get_config().settings.llm - bound_provider_id = resolve_bound_provider_id(llm_settings, task_key) - if bound_provider_id and any(p.id == bound_provider_id for p in providers): - return bound_provider_id - except Exception: - pass - - active = next((p for p in providers if p.is_active), None) - return active.id if active else providers[0].id - - -# [/DEF:_resolve_provider_id:Function] - - -# [DEF:_get_default_environment_id:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve default environment id from settings or first configured environment. -# @PRE: config_manager returns environments list. -# @POST: Returns default environment id or None when environment list is empty. -def _get_default_environment_id(config_manager: ConfigManager) -> Optional[str]: - configured = config_manager.get_environments() - if not configured: - return None - preferred = None - if hasattr(config_manager, "get_config"): - try: - preferred = config_manager.get_config().settings.default_environment_id - except Exception: - preferred = None - if preferred and any(env.id == preferred for env in configured): - return preferred - explicit_default = next( - (env.id for env in configured if getattr(env, "is_default", False)), None - ) - return explicit_default or configured[0].id - - -# [/DEF:_get_default_environment_id:Function] - - -# [DEF:_resolve_dashboard_id_by_ref:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve dashboard id by title or slug reference in selected environment. -# @PRE: dashboard_ref is a non-empty string-like token. -# @POST: Returns dashboard id when uniquely matched, otherwise None. -def _resolve_dashboard_id_by_ref( - dashboard_ref: Optional[str], - env_id: Optional[str], - config_manager: ConfigManager, -) -> Optional[int]: - if not dashboard_ref or not env_id: - return None - env = next( - (item for item in config_manager.get_environments() if item.id == env_id), None - ) - if not env: - return None - - needle = dashboard_ref.strip().lower() - try: - client = SupersetClient(env) - _, dashboards = client.get_dashboards(query={"page_size": 200}) - except Exception as exc: - logger.warning( - f"[assistant.dashboard_resolve][failed] ref={dashboard_ref} env={env_id} error={exc}" - ) - return None - - exact = next( - ( - d - for d in dashboards - if str(d.get("slug", "")).lower() == needle - or str(d.get("dashboard_title", "")).lower() == needle - or str(d.get("title", "")).lower() == needle - ), - None, - ) - if exact: - return int(exact.get("id")) - - partial = [ - d - for d in dashboards - if needle in str(d.get("dashboard_title", d.get("title", ""))).lower() - ] - if len(partial) == 1 and partial[0].get("id") is not None: - return int(partial[0]["id"]) - return None - - -# [/DEF:_resolve_dashboard_id_by_ref:Function] - - -# [DEF:_resolve_dashboard_id_entity:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve dashboard id from intent entities using numeric id or dashboard_ref fallback. -# @PRE: entities may contain dashboard_id as int/str and optional dashboard_ref. -# @POST: Returns resolved dashboard id or None when ambiguous/unresolvable. -def _resolve_dashboard_id_entity( - entities: Dict[str, Any], - config_manager: ConfigManager, - env_hint: Optional[str] = None, -) -> Optional[int]: - raw_dashboard_id = entities.get("dashboard_id") - dashboard_ref = entities.get("dashboard_ref") - - if isinstance(raw_dashboard_id, int): - return raw_dashboard_id - - if isinstance(raw_dashboard_id, str): - token = raw_dashboard_id.strip() - if token.isdigit(): - return int(token) - if token and not dashboard_ref: - dashboard_ref = token - - if not dashboard_ref: - return None - - env_token = ( - env_hint - or entities.get("environment") - or entities.get("source_env") - or entities.get("target_env") - ) - env_id = ( - _resolve_env_id(env_token, config_manager) - if env_token - else _get_default_environment_id(config_manager) - ) - return _resolve_dashboard_id_by_ref(str(dashboard_ref), env_id, config_manager) - - -# [/DEF:_resolve_dashboard_id_entity:Function] - - -# [DEF:_get_environment_name_by_id:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve human-readable environment name by id. -# @PRE: environment id may be None. -# @POST: Returns matching environment name or fallback id. -def _get_environment_name_by_id( - env_id: Optional[str], config_manager: ConfigManager -) -> str: - if not env_id: - return "unknown" - env = next( - (item for item in config_manager.get_environments() if item.id == env_id), None - ) - return env.name if env else env_id - - -# [/DEF:_get_environment_name_by_id:Function] - - -# [DEF:_extract_result_deep_links:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Build deep-link actions to verify task result from assistant chat. -# @PRE: task object is available. -# @POST: Returns zero or more assistant actions for dashboard open/diff. -def _extract_result_deep_links( - task: Any, config_manager: ConfigManager -) -> List[AssistantAction]: - plugin_id = getattr(task, "plugin_id", None) - params = getattr(task, "params", {}) or {} - result = getattr(task, "result", {}) or {} - actions: List[AssistantAction] = [] - dashboard_id: Optional[int] = None - env_id: Optional[str] = None - - if plugin_id == "superset-migration": - migrated = ( - result.get("migrated_dashboards") if isinstance(result, dict) else None - ) - if isinstance(migrated, list) and migrated: - first = migrated[0] - if isinstance(first, dict) and first.get("id") is not None: - dashboard_id = int(first.get("id")) - if ( - dashboard_id is None - and isinstance(params.get("selected_ids"), list) - and params["selected_ids"] - ): - dashboard_id = int(params["selected_ids"][0]) - env_id = params.get("target_env_id") - elif plugin_id == "superset-backup": - dashboards = result.get("dashboards") if isinstance(result, dict) else None - if isinstance(dashboards, list) and dashboards: - first = dashboards[0] - if isinstance(first, dict) and first.get("id") is not None: - dashboard_id = int(first.get("id")) - if ( - dashboard_id is None - and isinstance(params.get("dashboard_ids"), list) - and params["dashboard_ids"] - ): - dashboard_id = int(params["dashboard_ids"][0]) - env_id = params.get("environment_id") or _resolve_env_id( - result.get("environment"), config_manager - ) - elif plugin_id == "llm_dashboard_validation": - if params.get("dashboard_id") is not None: - dashboard_id = int(params["dashboard_id"]) - env_id = params.get("environment_id") - - if dashboard_id is not None and env_id: - env_name = _get_environment_name_by_id(env_id, config_manager) - actions.append( - AssistantAction( - type="open_route", - label=f"Открыть дашборд в {env_name}", - target=f"/dashboards/{dashboard_id}?env_id={env_id}", - ) - ) - if dashboard_id is not None: - actions.append( - AssistantAction( - type="open_diff", - label="Показать Diff", - target=str(dashboard_id), - ) - ) - return actions - - -# [/DEF:_extract_result_deep_links:Function] - - -# [DEF:_build_task_observability_summary:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Build compact textual summary for completed tasks to reduce "black box" effect. -# @PRE: task may contain plugin-specific result payload. -# @POST: Returns non-empty summary line for known task types or empty string fallback. -def _build_task_observability_summary(task: Any, config_manager: ConfigManager) -> str: - plugin_id = getattr(task, "plugin_id", None) - status = str(getattr(task, "status", "")).upper() - params = getattr(task, "params", {}) or {} - result = getattr(task, "result", {}) or {} - - if plugin_id == "superset-migration" and isinstance(result, dict): - migrated = len(result.get("migrated_dashboards") or []) - failed_rows = result.get("failed_dashboards") or [] - failed = len(failed_rows) - selected = result.get("selected_dashboards", migrated + failed) - mappings = result.get("mapping_count", 0) - target_env_id = params.get("target_env_id") - target_env_name = _get_environment_name_by_id(target_env_id, config_manager) - warning = "" - if failed_rows: - first = failed_rows[0] - warning = ( - f" Внимание: {first.get('title') or first.get('id')}: " - f"{first.get('error') or 'ошибка'}." - ) - return ( - f"Сводка миграции: выбрано {selected}, перенесено {migrated}, " - f"с ошибками {failed}, маппингов {mappings}, целевая среда {target_env_name}." - f"{warning}" - ) - - if plugin_id == "superset-backup" and isinstance(result, dict): - total = int(result.get("total_dashboards", 0) or 0) - ok = int(result.get("backed_up_dashboards", 0) or 0) - failed = int(result.get("failed_dashboards", 0) or 0) - env_id = params.get("environment_id") or _resolve_env_id( - result.get("environment"), config_manager - ) - env_name = _get_environment_name_by_id(env_id, config_manager) - failures = result.get("failures") or [] - warning = "" - if failures: - first = failures[0] - warning = ( - f" Внимание: {first.get('title') or first.get('id')}: " - f"{first.get('error') or 'ошибка'}." - ) - return ( - f"Сводка бэкапа: среда {env_name}, всего {total}, успешно {ok}, " - f"с ошибками {failed}. {status}.{warning}" - ) - - if plugin_id == "llm_dashboard_validation" and isinstance(result, dict): - report_status = result.get("status") or status - report_summary = result.get("summary") or "Итог недоступен." - issues = result.get("issues") or [] - return f"Сводка валидации: статус {report_status}, проблем {len(issues)}. {report_summary}" - - # Fallback for unknown task payloads. - if status in {"SUCCESS", "FAILED"}: - return f"Задача завершена со статусом {status}." - return "" - - -# [/DEF:_build_task_observability_summary:Function] - - -from src.logger import belief_scope, logger - -# [DEF:_parse_command:Function] -# @COMPLEXITY: 4 -# @PURPOSE: Deterministically parse RU/EN command text into intent payload. -# @DATA_CONTRACT: Input[message:str, config_manager:ConfigManager] -> Output[Dict[str,Any]{domain,operation,entities,confidence,risk_level,requires_confirmation}] -# @RELATION: DEPENDS_ON -> [_extract_id] -# @RELATION: DEPENDS_ON -> [_is_production_env] -# @SIDE_EFFECT: None (pure parsing logic). -# @PRE: message contains raw user text and config manager resolves environments. -# @POST: Returns intent dict with domain/operation/entities/confidence/risk fields. -# @INVARIANT: every return path includes domain, operation, entities, confidence, risk_level, requires_confirmation. -def _parse_command(message: str, config_manager: ConfigManager) -> Dict[str, Any]: - with belief_scope('_parse_command'): - logger.reason('Belief protocol reasoning checkpoint for _parse_command') - text = message.strip() - lower = text.lower() - if any((phrase in lower for phrase in ['что ты умеешь', 'что умеешь', 'что ты можешь', 'help', 'помощь', 'доступные команды', 'какие команды'])): - logger.reflect('Belief protocol postcondition checkpoint for _parse_command') - return {'domain': 'assistant', 'operation': 'show_capabilities', 'entities': {}, 'confidence': 0.98, 'risk_level': 'safe', 'requires_confirmation': False} - dashboard_id = _extract_id(lower, ['(?:дашборд\\w*|dashboard)\\s*(?:id\\s*)?(\\d+)']) - dashboard_ref = _extract_id(lower, ['(?:дашборд\\w*|dashboard)\\s*(?:id\\s*)?([a-zа-я0-9._-]+)']) - dataset_id = _extract_id(lower, ['(?:датасет\\w*|dataset)\\s*(?:id\\s*)?(\\d+)']) - task_id = _extract_id(lower, ['(task[-_a-z0-9]{1,}|[0-9a-f]{8}-[0-9a-f-]{27,})']) - if any((k in lower for k in ['статус', 'status', 'state', 'проверь задачу'])): - logger.reflect('Belief protocol postcondition checkpoint for _parse_command') - return {'domain': 'status', 'operation': 'get_task_status', 'entities': {'task_id': task_id}, 'confidence': 0.92 if task_id else 0.66, 'risk_level': 'safe', 'requires_confirmation': False} - if any((k in lower for k in ['ветк', 'branch'])) and any((k in lower for k in ['созд', 'сделай', 'create'])): - branch = _extract_id(lower, ['(?:ветк\\w*|branch)\\s+([a-z0-9._/-]+)']) - logger.reflect('Belief protocol postcondition checkpoint for _parse_command') - return {'domain': 'git', 'operation': 'create_branch', 'entities': {'dashboard_id': int(dashboard_id) if dashboard_id else None, 'branch_name': branch}, 'confidence': 0.95 if branch and dashboard_id else 0.7, 'risk_level': 'guarded', 'requires_confirmation': False} - if any((k in lower for k in ['коммит', 'commit'])): - quoted = re.search('"([^"]{3,120})"', text) - message_text = quoted.group(1) if quoted else 'assistant: update dashboard changes' - logger.reflect('Belief protocol postcondition checkpoint for _parse_command') - return {'domain': 'git', 'operation': 'commit_changes', 'entities': {'dashboard_id': int(dashboard_id) if dashboard_id else None, 'message': message_text}, 'confidence': 0.9 if dashboard_id else 0.7, 'risk_level': 'guarded', 'requires_confirmation': False} - if any((k in lower for k in ['деплой', 'deploy', 'разверн'])): - env_match = _extract_id(lower, ['(?:в|to)\\s+([a-z0-9_-]+)']) - is_dangerous = _is_production_env(env_match, config_manager) - logger.reflect('Belief protocol postcondition checkpoint for _parse_command') - return {'domain': 'git', 'operation': 'deploy_dashboard', 'entities': {'dashboard_id': int(dashboard_id) if dashboard_id else None, 'environment': env_match}, 'confidence': 0.92 if dashboard_id and env_match else 0.7, 'risk_level': 'dangerous' if is_dangerous else 'guarded', 'requires_confirmation': is_dangerous} - if any((k in lower for k in ['миграц', 'migration', 'migrate'])): - src = _extract_id(lower, ['(?:с|from)\\s+([a-z0-9_-]+)']) - tgt = _extract_id(lower, ['(?:на|to)\\s+([a-z0-9_-]+)']) - dry_run = '--dry-run' in lower or 'dry run' in lower - replace_db_config = '--replace-db-config' in lower - fix_cross_filters = '--no-fix-cross-filters' not in lower - is_dangerous = _is_production_env(tgt, config_manager) - logger.reflect('Belief protocol postcondition checkpoint for _parse_command') - return {'domain': 'migration', 'operation': 'execute_migration', 'entities': {'dashboard_id': int(dashboard_id) if dashboard_id else None, 'source_env': src, 'target_env': tgt, 'dry_run': dry_run, 'replace_db_config': replace_db_config, 'fix_cross_filters': fix_cross_filters}, 'confidence': 0.95 if dashboard_id and src and tgt else 0.72, 'risk_level': 'dangerous' if is_dangerous else 'guarded', 'requires_confirmation': is_dangerous or dry_run} - if any((k in lower for k in ['бэкап', 'backup', 'резерв'])): - env_match = _extract_id(lower, ['(?:в|for|из|from)\\s+([a-z0-9_-]+)']) - logger.reflect('Belief protocol postcondition checkpoint for _parse_command') - return {'domain': 'backup', 'operation': 'run_backup', 'entities': {'dashboard_id': int(dashboard_id) if dashboard_id else None, 'environment': env_match}, 'confidence': 0.9 if env_match else 0.7, 'risk_level': 'guarded', 'requires_confirmation': False} - if any((k in lower for k in ['здоровье', 'health', 'ошибки', 'failing', 'проблемы'])): - env_match = _extract_id(lower, ['(?:в|for|env|окружени[ея])\\s+([a-z0-9_-]+)']) - logger.reflect('Belief protocol postcondition checkpoint for _parse_command') - return {'domain': 'health', 'operation': 'get_health_summary', 'entities': {'environment': env_match}, 'confidence': 0.9, 'risk_level': 'safe', 'requires_confirmation': False} - if any((k in lower for k in ['валидац', 'validate', 'провер'])): - env_match = _extract_id(lower, ['(?:в|for|env|окружени[ея])\\s+([a-z0-9_-]+)']) - provider_match = _extract_id(lower, ['(?:provider|провайдер)\\s+([a-z0-9_-]+)']) - logger.reflect('Belief protocol postcondition checkpoint for _parse_command') - return {'domain': 'llm', 'operation': 'run_llm_validation', 'entities': {'dashboard_id': int(dashboard_id) if dashboard_id else None, 'dashboard_ref': dashboard_ref if dashboard_ref and (not dashboard_ref.isdigit()) else None, 'environment': env_match, 'provider': provider_match}, 'confidence': 0.88 if dashboard_id else 0.64, 'risk_level': 'guarded', 'requires_confirmation': False} - if any((k in lower for k in ['документац', 'documentation', 'generate docs', 'сгенерируй док'])): - env_match = _extract_id(lower, ['(?:в|for|env|окружени[ея])\\s+([a-z0-9_-]+)']) - provider_match = _extract_id(lower, ['(?:provider|провайдер)\\s+([a-z0-9_-]+)']) - logger.reflect('Belief protocol postcondition checkpoint for _parse_command') - return {'domain': 'llm', 'operation': 'run_llm_documentation', 'entities': {'dataset_id': int(dataset_id) if dataset_id else None, 'environment': env_match, 'provider': provider_match}, 'confidence': 0.88 if dataset_id else 0.64, 'risk_level': 'guarded', 'requires_confirmation': False} - logger.reflect('Belief protocol postcondition checkpoint for _parse_command') - return {'domain': 'unknown', 'operation': 'clarify', 'entities': {}, 'confidence': 0.3, 'risk_level': 'safe', 'requires_confirmation': False} -# [/DEF:_parse_command:Function] - - -# [DEF:_check_any_permission:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Validate user against alternative permission checks (logical OR). -# @PRE: checks list contains resource-action tuples. -# @POST: Returns on first successful permission; raises 403-like HTTPException otherwise. -def _check_any_permission(current_user: User, checks: List[Tuple[str, str]]): - errors: List[HTTPException] = [] - for resource, action in checks: - try: - has_permission(resource, action)(current_user) - return - except HTTPException as exc: - errors.append(exc) - - raise ( - errors[-1] - if errors - else HTTPException(status_code=403, detail="Permission denied") - ) - - -# [/DEF:_check_any_permission:Function] - - -# [DEF:_has_any_permission:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Check whether user has at least one permission tuple from the provided list. -# @PRE: current_user and checks list are valid. -# @POST: Returns True when at least one permission check passes. -def _has_any_permission(current_user: User, checks: List[Tuple[str, str]]) -> bool: - try: - _check_any_permission(current_user, checks) - return True - except HTTPException: - return False - - -# [/DEF:_has_any_permission:Function] - - -# [DEF:_build_tool_catalog:Function] -# @COMPLEXITY: 3 -# @PURPOSE: Build current-user tool catalog for LLM planner with operation contracts and defaults. -# @PRE: current_user is authenticated; config/db are available. -# @POST: Returns list of executable tools filtered by permission and runtime availability. -# @RELATION: CALLS -> LLMProviderService -def _build_tool_catalog( - current_user: User, - config_manager: ConfigManager, - db: Session, - dataset_review_context: Optional[Dict[str, Any]] = None, -) -> List[Dict[str, Any]]: - envs = config_manager.get_environments() - default_env_id = _get_default_environment_id(config_manager) - providers = LLMProviderService(db).get_all_providers() - llm_settings = {} - try: - llm_settings = config_manager.get_config().settings.llm - except Exception: - llm_settings = {} - active_provider = next((p.id for p in providers if p.is_active), None) - fallback_provider = active_provider or (providers[0].id if providers else None) - validation_provider = ( - resolve_bound_provider_id(llm_settings, "dashboard_validation") - or fallback_provider - ) - documentation_provider = ( - resolve_bound_provider_id(llm_settings, "documentation") or fallback_provider - ) - - candidates: List[Dict[str, Any]] = [ - { - "operation": "show_capabilities", - "domain": "assistant", - "description": "Show available assistant commands and examples", - "required_entities": [], - "optional_entities": [], - "risk_level": "safe", - "requires_confirmation": False, - }, - { - "operation": "get_task_status", - "domain": "status", - "description": "Get task status by task_id or latest user task", - "required_entities": [], - "optional_entities": ["task_id"], - "risk_level": "safe", - "requires_confirmation": False, - }, - { - "operation": "create_branch", - "domain": "git", - "description": "Create git branch for dashboard by id/slug/title", - "required_entities": ["branch_name"], - "optional_entities": ["dashboard_id", "dashboard_ref"], - "risk_level": "guarded", - "requires_confirmation": False, - }, - { - "operation": "commit_changes", - "domain": "git", - "description": "Commit dashboard repository changes by dashboard id/slug/title", - "required_entities": [], - "optional_entities": ["dashboard_id", "dashboard_ref", "message"], - "risk_level": "guarded", - "requires_confirmation": False, - }, - { - "operation": "deploy_dashboard", - "domain": "git", - "description": "Deploy dashboard (id/slug/title) to target environment", - "required_entities": ["environment"], - "optional_entities": ["dashboard_id", "dashboard_ref"], - "risk_level": "guarded", - "requires_confirmation": False, - }, - { - "operation": "execute_migration", - "domain": "migration", - "description": "Run dashboard migration (id/slug/title) between environments. Optional boolean flags: replace_db_config, fix_cross_filters", - "required_entities": ["source_env", "target_env"], - "optional_entities": [ - "dashboard_id", - "dashboard_ref", - "replace_db_config", - "fix_cross_filters", - ], - "risk_level": "guarded", - "requires_confirmation": False, - }, - { - "operation": "run_backup", - "domain": "backup", - "description": "Run backup for environment or specific dashboard by id/slug/title", - "required_entities": ["environment"], - "optional_entities": ["dashboard_id", "dashboard_ref"], - "risk_level": "guarded", - "requires_confirmation": False, - }, - { - "operation": "run_llm_validation", - "domain": "llm", - "description": "Run LLM dashboard validation by dashboard id/slug/title", - "required_entities": [], - "optional_entities": ["dashboard_ref", "environment", "provider"], - "defaults": { - "environment": default_env_id, - "provider": validation_provider, - }, - "risk_level": "guarded", - "requires_confirmation": False, - }, - { - "operation": "run_llm_documentation", - "domain": "llm", - "description": "Generate dataset documentation via LLM", - "required_entities": ["dataset_id"], - "optional_entities": ["environment", "provider"], - "defaults": { - "environment": default_env_id, - "provider": documentation_provider, - }, - "risk_level": "guarded", - "requires_confirmation": False, - }, - { - "operation": "get_health_summary", - "domain": "health", - "description": "Get summary of dashboard health and failing validations", - "required_entities": [], - "optional_entities": ["environment"], - "risk_level": "safe", - "requires_confirmation": False, - }, - ] - - available: List[Dict[str, Any]] = [] - for tool in candidates: - checks = INTENT_PERMISSION_CHECKS.get(tool["operation"], []) - if checks and not _has_any_permission(current_user, checks): - continue - available.append(tool) - - if dataset_review_context is not None: - dataset_tools: List[Dict[str, Any]] = [ - { - "operation": "dataset_review_answer_context", - "domain": "dataset_review", - "description": "Answer questions using the currently bound dataset review session context", - "required_entities": ["dataset_review_session_id"], - "optional_entities": [], - "risk_level": "safe", - "requires_confirmation": False, - }, - { - "operation": "dataset_review_approve_mappings", - "domain": "dataset_review", - "description": "Approve warning-sensitive execution mappings in the current dataset review session", - "required_entities": ["dataset_review_session_id", "session_version"], - "optional_entities": ["mapping_ids"], - "risk_level": "guarded", - "requires_confirmation": True, - }, - { - "operation": "dataset_review_set_field_semantics", - "domain": "dataset_review", - "description": "Apply explicit semantic field override or candidate selection in the current dataset review session", - "required_entities": [ - "dataset_review_session_id", - "session_version", - "field_id", - ], - "optional_entities": [ - "candidate_id", - "verbose_name", - "description", - "display_format", - "lock_field", - ], - "risk_level": "guarded", - "requires_confirmation": True, - }, - { - "operation": "dataset_review_generate_sql_preview", - "domain": "dataset_review", - "description": "Generate a Superset-compiled SQL preview for the current dataset review session", - "required_entities": ["dataset_review_session_id", "session_version"], - "optional_entities": [], - "risk_level": "guarded", - "requires_confirmation": True, - }, - ] - for tool in dataset_tools: - checks = INTENT_PERMISSION_CHECKS.get(tool["operation"], []) - if checks and not _has_any_permission(current_user, checks): - continue - available.append(tool) - return available - - -# [/DEF:_build_tool_catalog:Function] - - -# [DEF:_coerce_intent_entities:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Normalize intent entity value types from LLM output to route-compatible values. -# @PRE: intent contains entities dict or missing entities. -# @POST: Returned intent has numeric ids coerced where possible and string values stripped. -def _coerce_intent_entities(intent: Dict[str, Any]) -> Dict[str, Any]: - entities = intent.get("entities") - if not isinstance(entities, dict): - intent["entities"] = {} - entities = intent["entities"] - for key in ("dashboard_id", "dataset_id"): - value = entities.get(key) - if isinstance(value, str) and value.strip().isdigit(): - entities[key] = int(value.strip()) - for key, value in list(entities.items()): - if isinstance(value, str): - entities[key] = value.strip() - return intent - - -# [/DEF:_coerce_intent_entities:Function] - - -# Operations that are read-only and do not require confirmation. -_SAFE_OPS = { - "show_capabilities", - "get_task_status", - "get_health_summary", - "dataset_review_answer_context", -} - -_DATASET_REVIEW_OPS = { - "dataset_review_approve_mappings", - "dataset_review_set_field_semantics", - "dataset_review_generate_sql_preview", -} - - -# [DEF:_serialize_dataset_review_context:Function] -# @COMPLEXITY: 4 -# @PURPOSE: Build assistant-safe dataset-review context snapshot with masked imported-filter payloads for session-scoped assistant routing. -# @RELATION: [DEPENDS_ON] ->[DatasetReviewSession] -# @PRE: session_id is a valid active review session identifier. -# @POST: Returns a serializable dictionary containing the complete review context. -# @SIDE_EFFECT: Reads session data from the database. -def _serialize_dataset_review_context(session: DatasetReviewSession) -> Dict[str, Any]: - with belief_scope('_serialize_dataset_review_context'): - logger.reason('Belief protocol reasoning checkpoint for _serialize_dataset_review_context') - latest_preview = None - previews = getattr(session, 'previews', []) or [] - if previews: - latest_preview = previews[-1] - logger.reflect('Belief protocol postcondition checkpoint for _serialize_dataset_review_context') - return {'session_id': session.session_id, 'version': int(getattr(session, 'version', 0) or 0), 'dataset_ref': session.dataset_ref, 'dataset_id': session.dataset_id, 'environment_id': session.environment_id, 'readiness_state': session.readiness_state.value, 'recommended_action': session.recommended_action.value, 'status': session.status.value, 'current_phase': session.current_phase.value, 'findings': [{'finding_id': item.finding_id, 'code': item.code, 'severity': item.severity.value, 'message': item.message, 'resolution_state': item.resolution_state.value} for item in getattr(session, 'findings', [])], 'imported_filters': [sanitize_imported_filter_for_assistant({'filter_id': item.filter_id, 'filter_name': item.filter_name, 'display_name': item.display_name, 'raw_value': item.raw_value, 'raw_value_masked': bool(getattr(item, 'raw_value_masked', False)), 'normalized_value': item.normalized_value, 'source': getattr(item.source, 'value', item.source), 'confidence_state': getattr(item.confidence_state, 'value', item.confidence_state), 'requires_confirmation': bool(item.requires_confirmation), 'recovery_status': getattr(item.recovery_status, 'value', item.recovery_status), 'notes': item.notes}) for item in getattr(session, 'imported_filters', [])], 'mappings': [{'mapping_id': item.mapping_id, 'filter_id': item.filter_id, 'variable_id': item.variable_id, 'mapping_method': getattr(item.mapping_method, 'value', item.mapping_method), 'effective_value': item.effective_value, 'approval_state': getattr(item.approval_state, 'value', item.approval_state), 'requires_explicit_approval': bool(item.requires_explicit_approval)} for item in getattr(session, 'execution_mappings', [])], 'semantic_fields': [{'field_id': item.field_id, 'field_name': item.field_name, 'verbose_name': item.verbose_name, 'description': item.description, 'display_format': item.display_format, 'provenance': getattr(item.provenance, 'value', item.provenance), 'is_locked': bool(item.is_locked), 'needs_review': bool(item.needs_review), 'candidates': [{'candidate_id': candidate.candidate_id, 'proposed_verbose_name': candidate.proposed_verbose_name, 'proposed_description': candidate.proposed_description, 'proposed_display_format': candidate.proposed_display_format, 'status': getattr(candidate.status, 'value', candidate.status)} for candidate in getattr(item, 'candidates', [])]} for item in getattr(session, 'semantic_fields', [])], 'preview': {'preview_id': latest_preview.preview_id, 'preview_status': getattr(latest_preview.preview_status, 'value', latest_preview.preview_status), 'compiled_sql': latest_preview.compiled_sql} if latest_preview is not None else None} -# [/DEF:_serialize_dataset_review_context:Function] - - -# [DEF:_load_dataset_review_context:Function] -# @COMPLEXITY: 4 -# @PURPOSE: Load owner-scoped dataset-review context for assistant planning and grounded response generation. -# @RELATION: [DEPENDS_ON] ->[DatasetReviewSessionRepository] -# @PRE: session_id is a valid active review session identifier. -# @POST: Returns a loaded context object with session data and findings. -# @SIDE_EFFECT: Reads session data from the database. -def _load_dataset_review_context(dataset_review_session_id: Optional[str], current_user: User, db: Session) -> Optional[Dict[str, Any]]: - with belief_scope('_load_dataset_review_context'): - if not dataset_review_session_id: - return None - logger.reason('Belief protocol reasoning checkpoint for _load_dataset_review_context') - repository = DatasetReviewSessionRepository(db) - session = repository.load_session_detail(dataset_review_session_id, current_user.id) - if session is None or session.user_id != current_user.id: - raise HTTPException(status_code=404, detail='Dataset review session not found') - logger.reflect('Belief protocol postcondition checkpoint for _load_dataset_review_context') - return _serialize_dataset_review_context(session) -# [/DEF:_load_dataset_review_context:Function] - - -# [DEF:_extract_dataset_review_target:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Extract structured dataset-review focus target hints embedded in assistant prompts. -def _extract_dataset_review_target(message: str) -> Tuple[Optional[str], Optional[str]]: - match = re.search( - r"(?:target|focus)\s*[:=]\s*(field|mapping|finding|filter)[:=]([A-Za-z0-9._-]+)", - str(message or ""), - re.IGNORECASE, - ) - if not match: - return None, None - return match.group(1).lower(), match.group(2) - - -# [/DEF:_extract_dataset_review_target:Function] - - -# [DEF:_match_dataset_review_field:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve one semantic field from assistant-visible context by id or user-visible label. -def _match_dataset_review_field( - dataset_context: Dict[str, Any], - message: str, -) -> Optional[Dict[str, Any]]: - target_kind, target_id = _extract_dataset_review_target(message) - fields = dataset_context.get("semantic_fields", []) or [] - if target_kind == "field" and target_id: - return next( - (item for item in fields if str(item.get("field_id")) == str(target_id)), - None, - ) - - normalized_message = str(message or "").lower() - for field in fields: - if str(field.get("field_id", "")).lower() in normalized_message: - return field - field_name = str(field.get("field_name", "")).lower() - if field_name and field_name in normalized_message: - return field - verbose_name = str(field.get("verbose_name", "")).lower() - if verbose_name and verbose_name in normalized_message: - return field - return None - - -# [/DEF:_match_dataset_review_field:Function] - - -# [DEF:_extract_quoted_segment:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Extract one quoted assistant command segment after a label token. -def _extract_quoted_segment(message: str, label: str) -> Optional[str]: - pattern = rf"{label}\s*[=:]?\s*[\"']([^\"']+)[\"']" - match = re.search(pattern, str(message or ""), re.IGNORECASE) - return match.group(1).strip() if match else None - - -# [/DEF:_extract_quoted_segment:Function] - - -# [DEF:_dataset_review_conflict_http_exception:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Convert dataset-review optimistic-lock conflicts into shared 409 assistant semantics. -def _dataset_review_conflict_http_exception( - exc: DatasetReviewSessionVersionConflictError, -) -> HTTPException: - return HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail={ - "error_code": "session_version_conflict", - "message": str(exc), - "session_id": exc.session_id, - "expected_version": exc.expected_version, - "actual_version": exc.actual_version, - }, - ) - - -# [/DEF:_dataset_review_conflict_http_exception:Function] - - -# [DEF:_plan_dataset_review_intent:Function] -# @COMPLEXITY: 3 -# @PURPOSE: Parse session-scoped dataset-review assistant commands before falling back to generic assistant tool routing. -# @RELATION: CALLS -> DatasetReviewOrchestrator -def _plan_dataset_review_intent( - message: str, - dataset_context: Dict[str, Any], -) -> Optional[Dict[str, Any]]: - lower = message.strip().lower() - session_id = dataset_context["session_id"] - session_version = int(dataset_context.get("version", 0) or 0) - target_kind, target_id = _extract_dataset_review_target(message) - - if any( - token in lower - for token in [ - "approve mappings", - "approve mapping", - "подтверди мапп", - "одобри мапп", - ] - ): - pending_mapping_ids = [ - item["mapping_id"] - for item in dataset_context.get("mappings", []) - if item.get("requires_explicit_approval") - and item.get("approval_state") != ApprovalState.APPROVED.value - ] - return { - "domain": "dataset_review", - "operation": "dataset_review_approve_mappings", - "entities": { - "dataset_review_session_id": session_id, - "session_version": session_version, - "mapping_ids": pending_mapping_ids, - }, - "confidence": 0.95, - "risk_level": "guarded", - "requires_confirmation": True, - } - - if any( - token in lower - for token in [ - "generate sql preview", - "generate preview", - "сгенерируй превью", - "собери превью", - ] - ): - return { - "domain": "dataset_review", - "operation": "dataset_review_generate_sql_preview", - "entities": { - "dataset_review_session_id": session_id, - "session_version": session_version, - }, - "confidence": 0.94, - "risk_level": "guarded", - "requires_confirmation": True, - } - - if any( - token in lower - for token in [ - "set field semantics", - "apply field semantics", - "semantic override", - "update semantic field", - "установи семантик", - "обнови семантик", - ] - ): - field = _match_dataset_review_field(dataset_context, message) - if field is None: - return None - candidate_id = None - if any( - token in lower - for token in ["accept candidate", "apply candidate", "прими кандидат"] - ): - candidates = field.get("candidates") or [] - if candidates: - candidate_id = candidates[0].get("candidate_id") - verbose_name = _extract_quoted_segment( - message, "verbose_name|verbose name|label" - ) - description = _extract_quoted_segment(message, "description|desc") - display_format = _extract_quoted_segment( - message, "display_format|display format|format" - ) - lock_field = any( - token in lower for token in [" lock", "lock it", "зафикс", "закреп"] - ) - if not any([candidate_id, verbose_name, description, display_format]): - return None - return { - "domain": "dataset_review", - "operation": "dataset_review_set_field_semantics", - "entities": { - "dataset_review_session_id": session_id, - "session_version": session_version, - "field_id": field.get("field_id") or target_id, - "candidate_id": candidate_id, - "verbose_name": verbose_name, - "description": description, - "display_format": display_format, - "lock_field": lock_field, - }, - "confidence": 0.9, - "risk_level": "guarded", - "requires_confirmation": True, - } - - if any( - token in lower - for token in [ - "filters", - "фильтр", - "mapping", - "маппинг", - "preview", - "превью", - "finding", - "ошиб", - ] - ): - findings_count = len(dataset_context.get("findings", [])) - mappings_count = len(dataset_context.get("mappings", [])) - filters_count = len(dataset_context.get("imported_filters", [])) - preview = dataset_context.get("preview") or {} - preview_status = preview.get("preview_status") or "missing" - masked_filters = dataset_context.get("imported_filters", []) - return { - "domain": "dataset_review", - "operation": "dataset_review_answer_context", - "entities": { - "dataset_review_session_id": session_id, - "summary": ( - f"Session {session_id}: readiness={dataset_context['readiness_state']}, " - f"recommended_action={dataset_context['recommended_action']}, " - f"filters={filters_count}, mappings={mappings_count}, findings={findings_count}, " - f"preview_status={preview_status}, imported_filters={masked_filters}" - ), - }, - "confidence": 0.8, - "risk_level": "safe", - "requires_confirmation": False, - } - return None - - -# [/DEF:_plan_dataset_review_intent:Function] - - -# [DEF:_dispatch_dataset_review_intent:Function] -# @COMPLEXITY: 4 -# @PURPOSE: Route confirmed dataset-review assistant intents through existing backend dataset-review APIs and orchestration boundaries. -# @RELATION: CALLS -> DatasetReviewOrchestrator -# @PRE: context contains valid session data and user intent. -# @POST: Returns a structured response with planned actions and confirmations. -# @SIDE_EFFECT: May update session state and enqueue tasks. -async def _dispatch_dataset_review_intent( - intent: Dict[str, Any], - current_user: User, - config_manager: ConfigManager, - db: Session, -) -> Tuple[str, Optional[str], List[AssistantAction]]: - with belief_scope("_dispatch_dataset_review_intent"): - logger.reason( - "Dispatching assistant dataset-review intent", - extra={"operation": intent.get("operation")}, - ) - entities = intent.get("entities", {}) - session_id = entities.get("dataset_review_session_id") - session_version = entities.get("session_version") - if not session_id or session_version is None: - raise HTTPException( - status_code=422, - detail="Missing dataset_review_session_id/session_version", - ) - - operation = str(intent.get("operation") or "") - repository = DatasetReviewSessionRepository(db) - if operation == "dataset_review_answer_context": - summary = str(entities.get("summary") or "") - logger.reflect( - "Returned assistant-safe dataset review context summary", - extra={"session_id": session_id, "operation": operation}, - ) - return summary, None, [] - - session = repository.load_session_detail(session_id, current_user.id) - if session is None or session.user_id != current_user.id: - logger.explore( - "Assistant dataset-review intent rejected because session was not found", - extra={"session_id": session_id, "user_id": current_user.id}, - ) - raise HTTPException( - status_code=404, detail="Dataset review session not found" - ) - - try: - repository.require_session_version(session, int(session_version)) - except DatasetReviewSessionVersionConflictError as exc: - logger.explore( - "Assistant dataset-review intent rejected due to stale session version", - extra={ - "session_id": exc.session_id, - "expected_version": exc.expected_version, - "actual_version": exc.actual_version, - "operation": operation, - }, - ) - raise _dataset_review_conflict_http_exception(exc) from exc - - logger.reason( - "Dispatching confirmed assistant dataset-review intent", - extra={ - "session_id": session_id, - "session_version": session_version, - "operation": operation, - }, - ) - - if operation == "dataset_review_approve_mappings": - mapping_ids = list(dict.fromkeys(entities.get("mapping_ids") or [])) - if not mapping_ids: - raise HTTPException( - status_code=409, detail="No pending mappings to approve" - ) - updated_count = 0 - for mapping in session.execution_mappings: - if mapping.mapping_id not in mapping_ids: - continue - mapping.approval_state = ApprovalState.APPROVED - mapping.approved_by_user_id = current_user.id - mapping.approved_at = datetime.utcnow() - updated_count += 1 - if updated_count == 0: - raise HTTPException( - status_code=409, detail="No matching mappings available to approve" - ) - session.last_activity_at = datetime.utcnow() - if session.readiness_state == ReadinessState.MAPPING_REVIEW_NEEDED: - session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW - repository.bump_session_version(session) - repository.db.commit() - repository.db.refresh(session) - repository.event_logger.log_for_session( - session, - actor_user_id=current_user.id, - event_type="assistant_mapping_approval", - event_summary="Assistant-approved warning-sensitive mappings persisted", - event_details={ - "mapping_ids": mapping_ids, - "count": updated_count, - "version": int(getattr(session, "version", 0) or 0), - }, - ) - logger.reflect( - "Assistant mapping approval persisted within optimistic-lock boundary", - extra={ - "session_id": session_id, - "updated_count": updated_count, - "version": int(getattr(session, "version", 0) or 0), - }, - ) - return ( - f"Approved {updated_count} mapping(s) for dataset review session {session_id}.", - None, - [ - AssistantAction( - type="focus_target", - label="Open mapping review", - target="mapping", - ) - ], - ) - - if operation == "dataset_review_set_field_semantics": - field_id = str(entities.get("field_id") or "").strip() - if not field_id: - raise HTTPException(status_code=422, detail="Missing field_id") - field = next( - (item for item in session.semantic_fields if item.field_id == field_id), - None, - ) - if field is None: - raise HTTPException(status_code=404, detail="Semantic field not found") - update_request = FieldSemanticUpdateRequest( - candidate_id=entities.get("candidate_id"), - verbose_name=entities.get("verbose_name"), - description=entities.get("description"), - display_format=entities.get("display_format"), - lock_field=bool(entities.get("lock_field", False)), - ) - try: - _update_semantic_field_state( - field, update_request, changed_by="assistant" - ) - except HTTPException: - raise - except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - session.last_activity_at = datetime.utcnow() - repository.bump_session_version(session) - repository.db.commit() - repository.db.refresh(session) - repository.db.refresh(field) - repository.event_logger.log_for_session( - session, - actor_user_id=current_user.id, - event_type="assistant_field_semantics_updated", - event_summary="Assistant semantic field update persisted", - event_details={ - "field_id": field.field_id, - "candidate_id": entities.get("candidate_id"), - "lock_field": bool(entities.get("lock_field", False)), - "version": int(getattr(session, "version", 0) or 0), - }, - ) - logger.reflect( - "Assistant semantic field update committed safely", - extra={ - "session_id": session_id, - "field_id": field_id, - "version": int(getattr(session, "version", 0) or 0), - }, - ) - return ( - f"Updated semantic field {field.field_name} for dataset review session {session_id}.", - None, - [ - AssistantAction( - type="focus_target", - label="Open semantic review", - target=f"field:{field.field_id}", - ) - ], - ) - - if operation == "dataset_review_generate_sql_preview": - orchestrator = DatasetReviewOrchestrator( - repository=repository, - config_manager=config_manager, - ) - result = orchestrator.prepare_launch_preview( - PreparePreviewCommand( - user=current_user, - session_id=session_id, - expected_version=int(session_version), - ) - ) - preview_status = getattr( - result.preview.preview_status, "value", result.preview.preview_status - ) - logger.reflect( - "Assistant-triggered Superset preview generation completed", - extra={ - "session_id": session_id, - "preview_status": preview_status, - }, - ) - return ( - f"SQL preview {preview_status} for dataset review session {session_id}.", - None, - [ - AssistantAction( - type="focus_target", - label="Open SQL preview", - target="sql-preview", - ) - ], - ) - - raise HTTPException( - status_code=400, detail="Unsupported dataset review operation" - ) - - -# [/DEF:_dispatch_dataset_review_intent:Function] - - -# [DEF:_confirmation_summary:Function] -# @COMPLEXITY: 4 -# @PURPOSE: Build human-readable confirmation prompt for an intent before execution. -# @PRE: actions is a non-empty list of planned review actions. -# @POST: Returns a formatted summary string suitable for display to the user. -# @RELATION: CALLS -> DatasetReviewOrchestrator -# @SIDE_EFFECT: None - pure formatting function. -async def _async_confirmation_summary(intent: Dict[str, Any], config_manager: ConfigManager, db: Session) -> str: - with belief_scope('_confirmation_summary'): - logger.reason('Belief protocol reasoning checkpoint for _confirmation_summary') - operation = intent.get('operation', '') - entities = intent.get('entities', {}) - descriptions: Dict[str, str] = {'create_branch': 'создание ветки{branch} для дашборда{dashboard}', 'commit_changes': 'коммит изменений для дашборда{dashboard}', 'deploy_dashboard': 'деплой дашборда{dashboard} в окружение{env}', 'execute_migration': 'миграция дашборда{dashboard} с{src} на{tgt}', 'run_backup': 'бэкап окружения{env}{dashboard}', 'run_llm_validation': 'LLM-валидация дашборда{dashboard}{env}', 'run_llm_documentation': 'генерация документации для датасета{dataset}{env}'} - template = descriptions.get(operation) - if not template: - logger.reflect('Belief protocol postcondition checkpoint for _confirmation_summary') - return 'Подтвердите выполнение операции или отмените.' - - def _label(value: Any, prefix: str=' ') -> str: - logger.reflect('Belief protocol postcondition checkpoint for _confirmation_summary') - return f'{prefix}{value}' if value else '' - dashboard = entities.get('dashboard_id') or entities.get('dashboard_ref') - text = template.format(branch=_label(entities.get('branch_name')), dashboard=_label(dashboard), env=_label(entities.get('environment') or entities.get('target_env')), src=_label(entities.get('source_env')), tgt=_label(entities.get('target_env')), dataset=_label(entities.get('dataset_id'))) - if operation == 'execute_migration': - flags = [] - flags.append('маппинг БД: ' + ('ВКЛ' if _coerce_query_bool(entities.get('replace_db_config', False)) else 'ВЫКЛ')) - flags.append('исправление кроссфильтров: ' + ('ВКЛ' if _coerce_query_bool(entities.get('fix_cross_filters', True)) else 'ВЫКЛ')) - dry_run_enabled = _coerce_query_bool(entities.get('dry_run', False)) - flags.append('отчет dry-run: ' + ('ВКЛ' if dry_run_enabled else 'ВЫКЛ')) - text += f" ({', '.join(flags)})" - if dry_run_enabled: - try: - from ...core.migration.dry_run_orchestrator import MigrationDryRunService - from ...models.dashboard import DashboardSelection - from ...core.superset_client import SupersetClient - src_token = entities.get('source_env') - tgt_token = entities.get('target_env') - dashboard_id = _resolve_dashboard_id_entity(entities, config_manager, env_hint=src_token) - if dashboard_id and src_token and tgt_token: - src_env_id = _resolve_env_id(src_token, config_manager) - tgt_env_id = _resolve_env_id(tgt_token, config_manager) - if src_env_id and tgt_env_id: - env_map = {env.id: env for env in config_manager.get_environments()} - source_env = env_map.get(src_env_id) - target_env = env_map.get(tgt_env_id) - if source_env and target_env and (source_env.id != target_env.id): - selection = DashboardSelection(source_env_id=source_env.id, target_env_id=target_env.id, selected_ids=[dashboard_id], replace_db_config=_coerce_query_bool(entities.get('replace_db_config', False)), fix_cross_filters=_coerce_query_bool(entities.get('fix_cross_filters', True))) - service = MigrationDryRunService() - source_client = SupersetClient(source_env) - target_client = SupersetClient(target_env) - report = service.run(selection, source_client, target_client, db) - s = report.get('summary', {}) - dash_s = s.get('dashboards', {}) - charts_s = s.get('charts', {}) - ds_s = s.get('datasets', {}) - creates = dash_s.get('create', 0) + charts_s.get('create', 0) + ds_s.get('create', 0) - updates = dash_s.get('update', 0) + charts_s.get('update', 0) + ds_s.get('update', 0) - deletes = dash_s.get('delete', 0) + charts_s.get('delete', 0) + ds_s.get('delete', 0) - text += f'\n\nОтчет dry-run:\n- Будет создано новых объектов: {creates}\n- Будет обновлено: {updates}\n- Будет удалено: {deletes}' - else: - text += '\n\n(Не удалось загрузить отчет dry-run: неверные окружения).' - except Exception as e: - import traceback - logger.warning('[assistant.dry_run_summary][failed] Exception: %s\n%s', e, traceback.format_exc()) - text += f'\n\n(Не удалось загрузить отчет dry-run: {e}).' - logger.reflect('Belief protocol postcondition checkpoint for _confirmation_summary') - return f'Выполнить: {text}. Подтвердите или отмените.' -# [/DEF:_confirmation_summary:Function] - - -# [DEF:_clarification_text_for_intent:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Convert technical missing-parameter errors into user-facing clarification prompts. -# @PRE: state was classified as needs_clarification for current intent/error combination. -# @POST: Returned text is human-readable and actionable for target operation. -def _clarification_text_for_intent( - intent: Optional[Dict[str, Any]], detail_text: str -) -> str: - operation = (intent or {}).get("operation") - guidance_by_operation: Dict[str, str] = { - "run_llm_validation": ( - "Нужно уточнение для запуска LLM-валидации: Укажите дашборд (id или slug), окружение и провайдер LLM." - ), - "run_llm_documentation": ( - "Нужно уточнение для генерации документации: Укажите dataset_id, окружение и провайдер LLM." - ), - "create_branch": "Нужно уточнение: укажите дашборд (id/slug/title) и имя ветки.", - "commit_changes": "Нужно уточнение: укажите дашборд (id/slug/title) для коммита.", - "deploy_dashboard": "Нужно уточнение: укажите дашборд (id/slug/title) и целевое окружение.", - "execute_migration": "Нужно уточнение: укажите дашборд (id/slug/title), source_env и target_env.", - "run_backup": "Нужно уточнение: укажите окружение и при необходимости дашборд (id/slug/title).", - } - return guidance_by_operation.get(operation, detail_text) - - -# [/DEF:_clarification_text_for_intent:Function] - - -# [DEF:_plan_intent_with_llm:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Use active LLM provider to select best tool/operation from dynamic catalog. -# @PRE: tools list contains allowed operations for current user. -# @POST: Returns normalized intent dict when planning succeeds; otherwise None. -async def _plan_intent_with_llm( - message: str, - tools: List[Dict[str, Any]], - db: Session, - config_manager: ConfigManager, -) -> Optional[Dict[str, Any]]: - if not tools: - return None - - llm_settings = normalize_llm_settings(config_manager.get_config().settings.llm) - planner_provider_token = llm_settings.get("assistant_planner_provider") - planner_model_override = llm_settings.get("assistant_planner_model") - llm_service = LLMProviderService(db) - providers = llm_service.get_all_providers() - provider_id = _resolve_provider_id(planner_provider_token, db) - provider = next((p for p in providers if p.id == provider_id), None) - if not provider: - return None - api_key = llm_service.get_decrypted_api_key(provider.id) - if not api_key: - return None - - planner = LLMClient( - provider_type=LLMProviderType(provider.provider_type), - api_key=api_key, - base_url=provider.base_url, - default_model=planner_model_override or provider.default_model, - ) - - system_instruction = ( - "You are a deterministic intent planner for backend tools.\n" - "Choose exactly one operation from available_tools or return clarify.\n" - "Output strict JSON object:\n" - "{" - '"domain": string, ' - '"operation": string, ' - '"entities": object, ' - '"confidence": number, ' - '"risk_level": "safe"|"guarded"|"dangerous", ' - '"requires_confirmation": boolean' - "}\n" - "Rules:\n" - "- Use only operation names from available_tools.\n" - '- If input is ambiguous, operation must be "clarify" with low confidence.\n' - "- If dashboard is provided as name/slug (e.g., COVID), put it into entities.dashboard_ref.\n" - "- Keep entities minimal and factual.\n" - ) - payload = { - "available_tools": tools, - "user_message": message, - "known_environments": [ - {"id": e.id, "name": e.name} for e in config_manager.get_environments() - ], - } - try: - response = await planner.get_json_completion( - [ - {"role": "system", "content": system_instruction}, - {"role": "user", "content": json.dumps(payload, ensure_ascii=False)}, - ] - ) - except Exception as exc: - import traceback - - logger.warning( - f"[assistant.planner][fallback] LLM planner unavailable: {exc}\n{traceback.format_exc()}" - ) - return None - if not isinstance(response, dict): - return None - - operation = response.get("operation") - valid_ops = {tool["operation"] for tool in tools} - if operation == "clarify": - return { - "domain": "unknown", - "operation": "clarify", - "entities": {}, - "confidence": float(response.get("confidence", 0.3)), - "risk_level": "safe", - "requires_confirmation": False, - } - if operation not in valid_ops: - return None - - by_operation = {tool["operation"]: tool for tool in tools} - selected = by_operation[operation] - intent = { - "domain": response.get("domain") or selected["domain"], - "operation": operation, - "entities": response.get("entities", {}), - "confidence": float(response.get("confidence", 0.75)), - "risk_level": response.get("risk_level") or selected["risk_level"], - "requires_confirmation": bool( - response.get("requires_confirmation", selected["requires_confirmation"]) - ), - } - intent = _coerce_intent_entities(intent) - - defaults = selected.get("defaults") or {} - for key, value in defaults.items(): - if value and not intent["entities"].get(key): - intent["entities"][key] = value - - if operation in {"deploy_dashboard", "execute_migration"}: - env_token = intent["entities"].get("environment") or intent["entities"].get( - "target_env" - ) - if _is_production_env(env_token, config_manager): - intent["risk_level"] = "dangerous" - intent["requires_confirmation"] = True - return intent - - -# [/DEF:_plan_intent_with_llm:Function] - - -# [DEF:_authorize_intent:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Validate user permissions for parsed intent before confirmation/dispatch. -# @PRE: intent.operation is present for known assistant command domains. -# @POST: Returns if authorized; raises HTTPException(403) when denied. -def _authorize_intent(intent: Dict[str, Any], current_user: User): - operation = intent.get("operation") - if operation in INTENT_PERMISSION_CHECKS: - _check_any_permission(current_user, INTENT_PERMISSION_CHECKS[operation]) - - -# [/DEF:_authorize_intent:Function] - - -# [DEF:_dispatch_intent:Function] -# @COMPLEXITY: 5 -# @PURPOSE: Execute parsed assistant intent via existing task/plugin/git services. -# @DATA_CONTRACT: Input[intent,current_user,task_manager,config_manager,db] -> Output[Tuple[text:str,task_id:Optional[str],actions:List[AssistantAction]]] -# @RELATION: DEPENDS_ON -> [_check_any_permission] -# @RELATION: DEPENDS_ON -> [_resolve_dashboard_id_entity] -# @RELATION: DEPENDS_ON -> [TaskManager] -# @RELATION: DEPENDS_ON -> [GitService] -# @SIDE_EFFECT: May enqueue tasks, invoke git operations, and query/update external service state. -# @PRE: intent operation is known and actor permissions are validated per operation. -# @POST: Returns response text, optional task id, and UI actions for follow-up. -# @INVARIANT: unsupported operations are rejected via HTTPException(400). -async def _dispatch_intent(intent: Dict[str, Any], current_user: User, task_manager: TaskManager, config_manager: ConfigManager, db: Session) -> Tuple[str, Optional[str], List[AssistantAction]]: - with belief_scope('_dispatch_intent'): - logger.reason('Belief protocol reasoning checkpoint for _dispatch_intent') - operation = intent.get('operation') - entities = intent.get('entities', {}) - if operation in _DATASET_REVIEW_OPS or operation == 'dataset_review_answer_context': - logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') - return await _dispatch_dataset_review_intent(intent, current_user, config_manager, db) - if operation == 'show_capabilities': - tools_catalog = _build_tool_catalog(current_user, config_manager, db) - labels = {'create_branch': 'Git: создание ветки', 'commit_changes': 'Git: коммит', 'deploy_dashboard': 'Git: деплой дашборда', 'execute_migration': 'Миграции: запуск переноса', 'run_backup': 'Бэкапы: запуск резервного копирования', 'run_llm_validation': 'LLM: валидация дашборда', 'run_llm_documentation': 'LLM: генерация документации', 'get_task_status': 'Статус: проверка задачи', 'get_health_summary': 'Здоровье: сводка по дашбордам'} - available = [labels[t['operation']] for t in tools_catalog if t['operation'] in labels] - if not available: - logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') - return ('Сейчас нет доступных для вас операций ассистента.', None, []) - commands = '\n'.join((f'- {item}' for item in available)) - text = f'Вот что я могу сделать для вас:\n{commands}\n\nПример: `запусти миграцию с dev на prod для дашборда 42`.' - logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') - return (text, None, []) - if operation == 'get_health_summary': - from ...services.health_service import HealthService - env_token = entities.get('environment') - env_id = _resolve_env_id(env_token, config_manager) - service = HealthService(db) - summary = await service.get_health_summary(environment_id=env_id) - env_name = _get_environment_name_by_id(env_id, config_manager) if env_id else 'всех окружений' - text = f'Сводка здоровья дашбордов для {env_name}:\n- ✅ Прошли проверку: {summary.pass_count}\n- ⚠️ С предупреждениями: {summary.warn_count}\n- ❌ Ошибки валидации: {summary.fail_count}\n- ❓ Неизвестно: {summary.unknown_count}' - actions = [AssistantAction(type='open_route', label='Открыть Health Center', target='/dashboards/health')] - if summary.fail_count > 0: - text += '\n\nОбнаружены ошибки в следующих дашбордах:' - for item in summary.items: - if item.status == 'FAIL': - text += f"\n- {item.dashboard_id} ({item.environment_id}): {item.summary or 'Нет деталей'}" - actions.append(AssistantAction(type='open_route', label=f'Отчет {item.dashboard_id}', target=f'/reports/llm/{item.task_id}')) - logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') - return (text, None, actions[:5]) - if operation == 'get_task_status': - _check_any_permission(current_user, [('tasks', 'READ')]) - task_id = entities.get('task_id') - if not task_id: - recent = [t for t in task_manager.get_tasks(limit=20, offset=0) if t.user_id == current_user.id] - if not recent: - logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') - return ('У вас пока нет задач в истории.', None, []) - task = recent[0] - actions = [AssistantAction(type='open_task', label='Open Task', target=task.id)] - if str(task.status).upper() in {'SUCCESS', 'FAILED'}: - actions.extend(_extract_result_deep_links(task, config_manager)) - summary_line = _build_task_observability_summary(task, config_manager) - logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') - return (f'Последняя задача: {task.id}, статус: {task.status}.' + (f'\n{summary_line}' if summary_line else ''), task.id, actions) - task = task_manager.get_task(task_id) - if not task: - raise HTTPException(status_code=404, detail=f'Task {task_id} not found') - actions = [AssistantAction(type='open_task', label='Open Task', target=task.id)] - if str(task.status).upper() in {'SUCCESS', 'FAILED'}: - actions.extend(_extract_result_deep_links(task, config_manager)) - summary_line = _build_task_observability_summary(task, config_manager) - logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') - return (f'Статус задачи {task.id}: {task.status}.' + (f'\n{summary_line}' if summary_line else ''), task.id, actions) - if operation == 'create_branch': - _check_any_permission(current_user, [('plugin:git', 'EXECUTE')]) - dashboard_id = _resolve_dashboard_id_entity(entities, config_manager) - branch_name = entities.get('branch_name') - if not dashboard_id or not branch_name: - raise HTTPException(status_code=422, detail='Missing dashboard_id/dashboard_ref or branch_name') - git_service.create_branch(dashboard_id, branch_name, 'main') - logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') - return (f'Ветка `{branch_name}` создана для дашборда {dashboard_id}.', None, []) - if operation == 'commit_changes': - _check_any_permission(current_user, [('plugin:git', 'EXECUTE')]) - dashboard_id = _resolve_dashboard_id_entity(entities, config_manager) - commit_message = entities.get('message') - if not dashboard_id: - raise HTTPException(status_code=422, detail='Missing dashboard_id/dashboard_ref') - git_service.commit_changes(dashboard_id, commit_message, None) - logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') - return ('Коммит выполнен успешно.', None, []) - if operation == 'deploy_dashboard': - _check_any_permission(current_user, [('plugin:git', 'EXECUTE')]) - env_token = entities.get('environment') - env_id = _resolve_env_id(env_token, config_manager) - dashboard_id = _resolve_dashboard_id_entity(entities, config_manager, env_hint=env_token) - if not dashboard_id or not env_id: - raise HTTPException(status_code=422, detail='Missing dashboard_id/dashboard_ref or environment') - task = await task_manager.create_task(plugin_id='git-integration', params={'operation': 'deploy', 'dashboard_id': dashboard_id, 'environment_id': env_id}, user_id=current_user.id) - logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') - return (f'Деплой запущен. task_id={task.id}', task.id, [AssistantAction(type='open_task', label='Open Task', target=task.id), AssistantAction(type='open_reports', label='Open Reports', target='/reports')]) - if operation == 'execute_migration': - _check_any_permission(current_user, [('plugin:migration', 'EXECUTE'), ('plugin:superset-migration', 'EXECUTE')]) - src_token = entities.get('source_env') - dashboard_ref = entities.get('dashboard_ref') - dashboard_id = _resolve_dashboard_id_entity(entities, config_manager, env_hint=src_token) - src = _resolve_env_id(src_token, config_manager) - tgt = _resolve_env_id(entities.get('target_env'), config_manager) - if not src or not tgt: - raise HTTPException(status_code=422, detail='Missing source_env/target_env') - if not dashboard_id and (not dashboard_ref): - raise HTTPException(status_code=422, detail='Missing dashboard_id/dashboard_ref') - migration_params: Dict[str, Any] = {'source_env_id': src, 'target_env_id': tgt, 'replace_db_config': _coerce_query_bool(entities.get('replace_db_config', False)), 'fix_cross_filters': _coerce_query_bool(entities.get('fix_cross_filters', True))} - if dashboard_id: - migration_params['selected_ids'] = [dashboard_id] - else: - migration_params['dashboard_regex'] = str(dashboard_ref) - task = await task_manager.create_task(plugin_id='superset-migration', params=migration_params, user_id=current_user.id) - logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') - return (f'Миграция запущена. task_id={task.id}', task.id, [AssistantAction(type='open_task', label='Open Task', target=task.id), AssistantAction(type='open_reports', label='Open Reports', target='/reports'), *([AssistantAction(type='open_route', label=f'Открыть дашборд в {_get_environment_name_by_id(tgt, config_manager)}', target=f'/dashboards/{dashboard_id}?env_id={tgt}'), AssistantAction(type='open_diff', label='Показать Diff', target=str(dashboard_id))] if dashboard_id else [])]) - if operation == 'run_backup': - _check_any_permission(current_user, [('plugin:superset-backup', 'EXECUTE'), ('plugin:backup', 'EXECUTE')]) - env_token = entities.get('environment') - env_id = _resolve_env_id(env_token, config_manager) - if not env_id: - raise HTTPException(status_code=400, detail='Missing or unknown environment') - params: Dict[str, Any] = {'environment_id': env_id} - if entities.get('dashboard_id') or entities.get('dashboard_ref'): - dashboard_id = _resolve_dashboard_id_entity(entities, config_manager, env_hint=env_token) - if not dashboard_id: - raise HTTPException(status_code=422, detail='Missing dashboard_id/dashboard_ref') - params['dashboard_ids'] = [dashboard_id] - task = await task_manager.create_task(plugin_id='superset-backup', params=params, user_id=current_user.id) - logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') - return (f'Бэкап запущен. task_id={task.id}', task.id, [AssistantAction(type='open_task', label='Open Task', target=task.id), AssistantAction(type='open_reports', label='Open Reports', target='/reports'), *([AssistantAction(type='open_route', label=f'Открыть дашборд в {_get_environment_name_by_id(env_id, config_manager)}', target=f'/dashboards/{dashboard_id}?env_id={env_id}'), AssistantAction(type='open_diff', label='Показать Diff', target=str(dashboard_id))] if entities.get('dashboard_id') or entities.get('dashboard_ref') else [])]) - if operation == 'run_llm_validation': - _check_any_permission(current_user, [('plugin:llm_dashboard_validation', 'EXECUTE')]) - env_token = entities.get('environment') - env_id = _resolve_env_id(env_token, config_manager) or _get_default_environment_id(config_manager) - dashboard_id = _resolve_dashboard_id_entity(entities, config_manager, env_hint=env_token) - provider_id = _resolve_provider_id(entities.get('provider'), db, config_manager=config_manager, task_key='dashboard_validation') - if not dashboard_id or not env_id or (not provider_id): - raise HTTPException(status_code=422, detail='Missing dashboard_id/environment/provider. Укажите ID/slug дашборда или окружение.') - provider = LLMProviderService(db).get_provider(provider_id) - provider_model = provider.default_model if provider else '' - if not is_multimodal_model(provider_model, provider.provider_type if provider else None): - raise HTTPException(status_code=422, detail='Selected provider model is not multimodal for dashboard validation. Выберите мультимодальную модель (например, gpt-4o).') - task = await task_manager.create_task(plugin_id='llm_dashboard_validation', params={'dashboard_id': str(dashboard_id), 'environment_id': env_id, 'provider_id': provider_id}, user_id=current_user.id) - logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') - return (f'LLM-валидация запущена. task_id={task.id}', task.id, [AssistantAction(type='open_task', label='Open Task', target=task.id), AssistantAction(type='open_reports', label='Open Reports', target='/reports')]) - if operation == 'run_llm_documentation': - _check_any_permission(current_user, [('plugin:llm_documentation', 'EXECUTE')]) - dataset_id = entities.get('dataset_id') - env_id = _resolve_env_id(entities.get('environment'), config_manager) - provider_id = _resolve_provider_id(entities.get('provider'), db, config_manager=config_manager, task_key='documentation') - if not dataset_id or not env_id or (not provider_id): - raise HTTPException(status_code=400, detail='Missing dataset_id/environment/provider') - task = await task_manager.create_task(plugin_id='llm_documentation', params={'dataset_id': str(dataset_id), 'environment_id': env_id, 'provider_id': provider_id}, user_id=current_user.id) - logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') - return (f'Генерация документации запущена. task_id={task.id}', task.id, [AssistantAction(type='open_task', label='Open Task', target=task.id), AssistantAction(type='open_reports', label='Open Reports', target='/reports')]) - raise HTTPException(status_code=400, detail='Unsupported operation') -# [/DEF:_dispatch_intent:Function] - - -@router.post("/messages", response_model=AssistantMessageResponse) -# [DEF:send_message:Function] -# @COMPLEXITY: 5 -# @PURPOSE: Parse assistant command, enforce safety gates, and dispatch executable intent. -# @DATA_CONTRACT: Input[AssistantMessageRequest,User,TaskManager,ConfigManager,Session] -> Output[AssistantMessageResponse] -# @RELATION: DEPENDS_ON -> [_plan_intent_with_llm] -# @RELATION: DEPENDS_ON -> [_parse_command] -# @RELATION: DEPENDS_ON -> [_dispatch_intent] -# @RELATION: DEPENDS_ON -> [_append_history] -# @RELATION: DEPENDS_ON -> [_persist_message] -# @RELATION: DEPENDS_ON -> [_audit] -# @SIDE_EFFECT: Persists chat/audit state, mutates in-memory conversation and confirmation stores, and may create confirmation records. -# @PRE: Authenticated user is available and message text is non-empty. -# @POST: Response state is one of clarification/confirmation/started/success/denied/failed. -# @RETURN: AssistantMessageResponse with operation feedback and optional actions. -# @INVARIANT: non-safe operations are gated with confirmation before execution from this endpoint. -async def send_message(request: AssistantMessageRequest, current_user: User=Depends(get_current_user), task_manager: TaskManager=Depends(get_task_manager), config_manager: ConfigManager=Depends(get_config_manager), db: Session=Depends(get_db)): - with belief_scope('send_message'): - logger.reason('Belief protocol reasoning checkpoint for send_message') - user_id = current_user.id - dataset_review_context = _load_dataset_review_context(request.dataset_review_session_id, current_user, db) - conversation_id = _resolve_or_create_conversation(user_id, request.conversation_id, db) - _append_history(user_id, conversation_id, 'user', request.message) - _persist_message(db, user_id, conversation_id, 'user', request.message) - tools_catalog = _build_tool_catalog(current_user, config_manager, db) - intent = None - try: - intent = await _plan_intent_with_llm(request.message, tools_catalog, db, config_manager) - except Exception as exc: - logger.warning(f'[assistant.planner][fallback] Planner error: {exc}') - if not intent: - intent = _parse_command(request.message, config_manager) - if dataset_review_context: - dataset_review_intent = _plan_dataset_review_intent(request.message, dataset_review_context) - if dataset_review_intent is not None: - intent = dataset_review_intent - confidence = float(intent.get('confidence', 0.0)) - if intent.get('domain') == 'unknown' or confidence < 0.6: - text = 'Команда неоднозначна. Уточните действие: git / migration / backup / llm / status.' - _append_history(user_id, conversation_id, 'assistant', text, state='needs_clarification') - _persist_message(db, user_id, conversation_id, 'assistant', text, state='needs_clarification', metadata={'intent': intent}) - audit_payload = {'decision': 'needs_clarification', 'message': request.message, 'intent': intent, 'dataset_review_session_id': request.dataset_review_session_id} - _audit(user_id, audit_payload) - _persist_audit(db, user_id, audit_payload, conversation_id) - logger.reflect('Belief protocol postcondition checkpoint for send_message') - return AssistantMessageResponse(conversation_id=conversation_id, response_id=str(uuid.uuid4()), state='needs_clarification', text=text, intent=intent, actions=[AssistantAction(type='rephrase', label='Rephrase command')], created_at=datetime.utcnow()) - try: - _authorize_intent(intent, current_user) - operation = intent.get('operation') - if operation not in _SAFE_OPS: - confirmation_id = str(uuid.uuid4()) - confirm = ConfirmationRecord(id=confirmation_id, user_id=user_id, conversation_id=conversation_id, intent=intent, dispatch={'intent': intent}, expires_at=datetime.utcnow() + timedelta(minutes=5), created_at=datetime.utcnow()) - CONFIRMATIONS[confirmation_id] = confirm - _persist_confirmation(db, confirm) - text = await _async_confirmation_summary(intent, config_manager, db) - _append_history(user_id, conversation_id, 'assistant', text, state='needs_confirmation', confirmation_id=confirmation_id) - _persist_message(db, user_id, conversation_id, 'assistant', text, state='needs_confirmation', confirmation_id=confirmation_id, metadata={'intent': intent, 'dataset_review_context': dataset_review_context, 'actions': [{'type': 'confirm', 'label': '✅ Подтвердить', 'target': confirmation_id}, {'type': 'cancel', 'label': '❌ Отменить', 'target': confirmation_id}]}) - audit_payload = {'decision': 'needs_confirmation', 'message': request.message, 'intent': intent, 'confirmation_id': confirmation_id, 'dataset_review_session_id': request.dataset_review_session_id} - _audit(user_id, audit_payload) - _persist_audit(db, user_id, audit_payload, conversation_id) - logger.reflect('Belief protocol postcondition checkpoint for send_message') - return AssistantMessageResponse(conversation_id=conversation_id, response_id=str(uuid.uuid4()), state='needs_confirmation', text=text, intent=intent, confirmation_id=confirmation_id, actions=[AssistantAction(type='confirm', label='✅ Подтвердить', target=confirmation_id), AssistantAction(type='cancel', label='❌ Отменить', target=confirmation_id)], created_at=datetime.utcnow()) - text, task_id, actions = await _dispatch_intent(intent, current_user, task_manager, config_manager, db) - state = 'started' if task_id else 'success' - _append_history(user_id, conversation_id, 'assistant', text, state=state, task_id=task_id) - _persist_message(db, user_id, conversation_id, 'assistant', text, state=state, task_id=task_id, metadata={'intent': intent, 'dataset_review_context': dataset_review_context, 'actions': [a.model_dump() for a in actions]}) - audit_payload = {'decision': 'executed', 'message': request.message, 'intent': intent, 'task_id': task_id, 'dataset_review_session_id': request.dataset_review_session_id} - _audit(user_id, audit_payload) - _persist_audit(db, user_id, audit_payload, conversation_id) - logger.reflect('Belief protocol postcondition checkpoint for send_message') - return AssistantMessageResponse(conversation_id=conversation_id, response_id=str(uuid.uuid4()), state=state, text=text, intent=intent, task_id=task_id, actions=actions, created_at=datetime.utcnow()) - except HTTPException as exc: - detail_text = str(exc.detail) - is_clarification_error = exc.status_code in (400, 422) and (detail_text.lower().startswith('missing') or 'укажите' in detail_text.lower() or 'выберите' in detail_text.lower()) - if exc.status_code == status.HTTP_403_FORBIDDEN: - state = 'denied' - elif is_clarification_error: - state = 'needs_clarification' - else: - state = 'failed' - text = _clarification_text_for_intent(intent, detail_text) if state == 'needs_clarification' else detail_text - _append_history(user_id, conversation_id, 'assistant', text, state=state) - _persist_message(db, user_id, conversation_id, 'assistant', text, state=state, metadata={'intent': intent}) - audit_payload = {'decision': state, 'message': request.message, 'intent': intent, 'error': text, 'dataset_review_session_id': request.dataset_review_session_id} - _audit(user_id, audit_payload) - _persist_audit(db, user_id, audit_payload, conversation_id) - logger.reflect('Belief protocol postcondition checkpoint for send_message') - return AssistantMessageResponse(conversation_id=conversation_id, response_id=str(uuid.uuid4()), state=state, text=text, intent=intent, actions=[AssistantAction(type='rephrase', label='Rephrase command')] if state == 'needs_clarification' else [], created_at=datetime.utcnow()) -# [/DEF:send_message:Function] - - -@router.post( - "/confirmations/{confirmation_id}/confirm", response_model=AssistantMessageResponse -) -# [DEF:confirm_operation:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Execute previously requested risky operation after explicit user confirmation. -# @PRE: confirmation_id exists, belongs to current user, is pending, and not expired. -# @POST: Confirmation state becomes consumed and operation result is persisted in history. -# @RETURN: AssistantMessageResponse with task details when async execution starts. -async def confirm_operation( - confirmation_id: str, - current_user: User = Depends(get_current_user), - task_manager: TaskManager = Depends(get_task_manager), - config_manager: ConfigManager = Depends(get_config_manager), - db: Session = Depends(get_db), -): - with belief_scope("assistant.confirm"): - record = CONFIRMATIONS.get(confirmation_id) - if not record: - record = _load_confirmation_from_db(db, confirmation_id) - if record: - CONFIRMATIONS[confirmation_id] = record - else: - raise HTTPException(status_code=404, detail="Confirmation not found") - - if record.user_id != current_user.id: - raise HTTPException( - status_code=403, detail="Confirmation does not belong to current user" - ) - - if record.state != "pending": - raise HTTPException( - status_code=400, detail=f"Confirmation already {record.state}" - ) - - if datetime.utcnow() > record.expires_at: - record.state = "expired" - _update_confirmation_state(db, confirmation_id, "expired") - raise HTTPException(status_code=400, detail="Confirmation expired") - - intent = record.intent - text, task_id, actions = await _dispatch_intent( - intent, current_user, task_manager, config_manager, db - ) - record.state = "consumed" - _update_confirmation_state(db, confirmation_id, "consumed") - - _append_history( - current_user.id, - record.conversation_id, - "assistant", - text, - state="started" if task_id else "success", - task_id=task_id, - ) - _persist_message( - db, - current_user.id, - record.conversation_id, - "assistant", - text, - state="started" if task_id else "success", - task_id=task_id, - metadata={"intent": intent, "confirmation_id": confirmation_id}, - ) - audit_payload = { - "decision": "confirmed_execute", - "confirmation_id": confirmation_id, - "task_id": task_id, - "intent": intent, - } - _audit(current_user.id, audit_payload) - _persist_audit(db, current_user.id, audit_payload, record.conversation_id) - - return AssistantMessageResponse( - conversation_id=record.conversation_id, - response_id=str(uuid.uuid4()), - state="started" if task_id else "success", - text=text, - intent=intent, - task_id=task_id, - actions=actions, - created_at=datetime.utcnow(), - ) - - -# [/DEF:confirm_operation:Function] - - -@router.post( - "/confirmations/{confirmation_id}/cancel", response_model=AssistantMessageResponse -) -# [DEF:cancel_operation:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Cancel pending risky operation and mark confirmation token as cancelled. -# @PRE: confirmation_id exists, belongs to current user, and is still pending. -# @POST: Confirmation becomes cancelled and cannot be executed anymore. -# @RETURN: AssistantMessageResponse confirming cancellation. -async def cancel_operation( - confirmation_id: str, - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - with belief_scope("assistant.cancel"): - record = CONFIRMATIONS.get(confirmation_id) - if not record: - record = _load_confirmation_from_db(db, confirmation_id) - if record: - CONFIRMATIONS[confirmation_id] = record - else: - raise HTTPException(status_code=404, detail="Confirmation not found") - - if record.user_id != current_user.id: - raise HTTPException( - status_code=403, detail="Confirmation does not belong to current user" - ) - - if record.state != "pending": - raise HTTPException( - status_code=400, detail=f"Confirmation already {record.state}" - ) - - record.state = "cancelled" - _update_confirmation_state(db, confirmation_id, "cancelled") - text = "Операция отменена. Выполнение не запускалось." - _append_history( - current_user.id, - record.conversation_id, - "assistant", - text, - state="success", - confirmation_id=confirmation_id, - ) - _persist_message( - db, - current_user.id, - record.conversation_id, - "assistant", - text, - state="success", - confirmation_id=confirmation_id, - metadata={"intent": record.intent}, - ) - audit_payload = { - "decision": "cancelled", - "confirmation_id": confirmation_id, - "intent": record.intent, - } - _audit(current_user.id, audit_payload) - _persist_audit(db, current_user.id, audit_payload, record.conversation_id) - - return AssistantMessageResponse( - conversation_id=record.conversation_id, - response_id=str(uuid.uuid4()), - state="success", - text=text, - intent=record.intent, - confirmation_id=confirmation_id, - actions=[], - created_at=datetime.utcnow(), - ) - - -# [/DEF:cancel_operation:Function] - - -# [DEF:list_conversations:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Return paginated conversation list for current user with archived flag and last message preview. -# @PRE: Authenticated user context and valid pagination params. -# @POST: Conversations are grouped by conversation_id sorted by latest activity descending. -# @RETURN: Dict with items, paging metadata, and archive segmentation counts. -@router.get("/conversations") -async def list_conversations( - page: int = Query(1, ge=1), - page_size: int = Query(20, ge=1, le=100), - include_archived: bool = Query(False), - archived_only: bool = Query(False), - search: Optional[str] = Query(None), - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - with belief_scope("assistant.conversations"): - user_id = current_user.id - include_archived = _coerce_query_bool(include_archived) - archived_only = _coerce_query_bool(archived_only) - _cleanup_history_ttl(db, user_id) - - rows = ( - db.query(AssistantMessageRecord) - .filter(AssistantMessageRecord.user_id == user_id) - .order_by(desc(AssistantMessageRecord.created_at)) - .all() - ) - - summary: Dict[str, Dict[str, Any]] = {} - for row in rows: - conv_id = row.conversation_id - if not conv_id: - continue - created_at = row.created_at or datetime.utcnow() - if conv_id not in summary: - summary[conv_id] = { - "conversation_id": conv_id, - "title": "", - "updated_at": created_at, - "last_message": row.text, - "last_role": row.role, - "last_state": row.state, - "last_task_id": row.task_id, - "message_count": 0, - } - item = summary[conv_id] - item["message_count"] += 1 - if row.role == "user" and row.text and not item["title"]: - item["title"] = row.text.strip()[:80] - - items = [] - search_term = search.lower().strip() if search else "" - archived_total = sum( - 1 - for c in summary.values() - if _is_conversation_archived(c.get("updated_at")) - ) - active_total = len(summary) - archived_total - for conv in summary.values(): - conv["archived"] = _is_conversation_archived(conv.get("updated_at")) - if not conv.get("title"): - conv["title"] = f"Conversation {conv['conversation_id'][:8]}" - if search_term: - haystack = ( - f"{conv.get('title', '')} {conv.get('last_message', '')}".lower() - ) - if search_term not in haystack: - continue - if archived_only and not conv["archived"]: - continue - if not archived_only and not include_archived and conv["archived"]: - continue - updated = conv.get("updated_at") - conv["updated_at"] = ( - updated.isoformat() if isinstance(updated, datetime) else None - ) - items.append(conv) - - items.sort(key=lambda x: x.get("updated_at") or "", reverse=True) - total = len(items) - start = (page - 1) * page_size - page_items = items[start : start + page_size] - - return { - "items": page_items, - "total": total, - "page": page, - "page_size": page_size, - "has_next": start + page_size < total, - "active_total": active_total, - "archived_total": archived_total, - } - - -# [/DEF:list_conversations:Function] - - -# [DEF:delete_conversation:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Soft-delete or hard-delete a conversation and clear its in-memory trace. -# @PRE: conversation_id belongs to current_user. -# @POST: Conversation records are removed from DB and CONVERSATIONS cache. -@router.delete("/conversations/{conversation_id}") -async def delete_conversation( - conversation_id: str, - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - with belief_scope("assistant.conversations.delete"): - user_id = current_user.id - - # 1. Remove from in-memory cache - key = (user_id, conversation_id) - if key in CONVERSATIONS: - del CONVERSATIONS[key] - - # 2. Delete from database - deleted_count = ( - db.query(AssistantMessageRecord) - .filter( - AssistantMessageRecord.user_id == user_id, - AssistantMessageRecord.conversation_id == conversation_id, - ) - .delete() - ) - - db.commit() - - if deleted_count == 0: - raise HTTPException( - status_code=404, detail="Conversation not found or already deleted" - ) - - return { - "status": "success", - "deleted": deleted_count, - "conversation_id": conversation_id, - } - - -# [/DEF:delete_conversation:Function] - - -@router.get("/history") -# [DEF:get_history:Function] -# @PURPOSE: Retrieve paginated assistant conversation history for current user. -# @PRE: Authenticated user is available and page params are valid. -# @POST: Returns persistent messages and mirrored in-memory snapshot for diagnostics. -# @RETURN: Dict with items, paging metadata, and resolved conversation_id. -async def get_history( - page: int = Query(1, ge=1), - page_size: int = Query(20, ge=1, le=100), - conversation_id: Optional[str] = Query(None), - from_latest: bool = Query(False), - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - with belief_scope("assistant.history"): - user_id = current_user.id - _cleanup_history_ttl(db, user_id) - conv_id = _resolve_or_create_conversation(user_id, conversation_id, db) - - base_query = db.query(AssistantMessageRecord).filter( - AssistantMessageRecord.user_id == user_id, - AssistantMessageRecord.conversation_id == conv_id, - ) - total = base_query.count() - start = (page - 1) * page_size - if from_latest: - rows = ( - base_query.order_by(desc(AssistantMessageRecord.created_at)) - .offset(start) - .limit(page_size) - .all() - ) - rows = list(reversed(rows)) - else: - rows = ( - base_query.order_by(AssistantMessageRecord.created_at.asc()) - .offset(start) - .limit(page_size) - .all() - ) - - persistent_items = [ - { - "message_id": row.id, - "conversation_id": row.conversation_id, - "role": row.role, - "text": row.text, - "state": row.state, - "task_id": row.task_id, - "confirmation_id": row.confirmation_id, - "created_at": row.created_at.isoformat() if row.created_at else None, - "metadata": row.payload, - } - for row in rows - ] - - memory_items = CONVERSATIONS.get((user_id, conv_id), []) - return { - "items": persistent_items, - "memory_items": memory_items, - "total": total, - "page": page, - "page_size": page_size, - "has_next": start + page_size < total, - "from_latest": from_latest, - "conversation_id": conv_id, - } - - -# [/DEF:get_history:Function] - - -@router.get("/audit") -# [DEF:get_assistant_audit:Function] -# @PURPOSE: Return assistant audit decisions for current user from persistent and in-memory stores. -# @PRE: User has tasks:READ permission. -# @POST: Audit payload is returned in reverse chronological order from DB. -# @RETURN: Dict with persistent and memory audit slices. -async def get_assistant_audit( - limit: int = Query(50, ge=1, le=500), - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), - _=Depends(has_permission("tasks", "READ")), -): - with belief_scope("assistant.audit"): - memory_rows = ASSISTANT_AUDIT.get(current_user.id, []) - db_rows = ( - db.query(AssistantAuditRecord) - .filter(AssistantAuditRecord.user_id == current_user.id) - .order_by(AssistantAuditRecord.created_at.desc()) - .limit(limit) - .all() - ) - persistent = [ - { - "id": row.id, - "user_id": row.user_id, - "conversation_id": row.conversation_id, - "decision": row.decision, - "task_id": row.task_id, - "message": row.message, - "payload": row.payload, - "created_at": row.created_at.isoformat() if row.created_at else None, - } - for row in db_rows - ] - return { - "items": persistent, - "memory_items": memory_rows[-limit:], - "total": len(persistent), - "memory_total": len(memory_rows), - } - - -# [/DEF:get_assistant_audit:Function] - -# [/DEF:AssistantApi:Module] diff --git a/backend/src/api/routes/assistant/__init__.py b/backend/src/api/routes/assistant/__init__.py index c6208144..34aa145c 100644 --- a/backend/src/api/routes/assistant/__init__.py +++ b/backend/src/api/routes/assistant/__init__.py @@ -11,7 +11,7 @@ # Re-export public API for backward compatibility. from ._routes import router, send_message, confirm_operation, cancel_operation -from ._routes import list_conversations, delete_conversation, get_history, get_assistant_audit +from ._admin_routes import list_conversations, delete_conversation, get_history, get_assistant_audit from ._schemas import ( AssistantMessageRequest, AssistantMessageResponse, @@ -26,13 +26,14 @@ from ._schemas import ( _DATASET_REVIEW_OPS, ) from ._command_parser import _parse_command -from ._llm_planner import _plan_intent_with_llm, _build_tool_catalog, _authorize_intent +from ._llm_planner import _build_tool_catalog +from ._llm_planner_intent import _plan_intent_with_llm, _authorize_intent from ._dispatch import _dispatch_intent, _async_confirmation_summary, _clarification_text_for_intent from ._dataset_review import ( _load_dataset_review_context, _plan_dataset_review_intent, - _dispatch_dataset_review_intent, ) +from ._dataset_review_dispatch import _dispatch_dataset_review_intent from ._history import ( _append_history, _persist_message, diff --git a/backend/src/api/routes/assistant/_admin_routes.py b/backend/src/api/routes/assistant/_admin_routes.py new file mode 100644 index 00000000..54beb23e --- /dev/null +++ b/backend/src/api/routes/assistant/_admin_routes.py @@ -0,0 +1,305 @@ +# [DEF:AssistantAdminRoutes:Module] +# @COMPLEXITY: 5 +# @PURPOSE: FastAPI route handlers for assistant admin operations — conversation listing, deletion, history, audit. +# @LAYER: API +# @RELATION: DEPENDS_ON -> [AssistantRoutes] +# @RELATION: DEPENDS_ON -> [AssistantSchemas] +# @RELATION: DEPENDS_ON -> [AssistantHistory] +# @INVARIANT: Audit endpoint requires tasks:READ permission. + +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any, Dict, Optional + +from fastapi import Depends, HTTPException, Query +from sqlalchemy.orm import Session +from sqlalchemy import desc + +from src.core.logger import belief_scope, logger +from src.dependencies import get_current_user, has_permission +from src.core.database import get_db +from src.models.assistant import ( + AssistantAuditRecord, + AssistantMessageRecord, +) +from src.schemas.auth import User +from ._schemas import ( + ASSISTANT_AUDIT, + CONVERSATIONS, + AssistantAction, + AssistantMessageResponse, +) +from ._history import ( + _cleanup_history_ttl, + _coerce_query_bool, + _is_conversation_archived, + _persist_audit, + _resolve_or_create_conversation, +) +from ._routes import router + + +# [DEF:list_conversations:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Return paginated conversation list for current user with archived flag and last message preview. +# @PRE: Authenticated user context and valid pagination params. +# @POST: Conversations are grouped by conversation_id sorted by latest activity descending. +# @RETURN: Dict with items, paging metadata, and archive segmentation counts. +@router.get("/conversations") +async def list_conversations( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + include_archived: bool = Query(False), + archived_only: bool = Query(False), + search: Optional[str] = Query(None), + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + with belief_scope("assistant.conversations"): + user_id = current_user.id + include_archived = _coerce_query_bool(include_archived) + archived_only = _coerce_query_bool(archived_only) + _cleanup_history_ttl(db, user_id) + + rows = ( + db.query(AssistantMessageRecord) + .filter(AssistantMessageRecord.user_id == user_id) + .order_by(desc(AssistantMessageRecord.created_at)) + .all() + ) + + summary: Dict[str, Dict[str, Any]] = {} + for row in rows: + conv_id = row.conversation_id + if not conv_id: + continue + created_at = row.created_at or datetime.utcnow() + if conv_id not in summary: + summary[conv_id] = { + "conversation_id": conv_id, + "title": "", + "updated_at": created_at, + "last_message": row.text, + "last_role": row.role, + "last_state": row.state, + "last_task_id": row.task_id, + "message_count": 0, + } + item = summary[conv_id] + item["message_count"] += 1 + if row.role == "user" and row.text and not item["title"]: + item["title"] = row.text.strip()[:80] + + items = [] + search_term = search.lower().strip() if search else "" + archived_total = sum( + 1 + for c in summary.values() + if _is_conversation_archived(c.get("updated_at")) + ) + active_total = len(summary) - archived_total + for conv in summary.values(): + conv["archived"] = _is_conversation_archived(conv.get("updated_at")) + if not conv.get("title"): + conv["title"] = f"Conversation {conv['conversation_id'][:8]}" + if search_term: + haystack = ( + f"{conv.get('title', '')} {conv.get('last_message', '')}".lower() + ) + if search_term not in haystack: + continue + if archived_only and not conv["archived"]: + continue + if not archived_only and not include_archived and conv["archived"]: + continue + updated = conv.get("updated_at") + conv["updated_at"] = ( + updated.isoformat() if isinstance(updated, datetime) else None + ) + items.append(conv) + + items.sort(key=lambda x: x.get("updated_at") or "", reverse=True) + total = len(items) + start = (page - 1) * page_size + page_items = items[start : start + page_size] + + return { + "items": page_items, + "total": total, + "page": page, + "page_size": page_size, + "has_next": start + page_size < total, + "active_total": active_total, + "archived_total": archived_total, + } + + +# [/DEF:list_conversations:Function] + + +# [DEF:delete_conversation:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Soft-delete or hard-delete a conversation and clear its in-memory trace. +# @PRE: conversation_id belongs to current_user. +# @POST: Conversation records are removed from DB and CONVERSATIONS cache. +@router.delete("/conversations/{conversation_id}") +async def delete_conversation( + conversation_id: str, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + with belief_scope("assistant.conversations.delete"): + user_id = current_user.id + + # 1. Remove from in-memory cache + key = (user_id, conversation_id) + if key in CONVERSATIONS: + del CONVERSATIONS[key] + + # 2. Delete from database + deleted_count = ( + db.query(AssistantMessageRecord) + .filter( + AssistantMessageRecord.user_id == user_id, + AssistantMessageRecord.conversation_id == conversation_id, + ) + .delete() + ) + + db.commit() + + if deleted_count == 0: + raise HTTPException( + status_code=404, detail="Conversation not found or already deleted" + ) + + return { + "status": "success", + "deleted": deleted_count, + "conversation_id": conversation_id, + } + + +# [/DEF:delete_conversation:Function] + + +@router.get("/history") +# [DEF:get_history:Function] +# @PURPOSE: Retrieve paginated assistant conversation history for current user. +# @PRE: Authenticated user is available and page params are valid. +# @POST: Returns persistent messages and mirrored in-memory snapshot for diagnostics. +# @RETURN: Dict with items, paging metadata, and resolved conversation_id. +async def get_history( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + conversation_id: Optional[str] = Query(None), + from_latest: bool = Query(False), + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + with belief_scope("assistant.history"): + user_id = current_user.id + _cleanup_history_ttl(db, user_id) + conv_id = _resolve_or_create_conversation(user_id, conversation_id, db) + + base_query = db.query(AssistantMessageRecord).filter( + AssistantMessageRecord.user_id == user_id, + AssistantMessageRecord.conversation_id == conv_id, + ) + total = base_query.count() + start = (page - 1) * page_size + if from_latest: + rows = ( + base_query.order_by(desc(AssistantMessageRecord.created_at)) + .offset(start) + .limit(page_size) + .all() + ) + rows = list(reversed(rows)) + else: + rows = ( + base_query.order_by(AssistantMessageRecord.created_at.asc()) + .offset(start) + .limit(page_size) + .all() + ) + + persistent_items = [ + { + "message_id": row.id, + "conversation_id": row.conversation_id, + "role": row.role, + "text": row.text, + "state": row.state, + "task_id": row.task_id, + "confirmation_id": row.confirmation_id, + "created_at": row.created_at.isoformat() if row.created_at else None, + "metadata": row.payload, + } + for row in rows + ] + + memory_items = CONVERSATIONS.get((user_id, conv_id), []) + return { + "items": persistent_items, + "memory_items": memory_items, + "total": total, + "page": page, + "page_size": page_size, + "has_next": start + page_size < total, + "from_latest": from_latest, + "conversation_id": conv_id, + } + + +# [/DEF:get_history:Function] + + +@router.get("/audit") +# [DEF:get_assistant_audit:Function] +# @PURPOSE: Return assistant audit decisions for current user from persistent and in-memory stores. +# @PRE: User has tasks:READ permission. +# @POST: Audit payload is returned in reverse chronological order from DB. +# @RETURN: Dict with persistent and memory audit slices. +async def get_assistant_audit( + limit: int = Query(50, ge=1, le=500), + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), + _=Depends(has_permission("tasks", "READ")), +): + with belief_scope("assistant.audit"): + memory_rows = ASSISTANT_AUDIT.get(current_user.id, []) + db_rows = ( + db.query(AssistantAuditRecord) + .filter(AssistantAuditRecord.user_id == current_user.id) + .order_by(AssistantAuditRecord.created_at.desc()) + .limit(limit) + .all() + ) + persistent = [ + { + "id": row.id, + "user_id": row.user_id, + "conversation_id": row.conversation_id, + "decision": row.decision, + "task_id": row.task_id, + "message": row.message, + "payload": row.payload, + "created_at": row.created_at.isoformat() if row.created_at else None, + } + for row in db_rows + ] + return { + "items": persistent, + "memory_items": memory_rows[-limit:], + "total": len(persistent), + "memory_total": len(memory_rows), + } + + +# [/DEF:get_assistant_audit:Function] + + +# [/DEF:AssistantAdminRoutes:Module] diff --git a/backend/src/api/routes/assistant/_dataset_review.py b/backend/src/api/routes/assistant/_dataset_review.py index f956d98c..87b103fa 100644 --- a/backend/src/api/routes/assistant/_dataset_review.py +++ b/backend/src/api/routes/assistant/_dataset_review.py @@ -1,9 +1,10 @@ # [DEF:AssistantDatasetReview:Module] # @COMPLEXITY: 4 -# @PURPOSE: Dataset review context loading, intent planning, and dispatch for the assistant API. +# @PURPOSE: Dataset review context loading and intent planning for the assistant API. # @LAYER: API # @RELATION: DEPENDS_ON -> [DatasetReviewOrchestrator] # @RELATION: DEPENDS_ON -> [AssistantSchemas] +# @RELATION: DISPATCHES -> [AssistantDatasetReviewDispatch] # @INVARIANT: Dataset review operations are always scoped to the owner's session. from __future__ import annotations @@ -16,7 +17,6 @@ from fastapi import HTTPException, status from sqlalchemy.orm import Session from src.core.logger import belief_scope, logger -from src.core.config_manager import ConfigManager from src.core.utils.superset_context_extractor import ( sanitize_imported_filter_for_assistant, ) @@ -26,16 +26,11 @@ from src.models.dataset_review import ( ReadinessState, RecommendedAction, ) -from src.services.dataset_review.orchestrator import ( - DatasetReviewOrchestrator, - PreparePreviewCommand, -) from src.services.dataset_review.repositories.session_repository import ( DatasetReviewSessionRepository, DatasetReviewSessionVersionConflictError, ) from src.schemas.auth import User -from src.api.routes.dataset_review import FieldSemanticUpdateRequest, _update_semantic_field_state from ._schemas import ( AssistantAction, ) @@ -56,7 +51,7 @@ def _serialize_dataset_review_context(session: DatasetReviewSession) -> Dict[str if previews: latest_preview = previews[-1] logger.reflect('Belief protocol postcondition checkpoint for _serialize_dataset_review_context') - return {'session_id': session.session_id, 'version': int(getattr(session, 'version', 0) or 0), 'dataset_ref': session.dataset_ref, 'dataset_id': session.dataset_id, 'environment_id': session.environment_id, 'readiness_state': session.readiness_state.value, 'recommended_action': session.recommended_action.value, 'status': session.status.value, 'current_phase': session.current_phase.value, 'findings': [{'finding_id': item.finding_id, 'code': item.code, 'severity': item.severity.value, 'message': item.message, 'resolution_state': item.resolution_state.value} for item in getattr(session, 'findings', [])], 'imported_filters': [sanitize_imported_filter_for_assistant({'filter_id': item.filter_id, 'filter_name': item.filter_name, 'display_name': item.display_name, 'raw_value': item.raw_value, 'raw_value_masked': bool(getattr(item, 'raw_value_masked', False)), 'normalized_value': item.normalized_value, 'source': getattr(item.source, 'value', item.source), 'confidence_state': getattr(item.confidence_state, 'value', item.confidence_state), 'requires_confirmation': bool(item.requires_confirmation), 'recovery_status': getattr(item.recovery_status, 'value', item.recovery_status), 'notes': item.notes}) for item in getattr(session, 'imported_filters', [])], 'mappings': [{'mapping_id': item.mapping_id, 'filter_id': item.filter_id, 'variable_id': item.variable_id, 'mapping_method': getattr(item.mapping_method, 'value', item.mapping_method), 'effective_value': item.effective_value, 'approval_state': getattr(item.approval_state, 'value', item.approval_state), 'requires_explicit_approval': bool(item.requires_explicit_approval)} for item in getattr(session, 'execution_mappings', [])], 'semantic_fields': [{'field_id': item.field_id, 'field_name': item.field_name, 'verbose_name': item.verbose_name, 'description': item.description, 'display_format': item.display_format, 'provenance': getattr(item.provenance, 'value', item.provenance), 'is_locked': bool(item.is_locked), 'needs_review': bool(item.needs_review), 'candidates': [{'candidate_id': c.candidate_id, 'semantic_type': getattr(c.semantic_type, 'value', c.semantic_type), 'confidence': float(c.confidence or 0), 'reasoning': c.reasoning or ''} for c in (getattr(item, 'candidates', None) or [])]} for item in getattr(session, 'semantic_fields', [])], 'preview': {'preview_status': getattr(latest_preview, 'preview_status', None), 'compiled_sql': getattr(latest_preview, 'compiled_sql', None)} if latest_preview else None} + return {'session_id': session.session_id, 'version': int(getattr(session, 'version', 0) or 0), 'dataset_ref': session.dataset_ref, 'dataset_id': session.dataset_id, 'environment_id': session.environment_id, 'readiness_state': session.readiness_state.value, 'recommended_action': session.recommended_action.value, 'status': session.status.value, 'current_phase': session.current_phase.value, 'findings': [{'finding_id': item.finding_id, 'code': item.code, 'severity': item.severity.value, 'message': item.message, 'resolution_state': item.resolution_state.value} for item in getattr(session, 'findings', [])], 'imported_filters': [sanitize_imported_filter_for_assistant({'filter_id': item.filter_id, 'filter_name': item.filter_name, 'display_name': item.display_name, 'raw_value': item.raw_value, 'raw_value_masked': bool(getattr(item, 'raw_value_masked', False)), 'normalized_value': item.normalized_value, 'source': getattr(item.source, 'value', item.source), 'confidence_state': getattr(item.confidence_state, 'value', item.confidence_state), 'requires_confirmation': bool(item.requires_confirmation), 'recovery_status': getattr(item.recovery_status, 'value', item.recovery_status), 'notes': item.notes}) for item in getattr(session, 'imported_filters', [])], 'mappings': [{'mapping_id': item.mapping_id, 'filter_id': item.filter_id, 'variable_id': item.variable_id, 'mapping_method': getattr(item.mapping_method, 'value', item.mapping_method), 'effective_value': item.effective_value, 'approval_state': getattr(item.approval_state, 'value', item.approval_state), 'requires_explicit_approval': bool(item.requires_explicit_approval)} for item in getattr(session, 'execution_mappings', [])], 'semantic_fields': [{'field_id': item.field_id, 'field_name': item.field_name, 'verbose_name': item.verbose_name, 'description': item.description, 'display_format': item.display_format, 'provenance': getattr(item.provenance, 'value', item.provenance), 'is_locked': bool(item.is_locked), 'needs_review': bool(getattr(item, 'needs_review', False)), 'candidates': [{'candidate_id': c.candidate_id, 'field_id': c.field_id, 'verbose_name': c.proposed_verbose_name, 'description': c.proposed_description, 'display_format': c.proposed_display_format, 'status': getattr(c.status, 'value', c.status), 'source': c.source_id, 'score': c.confidence_score, 'created_at': c.created_at.isoformat() if c.created_at else None} for c in getattr(item, 'candidates', [])]} for item in getattr(session, 'semantic_fields', [])]} # [/DEF:_serialize_dataset_review_context:Function] @@ -145,27 +140,6 @@ def _extract_quoted_segment(message: str, label: str) -> Optional[str]: # [/DEF:_extract_quoted_segment:Function] -# [DEF:_dataset_review_conflict_http_exception:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Convert dataset-review optimistic-lock conflicts into shared 409 assistant semantics. -def _dataset_review_conflict_http_exception( - exc: DatasetReviewSessionVersionConflictError, -) -> HTTPException: - return HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail={ - "error_code": "session_version_conflict", - "message": str(exc), - "session_id": exc.session_id, - "expected_version": exc.expected_version, - "actual_version": exc.actual_version, - }, - ) - - -# [/DEF:_dataset_review_conflict_http_exception:Function] - - # [DEF:_plan_dataset_review_intent:Function] # @COMPLEXITY: 3 # @PURPOSE: Parse session-scoped dataset-review assistant commands before falling back to generic assistant tool routing. @@ -320,233 +294,4 @@ def _plan_dataset_review_intent( # [/DEF:_plan_dataset_review_intent:Function] -# [DEF:_dispatch_dataset_review_intent:Function] -# @COMPLEXITY: 4 -# @PURPOSE: Route confirmed dataset-review assistant intents through existing backend dataset-review APIs and orchestration boundaries. -# @RELATION: CALLS -> DatasetReviewOrchestrator -# @PRE: context contains valid session data and user intent. -# @POST: Returns a structured response with planned actions and confirmations. -# @SIDE_EFFECT: May update session state and enqueue tasks. -async def _dispatch_dataset_review_intent( - intent: Dict[str, Any], - current_user: User, - config_manager: ConfigManager, - db: Session, -) -> Tuple[str, Optional[str], List[AssistantAction]]: - with belief_scope("_dispatch_dataset_review_intent"): - logger.reason( - "Dispatching assistant dataset-review intent", - extra={"operation": intent.get("operation")}, - ) - entities = intent.get("entities", {}) - session_id = entities.get("dataset_review_session_id") - session_version = entities.get("session_version") - if not session_id or session_version is None: - raise HTTPException( - status_code=422, - detail="Missing dataset_review_session_id/session_version", - ) - - operation = str(intent.get("operation") or "") - repository = DatasetReviewSessionRepository(db) - if operation == "dataset_review_answer_context": - summary = str(entities.get("summary") or "") - logger.reflect( - "Returned assistant-safe dataset review context summary", - extra={"session_id": session_id, "operation": operation}, - ) - return summary, None, [] - - session = repository.load_session_detail(session_id, current_user.id) - if session is None or session.user_id != current_user.id: - logger.explore( - "Assistant dataset-review intent rejected because session was not found", - extra={"session_id": session_id, "user_id": current_user.id}, - ) - raise HTTPException( - status_code=404, detail="Dataset review session not found" - ) - - try: - repository.require_session_version(session, int(session_version)) - except DatasetReviewSessionVersionConflictError as exc: - logger.explore( - "Assistant dataset-review intent rejected due to stale session version", - extra={ - "session_id": exc.session_id, - "expected_version": exc.expected_version, - "actual_version": exc.actual_version, - "operation": operation, - }, - ) - raise _dataset_review_conflict_http_exception(exc) from exc - - logger.reason( - "Dispatching confirmed assistant dataset-review intent", - extra={ - "session_id": session_id, - "session_version": session_version, - "operation": operation, - }, - ) - - if operation == "dataset_review_approve_mappings": - mapping_ids = list(dict.fromkeys(entities.get("mapping_ids") or [])) - if not mapping_ids: - raise HTTPException( - status_code=409, detail="No pending mappings to approve" - ) - updated_count = 0 - for mapping in session.execution_mappings: - if mapping.mapping_id not in mapping_ids: - continue - mapping.approval_state = ApprovalState.APPROVED - mapping.approved_by_user_id = current_user.id - mapping.approved_at = datetime.utcnow() - updated_count += 1 - if updated_count == 0: - raise HTTPException( - status_code=409, detail="No matching mappings available to approve" - ) - session.last_activity_at = datetime.utcnow() - if session.readiness_state == ReadinessState.MAPPING_REVIEW_NEEDED: - session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW - repository.bump_session_version(session) - repository.db.commit() - repository.db.refresh(session) - repository.event_logger.log_for_session( - session, - actor_user_id=current_user.id, - event_type="assistant_mapping_approval", - event_summary="Assistant-approved warning-sensitive mappings persisted", - event_details={ - "mapping_ids": mapping_ids, - "count": updated_count, - "version": int(getattr(session, "version", 0) or 0), - }, - ) - logger.reflect( - "Assistant mapping approval persisted within optimistic-lock boundary", - extra={ - "session_id": session_id, - "updated_count": updated_count, - "version": int(getattr(session, "version", 0) or 0), - }, - ) - return ( - f"Approved {updated_count} mapping(s) for dataset review session {session_id}.", - None, - [ - AssistantAction( - type="focus_target", - label="Open mapping review", - target="mapping", - ) - ], - ) - - if operation == "dataset_review_set_field_semantics": - field_id = str(entities.get("field_id") or "").strip() - if not field_id: - raise HTTPException(status_code=422, detail="Missing field_id") - field = next( - (item for item in session.semantic_fields if item.field_id == field_id), - None, - ) - if field is None: - raise HTTPException(status_code=404, detail="Semantic field not found") - update_request = FieldSemanticUpdateRequest( - candidate_id=entities.get("candidate_id"), - verbose_name=entities.get("verbose_name"), - description=entities.get("description"), - display_format=entities.get("display_format"), - lock_field=bool(entities.get("lock_field", False)), - ) - try: - _update_semantic_field_state( - field, update_request, changed_by="assistant" - ) - except HTTPException: - raise - except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - session.last_activity_at = datetime.utcnow() - repository.bump_session_version(session) - repository.db.commit() - repository.db.refresh(session) - repository.db.refresh(field) - repository.event_logger.log_for_session( - session, - actor_user_id=current_user.id, - event_type="assistant_field_semantics_updated", - event_summary="Assistant semantic field update persisted", - event_details={ - "field_id": field.field_id, - "candidate_id": entities.get("candidate_id"), - "lock_field": bool(entities.get("lock_field", False)), - "version": int(getattr(session, "version", 0) or 0), - }, - ) - logger.reflect( - "Assistant semantic field update committed safely", - extra={ - "session_id": session_id, - "field_id": field_id, - "version": int(getattr(session, "version", 0) or 0), - }, - ) - return ( - f"Updated semantic field {field.field_name} for dataset review session {session_id}.", - None, - [ - AssistantAction( - type="focus_target", - label="Open semantic review", - target=f"field:{field.field_id}", - ) - ], - ) - - if operation == "dataset_review_generate_sql_preview": - orchestrator = DatasetReviewOrchestrator( - repository=repository, - config_manager=config_manager, - ) - result = orchestrator.prepare_launch_preview( - PreparePreviewCommand( - user=current_user, - session_id=session_id, - expected_version=int(session_version), - ) - ) - preview_status = getattr( - result.preview.preview_status, "value", result.preview.preview_status - ) - logger.reflect( - "Assistant-triggered Superset preview generation completed", - extra={ - "session_id": session_id, - "preview_status": preview_status, - }, - ) - return ( - f"SQL preview {preview_status} for dataset review session {session_id}.", - None, - [ - AssistantAction( - type="focus_target", - label="Open SQL preview", - target="sql-preview", - ) - ], - ) - - raise HTTPException( - status_code=400, detail="Unsupported dataset review operation" - ) - - -# [/DEF:_dispatch_dataset_review_intent:Function] - - # [/DEF:AssistantDatasetReview:Module] diff --git a/backend/src/api/routes/assistant/_dataset_review_dispatch.py b/backend/src/api/routes/assistant/_dataset_review_dispatch.py new file mode 100644 index 00000000..31f4a286 --- /dev/null +++ b/backend/src/api/routes/assistant/_dataset_review_dispatch.py @@ -0,0 +1,290 @@ +# [DEF:AssistantDatasetReviewDispatch:Module] +# @COMPLEXITY: 4 +# @PURPOSE: Dispatch and confirmation handling for dataset-review assistant intents. +# @LAYER: API +# @RELATION: DEPENDS_ON -> [AssistantDatasetReview] +# @RELATION: DEPENDS_ON -> [DatasetReviewOrchestrator] +# @RELATION: DEPENDS_ON -> [AssistantSchemas] +# @INVARIANT: Dataset review dispatch requires valid session version for write operations. + +from __future__ import annotations + +from datetime import datetime +from typing import Any, Dict, List, Optional, Tuple + +from fastapi import HTTPException, status +from sqlalchemy.orm import Session + +from src.core.logger import belief_scope, logger +from src.core.config_manager import ConfigManager +from src.models.dataset_review import ( + ApprovalState, + ReadinessState, + RecommendedAction, +) +from src.services.dataset_review.orchestrator import ( + DatasetReviewOrchestrator, + PreparePreviewCommand, +) +from src.services.dataset_review.repositories.session_repository import ( + DatasetReviewSessionRepository, + DatasetReviewSessionVersionConflictError, +) +from src.schemas.auth import User +from src.api.routes.dataset_review import FieldSemanticUpdateRequest, _update_semantic_field_state +from ._schemas import ( + AssistantAction, +) + + +# [DEF:_dataset_review_conflict_http_exception:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Convert dataset-review optimistic-lock conflicts into shared 409 assistant semantics. +def _dataset_review_conflict_http_exception( + exc: DatasetReviewSessionVersionConflictError, +) -> HTTPException: + return HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "error_code": "session_version_conflict", + "message": str(exc), + "session_id": exc.session_id, + "expected_version": exc.expected_version, + "actual_version": exc.actual_version, + }, + ) + + +# [/DEF:_dataset_review_conflict_http_exception:Function] + + +# [DEF:_dispatch_dataset_review_intent:Function] +# @COMPLEXITY: 4 +# @PURPOSE: Route confirmed dataset-review assistant intents through existing backend dataset-review APIs and orchestration boundaries. +# @RELATION: CALLS -> DatasetReviewOrchestrator +# @PRE: context contains valid session data and user intent. +# @POST: Returns a structured response with planned actions and confirmations. +# @SIDE_EFFECT: May update session state and enqueue tasks. +async def _dispatch_dataset_review_intent( + intent: Dict[str, Any], + current_user: User, + config_manager: ConfigManager, + db: Session, +) -> Tuple[str, Optional[str], List[AssistantAction]]: + with belief_scope("_dispatch_dataset_review_intent"): + logger.reason( + "Dispatching assistant dataset-review intent", + extra={"operation": intent.get("operation")}, + ) + entities = intent.get("entities", {}) + session_id = entities.get("dataset_review_session_id") + session_version = entities.get("session_version") + if not session_id or session_version is None: + raise HTTPException( + status_code=422, + detail="Missing dataset_review_session_id/session_version", + ) + + operation = str(intent.get("operation") or "") + repository = DatasetReviewSessionRepository(db) + if operation == "dataset_review_answer_context": + summary = str(entities.get("summary") or "") + logger.reflect( + "Returned assistant-safe dataset review context summary", + extra={"session_id": session_id, "operation": operation}, + ) + return summary, None, [] + + session = repository.load_session_detail(session_id, current_user.id) + if session is None or session.user_id != current_user.id: + logger.explore( + "Assistant dataset-review intent rejected because session was not found", + extra={"session_id": session_id, "user_id": current_user.id}, + ) + raise HTTPException( + status_code=404, detail="Dataset review session not found" + ) + + try: + repository.require_session_version(session, int(session_version)) + except DatasetReviewSessionVersionConflictError as exc: + logger.explore( + "Assistant dataset-review intent rejected due to stale session version", + extra={ + "session_id": exc.session_id, + "expected_version": exc.expected_version, + "actual_version": exc.actual_version, + "operation": operation, + }, + ) + raise _dataset_review_conflict_http_exception(exc) from exc + + logger.reason( + "Dispatching confirmed assistant dataset-review intent", + extra={ + "session_id": session_id, + "session_version": session_version, + "operation": operation, + }, + ) + + if operation == "dataset_review_approve_mappings": + mapping_ids = list(dict.fromkeys(entities.get("mapping_ids") or [])) + if not mapping_ids: + raise HTTPException( + status_code=409, detail="No pending mappings to approve" + ) + updated_count = 0 + for mapping in session.execution_mappings: + if mapping.mapping_id not in mapping_ids: + continue + mapping.approval_state = ApprovalState.APPROVED + mapping.approved_by_user_id = current_user.id + mapping.approved_at = datetime.utcnow() + updated_count += 1 + if updated_count == 0: + raise HTTPException( + status_code=409, detail="No matching mappings available to approve" + ) + session.last_activity_at = datetime.utcnow() + if session.readiness_state == ReadinessState.MAPPING_REVIEW_NEEDED: + session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW + repository.bump_session_version(session) + repository.db.commit() + repository.db.refresh(session) + repository.event_logger.log_for_session( + session, + actor_user_id=current_user.id, + event_type="assistant_mapping_approval", + event_summary="Assistant-approved warning-sensitive mappings persisted", + event_details={ + "mapping_ids": mapping_ids, + "count": updated_count, + "version": int(getattr(session, "version", 0) or 0), + }, + ) + logger.reflect( + "Assistant mapping approval persisted within optimistic-lock boundary", + extra={ + "session_id": session_id, + "updated_count": updated_count, + "version": int(getattr(session, "version", 0) or 0), + }, + ) + return ( + f"Approved {updated_count} mapping(s) for dataset review session {session_id}.", + None, + [ + AssistantAction( + type="focus_target", + label="Open mapping review", + target="mapping", + ) + ], + ) + + if operation == "dataset_review_set_field_semantics": + field_id = str(entities.get("field_id") or "").strip() + if not field_id: + raise HTTPException(status_code=422, detail="Missing field_id") + field = next( + (item for item in session.semantic_fields if item.field_id == field_id), + None, + ) + if field is None: + raise HTTPException(status_code=404, detail="Semantic field not found") + update_request = FieldSemanticUpdateRequest( + candidate_id=entities.get("candidate_id"), + verbose_name=entities.get("verbose_name"), + description=entities.get("description"), + display_format=entities.get("display_format"), + lock_field=bool(entities.get("lock_field", False)), + ) + try: + _update_semantic_field_state( + field, update_request, changed_by="assistant" + ) + except HTTPException: + raise + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + session.last_activity_at = datetime.utcnow() + repository.bump_session_version(session) + repository.db.commit() + repository.db.refresh(session) + repository.db.refresh(field) + repository.event_logger.log_for_session( + session, + actor_user_id=current_user.id, + event_type="assistant_field_semantics_updated", + event_summary="Assistant semantic field update persisted", + event_details={ + "field_id": field.field_id, + "candidate_id": entities.get("candidate_id"), + "lock_field": bool(entities.get("lock_field", False)), + "version": int(getattr(session, "version", 0) or 0), + }, + ) + logger.reflect( + "Assistant semantic field update committed safely", + extra={ + "session_id": session_id, + "field_id": field_id, + "version": int(getattr(session, "version", 0) or 0), + }, + ) + return ( + f"Updated semantic field {field.field_name} for dataset review session {session_id}.", + None, + [ + AssistantAction( + type="focus_target", + label="Open semantic review", + target=f"field:{field.field_id}", + ) + ], + ) + + if operation == "dataset_review_generate_sql_preview": + orchestrator = DatasetReviewOrchestrator( + repository=repository, + config_manager=config_manager, + ) + result = orchestrator.prepare_launch_preview( + PreparePreviewCommand( + user=current_user, + session_id=session_id, + expected_version=int(session_version), + ) + ) + preview_status = getattr( + result.preview.preview_status, "value", result.preview.preview_status + ) + logger.reflect( + "Assistant-triggered Superset preview generation completed", + extra={ + "session_id": session_id, + "preview_status": preview_status, + }, + ) + return ( + f"SQL preview {preview_status} for dataset review session {session_id}.", + None, + [ + AssistantAction( + type="focus_target", + label="Open SQL preview", + target="sql-preview", + ) + ], + ) + + raise HTTPException( + status_code=400, detail="Unsupported dataset review operation" + ) + + +# [/DEF:_dispatch_dataset_review_intent:Function] + + +# [/DEF:AssistantDatasetReviewDispatch:Module] diff --git a/backend/src/api/routes/assistant/_dispatch.py b/backend/src/api/routes/assistant/_dispatch.py index e16271ec..1b377520 100644 --- a/backend/src/api/routes/assistant/_dispatch.py +++ b/backend/src/api/routes/assistant/_dispatch.py @@ -36,7 +36,7 @@ from ._resolvers import ( ) from ._history import _coerce_query_bool from ._llm_planner import _check_any_permission -from ._dataset_review import _dispatch_dataset_review_intent +from ._dataset_review_dispatch import _dispatch_dataset_review_intent git_service = GitService() diff --git a/backend/src/api/routes/assistant/_llm_planner.py b/backend/src/api/routes/assistant/_llm_planner.py index dd0ed607..fb334b89 100644 --- a/backend/src/api/routes/assistant/_llm_planner.py +++ b/backend/src/api/routes/assistant/_llm_planner.py @@ -4,11 +4,11 @@ # @LAYER: API # @RELATION: DEPENDS_ON -> [AssistantSchemas] # @RELATION: DEPENDS_ON -> [AssistantResolvers] +# @RELATION: DISPATCHES -> [AssistantLlmPlannerIntent] # @INVARIANT: Tool catalog is filtered by user permissions before being sent to LLM. from __future__ import annotations -import json from typing import Any, Dict, List, Optional, Tuple from fastapi import HTTPException @@ -19,20 +19,13 @@ from src.core.config_manager import ConfigManager from src.dependencies import has_permission from src.schemas.auth import User from src.services.llm_provider import LLMProviderService -from src.services.llm_prompt_templates import ( - normalize_llm_settings, - resolve_bound_provider_id, -) -from src.plugins.llm_analysis.service import LLMClient -from src.plugins.llm_analysis.models import LLMProviderType +from src.services.llm_prompt_templates import resolve_bound_provider_id from ._schemas import ( INTENT_PERMISSION_CHECKS, AssistantAction, ) from ._resolvers import ( _get_default_environment_id, - _is_production_env, - _resolve_provider_id, ) @@ -302,140 +295,4 @@ def _coerce_intent_entities(intent: Dict[str, Any]) -> Dict[str, Any]: # [/DEF:_coerce_intent_entities:Function] -# [DEF:_plan_intent_with_llm:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Use active LLM provider to select best tool/operation from dynamic catalog. -# @PRE: tools list contains allowed operations for current user. -# @POST: Returns normalized intent dict when planning succeeds; otherwise None. -async def _plan_intent_with_llm( - message: str, - tools: List[Dict[str, Any]], - db: Session, - config_manager: ConfigManager, -) -> Optional[Dict[str, Any]]: - if not tools: - return None - - llm_settings = normalize_llm_settings(config_manager.get_config().settings.llm) - planner_provider_token = llm_settings.get("assistant_planner_provider") - planner_model_override = llm_settings.get("assistant_planner_model") - llm_service = LLMProviderService(db) - providers = llm_service.get_all_providers() - provider_id = _resolve_provider_id(planner_provider_token, db) - provider = next((p for p in providers if p.id == provider_id), None) - if not provider: - return None - api_key = llm_service.get_decrypted_api_key(provider.id) - if not api_key: - return None - - planner = LLMClient( - provider_type=LLMProviderType(provider.provider_type), - api_key=api_key, - base_url=provider.base_url, - default_model=planner_model_override or provider.default_model, - ) - - system_instruction = ( - "You are a deterministic intent planner for backend tools.\n" - "Choose exactly one operation from available_tools or return clarify.\n" - "Output strict JSON object:\n" - "{" - '"domain": string, ' - '"operation": string, ' - '"entities": object, ' - '"confidence": number, ' - '"risk_level": "safe"|"guarded"|"dangerous", ' - '"requires_confirmation": boolean' - "}\n" - "Rules:\n" - "- Use only operation names from available_tools.\n" - '- If input is ambiguous, operation must be "clarify" with low confidence.\n' - "- If dashboard is provided as name/slug (e.g., COVID), put it into entities.dashboard_ref.\n" - "- Keep entities minimal and factual.\n" - ) - payload = { - "available_tools": tools, - "user_message": message, - "known_environments": [ - {"id": e.id, "name": e.name} for e in config_manager.get_environments() - ], - } - try: - response = await planner.get_json_completion( - [ - {"role": "system", "content": system_instruction}, - {"role": "user", "content": json.dumps(payload, ensure_ascii=False)}, - ] - ) - except Exception as exc: - import traceback - - logger.warning( - f"[assistant.planner][fallback] LLM planner unavailable: {exc}\n{traceback.format_exc()}" - ) - return None - if not isinstance(response, dict): - return None - - operation = response.get("operation") - valid_ops = {tool["operation"] for tool in tools} - if operation == "clarify": - return { - "domain": "unknown", - "operation": "clarify", - "entities": {}, - "confidence": float(response.get("confidence", 0.3)), - "risk_level": "safe", - "requires_confirmation": False, - } - if operation not in valid_ops: - return None - - by_operation = {tool["operation"]: tool for tool in tools} - selected = by_operation[operation] - intent = { - "domain": response.get("domain") or selected["domain"], - "operation": operation, - "entities": response.get("entities", {}), - "confidence": float(response.get("confidence", 0.75)), - "risk_level": response.get("risk_level") or selected["risk_level"], - "requires_confirmation": bool( - response.get("requires_confirmation", selected["requires_confirmation"]) - ), - } - intent = _coerce_intent_entities(intent) - - defaults = selected.get("defaults") or {} - for key, value in defaults.items(): - if value and not intent["entities"].get(key): - intent["entities"][key] = value - - if operation in {"deploy_dashboard", "execute_migration"}: - env_token = intent["entities"].get("environment") or intent["entities"].get( - "target_env" - ) - if _is_production_env(env_token, config_manager): - intent["risk_level"] = "dangerous" - intent["requires_confirmation"] = True - return intent - - -# [/DEF:_plan_intent_with_llm:Function] - - -# [DEF:_authorize_intent:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Validate user permissions for parsed intent before confirmation/dispatch. -# @PRE: intent.operation is present for known assistant command domains. -# @POST: Returns if authorized; raises HTTPException(403) when denied. -def _authorize_intent(intent: Dict[str, Any], current_user: User): - operation = intent.get("operation") - if operation in INTENT_PERMISSION_CHECKS: - _check_any_permission(current_user, INTENT_PERMISSION_CHECKS[operation]) - - -# [/DEF:_authorize_intent:Function] - - # [/DEF:AssistantLlmPlanner:Module] diff --git a/backend/src/api/routes/assistant/_llm_planner_intent.py b/backend/src/api/routes/assistant/_llm_planner_intent.py new file mode 100644 index 00000000..6661a86e --- /dev/null +++ b/backend/src/api/routes/assistant/_llm_planner_intent.py @@ -0,0 +1,173 @@ +# [DEF:AssistantLlmPlannerIntent:Module] +# @COMPLEXITY: 3 +# @PURPOSE: LLM-based intent planning and authorization for the assistant API — separated from tool catalog. +# @LAYER: API +# @RELATION: DEPENDS_ON -> [AssistantLlmPlanner] +# @RELATION: DEPENDS_ON -> [AssistantResolvers] +# @INVARIANT: Production deployments always require confirmation. + +from __future__ import annotations + +import json +from typing import Any, Dict, List, Optional + +from sqlalchemy.orm import Session + +from src.core.logger import belief_scope, logger +from src.core.config_manager import ConfigManager +from src.schemas.auth import User +from src.services.llm_provider import LLMProviderService +from src.services.llm_prompt_templates import ( + normalize_llm_settings, + resolve_bound_provider_id, +) +from src.plugins.llm_analysis.service import LLMClient +from src.plugins.llm_analysis.models import LLMProviderType +from ._schemas import INTENT_PERMISSION_CHECKS +from ._resolvers import ( + _is_production_env, + _resolve_provider_id, +) +from ._llm_planner import ( + _check_any_permission, + _coerce_intent_entities, +) + + +# [DEF:_plan_intent_with_llm:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Use active LLM provider to select best tool/operation from dynamic catalog. +# @PRE: tools list contains allowed operations for current user. +# @POST: Returns normalized intent dict when planning succeeds; otherwise None. +async def _plan_intent_with_llm( + message: str, + tools: List[Dict[str, Any]], + db: Session, + config_manager: ConfigManager, +) -> Optional[Dict[str, Any]]: + if not tools: + return None + + llm_settings = normalize_llm_settings(config_manager.get_config().settings.llm) + planner_provider_token = llm_settings.get("assistant_planner_provider") + planner_model_override = llm_settings.get("assistant_planner_model") + llm_service = LLMProviderService(db) + providers = llm_service.get_all_providers() + provider_id = _resolve_provider_id(planner_provider_token, db) + provider = next((p for p in providers if p.id == provider_id), None) + if not provider: + return None + api_key = llm_service.get_decrypted_api_key(provider.id) + if not api_key: + return None + + planner = LLMClient( + provider_type=LLMProviderType(provider.provider_type), + api_key=api_key, + base_url=provider.base_url, + default_model=planner_model_override or provider.default_model, + ) + + system_instruction = ( + "You are a deterministic intent planner for backend tools.\n" + "Choose exactly one operation from available_tools or return clarify.\n" + "Output strict JSON object:\n" + "{" + '"domain": string, ' + '"operation": string, ' + '"entities": object, ' + '"confidence": number, ' + '"risk_level": "safe"|"guarded"|"dangerous", ' + '"requires_confirmation": boolean' + "}\n" + "Rules:\n" + "- Use only operation names from available_tools.\n" + '- If input is ambiguous, operation must be "clarify" with low confidence.\n' + "- If dashboard is provided as name/slug (e.g., COVID), put it into entities.dashboard_ref.\n" + "- Keep entities minimal and factual.\n" + ) + payload = { + "available_tools": tools, + "user_message": message, + "known_environments": [ + {"id": e.id, "name": e.name} for e in config_manager.get_environments() + ], + } + try: + response = await planner.get_json_completion( + [ + {"role": "system", "content": system_instruction}, + {"role": "user", "content": json.dumps(payload, ensure_ascii=False)}, + ] + ) + except Exception as exc: + import traceback + + logger.warning( + f"[assistant.planner][fallback] LLM planner unavailable: {exc}\n{traceback.format_exc()}" + ) + return None + if not isinstance(response, dict): + return None + + operation = response.get("operation") + valid_ops = {tool["operation"] for tool in tools} + if operation == "clarify": + return { + "domain": "unknown", + "operation": "clarify", + "entities": {}, + "confidence": float(response.get("confidence", 0.3)), + "risk_level": "safe", + "requires_confirmation": False, + } + if operation not in valid_ops: + return None + + by_operation = {tool["operation"]: tool for tool in tools} + selected = by_operation[operation] + intent = { + "domain": response.get("domain") or selected["domain"], + "operation": operation, + "entities": response.get("entities", {}), + "confidence": float(response.get("confidence", 0.75)), + "risk_level": response.get("risk_level") or selected["risk_level"], + "requires_confirmation": bool( + response.get("requires_confirmation", selected["requires_confirmation"]) + ), + } + intent = _coerce_intent_entities(intent) + + defaults = selected.get("defaults") or {} + for key, value in defaults.items(): + if value and not intent["entities"].get(key): + intent["entities"][key] = value + + if operation in {"deploy_dashboard", "execute_migration"}: + env_token = intent["entities"].get("environment") or intent["entities"].get( + "target_env" + ) + if _is_production_env(env_token, config_manager): + intent["risk_level"] = "dangerous" + intent["requires_confirmation"] = True + return intent + + +# [/DEF:_plan_intent_with_llm:Function] + + +# [DEF:_authorize_intent:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Validate user permissions for parsed intent before confirmation/dispatch. +# @PRE: intent.operation is present for known assistant command domains. +# @POST: Returns if authorized; raises HTTPException(403) when denied. +def _authorize_intent(intent: Dict[str, Any], current_user: User): + operation = intent.get("operation") + if operation in INTENT_PERMISSION_CHECKS: + _check_any_permission(current_user, INTENT_PERMISSION_CHECKS[operation]) + + +# [/DEF:_authorize_intent:Function] + + +# [/DEF:AssistantLlmPlannerIntent:Module] diff --git a/backend/src/api/routes/assistant/_resolvers.py b/backend/src/api/routes/assistant/_resolvers.py index 1a15abee..8996b61b 100644 --- a/backend/src/api/routes/assistant/_resolvers.py +++ b/backend/src/api/routes/assistant/_resolvers.py @@ -38,7 +38,6 @@ def _extract_id(text: str, patterns: List[str]) -> Optional[str]: # [/DEF:_extract_id:Function] - # [DEF:_resolve_env_id:Function] # @COMPLEXITY: 2 # @PURPOSE: Resolve environment identifier/name token to canonical environment id. @@ -60,7 +59,6 @@ def _resolve_env_id( # [/DEF:_resolve_env_id:Function] - # [DEF:_is_production_env:Function] # @COMPLEXITY: 2 # @PURPOSE: Determine whether environment token resolves to production-like target. @@ -80,7 +78,6 @@ def _is_production_env(token: Optional[str], config_manager: ConfigManager) -> b # [/DEF:_is_production_env:Function] - # [DEF:_resolve_provider_id:Function] # @COMPLEXITY: 2 # @PURPOSE: Resolve provider token to provider id with active/default fallback. @@ -118,7 +115,6 @@ def _resolve_provider_id( # [/DEF:_resolve_provider_id:Function] - # [DEF:_get_default_environment_id:Function] # @COMPLEXITY: 2 # @PURPOSE: Resolve default environment id from settings or first configured environment. @@ -144,7 +140,6 @@ def _get_default_environment_id(config_manager: ConfigManager) -> Optional[str]: # [/DEF:_get_default_environment_id:Function] - # [DEF:_resolve_dashboard_id_by_ref:Function] # @COMPLEXITY: 2 # @PURPOSE: Resolve dashboard id by title or slug reference in selected environment. @@ -198,7 +193,6 @@ def _resolve_dashboard_id_by_ref( # [/DEF:_resolve_dashboard_id_by_ref:Function] - # [DEF:_resolve_dashboard_id_entity:Function] # @COMPLEXITY: 2 # @PURPOSE: Resolve dashboard id from intent entities using numeric id or dashboard_ref fallback. @@ -241,7 +235,6 @@ def _resolve_dashboard_id_entity( # [/DEF:_resolve_dashboard_id_entity:Function] - # [DEF:_get_environment_name_by_id:Function] # @COMPLEXITY: 2 # @PURPOSE: Resolve human-readable environment name by id. @@ -260,7 +253,6 @@ def _get_environment_name_by_id( # [/DEF:_get_environment_name_by_id:Function] - # [DEF:_extract_result_deep_links:Function] # @COMPLEXITY: 2 # @PURPOSE: Build deep-link actions to verify task result from assistant chat. @@ -335,7 +327,6 @@ def _extract_result_deep_links( # [/DEF:_extract_result_deep_links:Function] - # [DEF:_build_task_observability_summary:Function] # @COMPLEXITY: 2 # @PURPOSE: Build compact textual summary for completed tasks to reduce "black box" effect. @@ -403,5 +394,4 @@ def _build_task_observability_summary(task: Any, config_manager: ConfigManager) # [/DEF:_build_task_observability_summary:Function] - # [/DEF:AssistantResolvers:Module] diff --git a/backend/src/api/routes/assistant/_routes.py b/backend/src/api/routes/assistant/_routes.py index f891cb35..b197dad3 100644 --- a/backend/src/api/routes/assistant/_routes.py +++ b/backend/src/api/routes/assistant/_routes.py @@ -8,6 +8,7 @@ # @RELATION: DEPENDS_ON -> [AssistantLlmPlanner] # @RELATION: DEPENDS_ON -> [AssistantDatasetReview] # @RELATION: DEPENDS_ON -> [AssistantDispatch] +# @RELATION: DISPATCHES -> [AssistantAdminRoutes] # @INVARIANT: Risky operations are never executed without valid confirmation token. from __future__ import annotations @@ -60,11 +61,8 @@ from ._history import ( _update_confirmation_state, ) from ._command_parser import _parse_command -from ._llm_planner import ( - _authorize_intent, - _build_tool_catalog, - _plan_intent_with_llm, -) +from ._llm_planner import _build_tool_catalog +from ._llm_planner_intent import _authorize_intent, _plan_intent_with_llm from ._dataset_review import ( _load_dataset_review_context, _plan_dataset_review_intent, @@ -337,265 +335,4 @@ async def cancel_operation( # [/DEF:cancel_operation:Function] -# [DEF:list_conversations:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Return paginated conversation list for current user with archived flag and last message preview. -# @PRE: Authenticated user context and valid pagination params. -# @POST: Conversations are grouped by conversation_id sorted by latest activity descending. -# @RETURN: Dict with items, paging metadata, and archive segmentation counts. -@router.get("/conversations") -async def list_conversations( - page: int = Query(1, ge=1), - page_size: int = Query(20, ge=1, le=100), - include_archived: bool = Query(False), - archived_only: bool = Query(False), - search: Optional[str] = Query(None), - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - with belief_scope("assistant.conversations"): - user_id = current_user.id - include_archived = _coerce_query_bool(include_archived) - archived_only = _coerce_query_bool(archived_only) - _cleanup_history_ttl(db, user_id) - - rows = ( - db.query(AssistantMessageRecord) - .filter(AssistantMessageRecord.user_id == user_id) - .order_by(desc(AssistantMessageRecord.created_at)) - .all() - ) - - summary: Dict[str, Dict[str, Any]] = {} - for row in rows: - conv_id = row.conversation_id - if not conv_id: - continue - created_at = row.created_at or datetime.utcnow() - if conv_id not in summary: - summary[conv_id] = { - "conversation_id": conv_id, - "title": "", - "updated_at": created_at, - "last_message": row.text, - "last_role": row.role, - "last_state": row.state, - "last_task_id": row.task_id, - "message_count": 0, - } - item = summary[conv_id] - item["message_count"] += 1 - if row.role == "user" and row.text and not item["title"]: - item["title"] = row.text.strip()[:80] - - items = [] - search_term = search.lower().strip() if search else "" - archived_total = sum( - 1 - for c in summary.values() - if _is_conversation_archived(c.get("updated_at")) - ) - active_total = len(summary) - archived_total - for conv in summary.values(): - conv["archived"] = _is_conversation_archived(conv.get("updated_at")) - if not conv.get("title"): - conv["title"] = f"Conversation {conv['conversation_id'][:8]}" - if search_term: - haystack = ( - f"{conv.get('title', '')} {conv.get('last_message', '')}".lower() - ) - if search_term not in haystack: - continue - if archived_only and not conv["archived"]: - continue - if not archived_only and not include_archived and conv["archived"]: - continue - updated = conv.get("updated_at") - conv["updated_at"] = ( - updated.isoformat() if isinstance(updated, datetime) else None - ) - items.append(conv) - - items.sort(key=lambda x: x.get("updated_at") or "", reverse=True) - total = len(items) - start = (page - 1) * page_size - page_items = items[start : start + page_size] - - return { - "items": page_items, - "total": total, - "page": page, - "page_size": page_size, - "has_next": start + page_size < total, - "active_total": active_total, - "archived_total": archived_total, - } - - -# [/DEF:list_conversations:Function] - - -# [DEF:delete_conversation:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Soft-delete or hard-delete a conversation and clear its in-memory trace. -# @PRE: conversation_id belongs to current_user. -# @POST: Conversation records are removed from DB and CONVERSATIONS cache. -@router.delete("/conversations/{conversation_id}") -async def delete_conversation( - conversation_id: str, - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - with belief_scope("assistant.conversations.delete"): - user_id = current_user.id - - # 1. Remove from in-memory cache - key = (user_id, conversation_id) - if key in CONVERSATIONS: - del CONVERSATIONS[key] - - # 2. Delete from database - deleted_count = ( - db.query(AssistantMessageRecord) - .filter( - AssistantMessageRecord.user_id == user_id, - AssistantMessageRecord.conversation_id == conversation_id, - ) - .delete() - ) - - db.commit() - - if deleted_count == 0: - raise HTTPException( - status_code=404, detail="Conversation not found or already deleted" - ) - - return { - "status": "success", - "deleted": deleted_count, - "conversation_id": conversation_id, - } - - -# [/DEF:delete_conversation:Function] - - -@router.get("/history") -# [DEF:get_history:Function] -# @PURPOSE: Retrieve paginated assistant conversation history for current user. -# @PRE: Authenticated user is available and page params are valid. -# @POST: Returns persistent messages and mirrored in-memory snapshot for diagnostics. -# @RETURN: Dict with items, paging metadata, and resolved conversation_id. -async def get_history( - page: int = Query(1, ge=1), - page_size: int = Query(20, ge=1, le=100), - conversation_id: Optional[str] = Query(None), - from_latest: bool = Query(False), - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - with belief_scope("assistant.history"): - user_id = current_user.id - _cleanup_history_ttl(db, user_id) - conv_id = _resolve_or_create_conversation(user_id, conversation_id, db) - - base_query = db.query(AssistantMessageRecord).filter( - AssistantMessageRecord.user_id == user_id, - AssistantMessageRecord.conversation_id == conv_id, - ) - total = base_query.count() - start = (page - 1) * page_size - if from_latest: - rows = ( - base_query.order_by(desc(AssistantMessageRecord.created_at)) - .offset(start) - .limit(page_size) - .all() - ) - rows = list(reversed(rows)) - else: - rows = ( - base_query.order_by(AssistantMessageRecord.created_at.asc()) - .offset(start) - .limit(page_size) - .all() - ) - - persistent_items = [ - { - "message_id": row.id, - "conversation_id": row.conversation_id, - "role": row.role, - "text": row.text, - "state": row.state, - "task_id": row.task_id, - "confirmation_id": row.confirmation_id, - "created_at": row.created_at.isoformat() if row.created_at else None, - "metadata": row.payload, - } - for row in rows - ] - - memory_items = CONVERSATIONS.get((user_id, conv_id), []) - return { - "items": persistent_items, - "memory_items": memory_items, - "total": total, - "page": page, - "page_size": page_size, - "has_next": start + page_size < total, - "from_latest": from_latest, - "conversation_id": conv_id, - } - - -# [/DEF:get_history:Function] - - -@router.get("/audit") -# [DEF:get_assistant_audit:Function] -# @PURPOSE: Return assistant audit decisions for current user from persistent and in-memory stores. -# @PRE: User has tasks:READ permission. -# @POST: Audit payload is returned in reverse chronological order from DB. -# @RETURN: Dict with persistent and memory audit slices. -async def get_assistant_audit( - limit: int = Query(50, ge=1, le=500), - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), - _=Depends(has_permission("tasks", "READ")), -): - with belief_scope("assistant.audit"): - memory_rows = ASSISTANT_AUDIT.get(current_user.id, []) - db_rows = ( - db.query(AssistantAuditRecord) - .filter(AssistantAuditRecord.user_id == current_user.id) - .order_by(AssistantAuditRecord.created_at.desc()) - .limit(limit) - .all() - ) - persistent = [ - { - "id": row.id, - "user_id": row.user_id, - "conversation_id": row.conversation_id, - "decision": row.decision, - "task_id": row.task_id, - "message": row.message, - "payload": row.payload, - "created_at": row.created_at.isoformat() if row.created_at else None, - } - for row in db_rows - ] - return { - "items": persistent, - "memory_items": memory_rows[-limit:], - "total": len(persistent), - "memory_total": len(memory_rows), - } - - -# [/DEF:get_assistant_audit:Function] - - # [/DEF:AssistantRoutes:Module] diff --git a/backend/src/api/routes/dashboards.py b/backend/src/api/routes/dashboards.py deleted file mode 100644 index bb4f212d..00000000 --- a/backend/src/api/routes/dashboards.py +++ /dev/null @@ -1,1619 +0,0 @@ -# [DEF:DashboardsApi:Module] -# -# @COMPLEXITY: 5 -# @SEMANTICS: api, dashboards, resources, hub -# @PURPOSE: API endpoints for the Dashboard Hub - listing dashboards with Git and task status -# @LAYER: API -# @RELATION: DEPENDS_ON ->[AppDependencies] -# @RELATION: DEPENDS_ON ->[ResourceService] -# @RELATION: DEPENDS_ON ->[SupersetClient] -# -# @INVARIANT: All dashboard responses include git_status and last_task metadata -# -# @PRE: Valid environment configurations exist in ConfigManager. -# @POST: Dashboard responses are projected into DashboardsResponse DTO. -# @SIDE_EFFECT: Performs external calls to Superset API and potentially Git providers. -# @DATA_CONTRACT: Input(env_id, filters) -> Output(DashboardsResponse) -# -# @TEST_CONTRACT: DashboardsAPI -> { -# required_fields: {env_id: string, page: integer, page_size: integer}, -# optional_fields: {search: string}, -# invariants: ["Pagination must be valid", "Environment must exist"] -# } -# -# @TEST_FIXTURE: dashboard_list_happy -> { -# "env_id": "prod", -# "expected_count": 1, -# "dashboards": [{"id": 1, "title": "Main Revenue"}] -# } -# -# @TEST_EDGE: pagination_zero_page -> {"env_id": "prod", "page": 0, "status": 400} -# @TEST_EDGE: pagination_oversize -> {"env_id": "prod", "page_size": 101, "status": 400} -# @TEST_EDGE: missing_env -> {"env_id": "ghost", "status": 404} -# @TEST_EDGE: empty_dashboards -> {"env_id": "empty_env", "expected_total": 0} -# @TEST_EDGE: external_superset_failure -> {"env_id": "bad_conn", "status": 503} -# -# @TEST_INVARIANT: metadata_consistency -> verifies: [dashboard_list_happy, empty_dashboards] -# - -# [SECTION: IMPORTS] -from fastapi import APIRouter, Depends, HTTPException, Query, Response -from fastapi.responses import JSONResponse -from typing import List, Optional, Dict, Any, Literal -import re -from urllib.parse import urlparse -from pydantic import BaseModel, Field -from sqlalchemy.orm import Session -from ...dependencies import ( - get_config_manager, - get_task_manager, - get_resource_service, - get_mapping_service, - get_current_user, - has_permission, -) -from ...core.database import get_db -from ...core.async_superset_client import AsyncSupersetClient -from ...core.logger import logger, belief_scope -from ...core.superset_client import SupersetClient -from ...core.superset_profile_lookup import SupersetAccountLookupAdapter -from ...core.utils.network import DashboardNotFoundError -from ...models.auth import User -from ...services.profile_service import ProfileService -from ...services.resource_service import ResourceService -# [/SECTION] - -router = APIRouter(prefix="/api/dashboards", tags=["Dashboards"]) - - -# [DEF:GitStatus:DataClass] -# @COMPLEXITY: 1 -# @PURPOSE: DTO for dashboard Git synchronization status. -class GitStatus(BaseModel): - branch: Optional[str] = None - sync_status: Optional[str] = Field(None, pattern="^OK|DIFF|NO_REPO|ERROR$") - has_repo: Optional[bool] = None - has_changes_for_commit: Optional[bool] = None - - -# [/DEF:GitStatus:DataClass] - - -# [DEF:LastTask:DataClass] -# @COMPLEXITY: 2 -# @PURPOSE: DTO for the most recent background task associated with a dashboard. -class LastTask(BaseModel): - task_id: Optional[str] = None - status: Optional[str] = Field( - None, - pattern="^PENDING|RUNNING|SUCCESS|FAILED|ERROR|AWAITING_INPUT|WAITING_INPUT|AWAITING_MAPPING$", - ) - validation_status: Optional[str] = Field(None, pattern="^PASS|FAIL|WARN|UNKNOWN$") - - -# [/DEF:LastTask:DataClass] - - -# [DEF:DashboardItem:DataClass] -# @COMPLEXITY: 1 -# @PURPOSE: DTO representing a single dashboard with projected metadata. -class DashboardItem(BaseModel): - id: int - title: str - slug: Optional[str] = None - url: Optional[str] = None - last_modified: Optional[str] = None - created_by: Optional[str] = None - modified_by: Optional[str] = None - owners: Optional[List[str]] = None - git_status: Optional[GitStatus] = None - last_task: Optional[LastTask] = None - - -# [/DEF:DashboardItem:DataClass] - - -# [DEF:EffectiveProfileFilter:DataClass] -# @COMPLEXITY: 1 -# @PURPOSE: Metadata about applied profile filters for UI context. -class EffectiveProfileFilter(BaseModel): - applied: bool - source_page: Literal["dashboards_main", "other"] = "dashboards_main" - override_show_all: bool = False - username: Optional[str] = None - match_logic: Optional[ - Literal["owners_or_modified_by", "slug_only", "owners_or_modified_by+slug_only"] - ] = None - - -# [/DEF:EffectiveProfileFilter:DataClass] - - -# [DEF:DashboardsResponse:DataClass] -# @COMPLEXITY: 1 -# @PURPOSE: Envelope DTO for paginated dashboards list. -class DashboardsResponse(BaseModel): - dashboards: List[DashboardItem] - total: int - page: int - page_size: int - total_pages: int - effective_profile_filter: Optional[EffectiveProfileFilter] = None - - -# [/DEF:DashboardsResponse:DataClass] - - -# [DEF:DashboardChartItem:DataClass] -# @COMPLEXITY: 1 -# @PURPOSE: DTO for a chart linked to a dashboard. -class DashboardChartItem(BaseModel): - id: int - title: str - viz_type: Optional[str] = None - dataset_id: Optional[int] = None - last_modified: Optional[str] = None - overview: Optional[str] = None - - -# [/DEF:DashboardChartItem:DataClass] - - -# [DEF:DashboardDatasetItem:DataClass] -# @COMPLEXITY: 1 -# @PURPOSE: DTO for a dataset associated with a dashboard. -class DashboardDatasetItem(BaseModel): - id: int - table_name: str - schema: Optional[str] = None - database: str - last_modified: Optional[str] = None - overview: Optional[str] = None - - -# [/DEF:DashboardDatasetItem:DataClass] - - -# [DEF:DashboardDetailResponse:DataClass] -# @COMPLEXITY: 1 -# @PURPOSE: Detailed dashboard metadata including children. -class DashboardDetailResponse(BaseModel): - id: int - title: str - slug: Optional[str] = None - url: Optional[str] = None - description: Optional[str] = None - last_modified: Optional[str] = None - published: Optional[bool] = None - charts: List[DashboardChartItem] - datasets: List[DashboardDatasetItem] - chart_count: int - dataset_count: int - - -# [/DEF:DashboardDetailResponse:DataClass] - - -# [DEF:DashboardTaskHistoryItem:DataClass] -# @COMPLEXITY: 1 -# @PURPOSE: Individual history record entry. -class DashboardTaskHistoryItem(BaseModel): - id: str - plugin_id: str - status: str - validation_status: Optional[str] = None - started_at: Optional[str] = None - finished_at: Optional[str] = None - env_id: Optional[str] = None - summary: Optional[str] = None - - -# [/DEF:DashboardTaskHistoryItem:DataClass] - - -# [DEF:DashboardTaskHistoryResponse:DataClass] -# @COMPLEXITY: 1 -# @PURPOSE: Collection DTO for task history. -class DashboardTaskHistoryResponse(BaseModel): - dashboard_id: int - items: List[DashboardTaskHistoryItem] - - -# [/DEF:DashboardTaskHistoryResponse:DataClass] - - -# [DEF:DatabaseMapping:DataClass] -# @COMPLEXITY: 2 -# @PURPOSE: DTO for cross-environment database ID mapping. -class DatabaseMapping(BaseModel): - source_db: str - target_db: str - source_db_uuid: Optional[str] = None - target_db_uuid: Optional[str] = None - confidence: float - - -# [/DEF:DatabaseMapping:DataClass] - - -# [DEF:DatabaseMappingsResponse:DataClass] -# @COMPLEXITY: 1 -# @PURPOSE: Wrapper for database mappings. -class DatabaseMappingsResponse(BaseModel): - mappings: List[DatabaseMapping] - - -# [/DEF:DatabaseMappingsResponse:DataClass] - - -# [DEF:_find_dashboard_id_by_slug:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve dashboard numeric ID by slug using Superset list endpoint. -# @PRE: `dashboard_slug` is non-empty. -# @POST: Returns dashboard ID when found, otherwise None. -def _find_dashboard_id_by_slug( - client: SupersetClient, - dashboard_slug: str, -) -> Optional[int]: - query_variants = [ - { - "filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}], - "page": 0, - "page_size": 1, - }, - { - "filters": [{"col": "slug", "op": "eq", "value": dashboard_slug}], - "page": 0, - "page_size": 1, - }, - ] - - for query in query_variants: - try: - _count, dashboards = client.get_dashboards_page(query=query) - if dashboards: - resolved_id = dashboards[0].get("id") - if resolved_id is not None: - return int(resolved_id) - except Exception: - continue - - return None - - -# [/DEF:_find_dashboard_id_by_slug:Function] - - -# [DEF:_resolve_dashboard_id_from_ref:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve dashboard ID from slug-first reference with numeric fallback. -# @PRE: `dashboard_ref` is provided in route path. -# @POST: Returns a valid dashboard ID or raises HTTPException(404). -def _resolve_dashboard_id_from_ref( - dashboard_ref: str, - client: SupersetClient, -) -> int: - normalized_ref = str(dashboard_ref or "").strip() - if not normalized_ref: - raise HTTPException(status_code=404, detail="Dashboard not found") - - # Slug-first: even if ref looks numeric, try slug first. - slug_match_id = _find_dashboard_id_by_slug(client, normalized_ref) - if slug_match_id is not None: - return slug_match_id - - if normalized_ref.isdigit(): - return int(normalized_ref) - - raise HTTPException(status_code=404, detail="Dashboard not found") - - -# [/DEF:_resolve_dashboard_id_from_ref:Function] - - -# [DEF:_find_dashboard_id_by_slug_async:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve dashboard numeric ID by slug using async Superset list endpoint. -# @PRE: dashboard_slug is non-empty. -# @POST: Returns dashboard ID when found, otherwise None. -async def _find_dashboard_id_by_slug_async( - client: AsyncSupersetClient, - dashboard_slug: str, -) -> Optional[int]: - query_variants = [ - { - "filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}], - "page": 0, - "page_size": 1, - }, - { - "filters": [{"col": "slug", "op": "eq", "value": dashboard_slug}], - "page": 0, - "page_size": 1, - }, - ] - - for query in query_variants: - try: - _count, dashboards = await client.get_dashboards_page_async(query=query) - if dashboards: - resolved_id = dashboards[0].get("id") - if resolved_id is not None: - return int(resolved_id) - except Exception: - continue - - return None - - -# [/DEF:_find_dashboard_id_by_slug_async:Function] - - -# [DEF:_resolve_dashboard_id_from_ref_async:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve dashboard ID from slug-first reference using async Superset client. -# @PRE: dashboard_ref is provided in route path. -# @POST: Returns valid dashboard ID or raises HTTPException(404). -async def _resolve_dashboard_id_from_ref_async( - dashboard_ref: str, - client: AsyncSupersetClient, -) -> int: - normalized_ref = str(dashboard_ref or "").strip() - if not normalized_ref: - raise HTTPException(status_code=404, detail="Dashboard not found") - - slug_match_id = await _find_dashboard_id_by_slug_async(client, normalized_ref) - if slug_match_id is not None: - return slug_match_id - - if normalized_ref.isdigit(): - return int(normalized_ref) - - raise HTTPException(status_code=404, detail="Dashboard not found") - - -# [/DEF:_resolve_dashboard_id_from_ref_async:Function] - - -# [DEF:_normalize_filter_values:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Normalize query filter values to lower-cased non-empty tokens. -# @PRE: values may be None or list of strings. -# @POST: Returns trimmed normalized list preserving input order. -def _normalize_filter_values(values: Optional[List[str]]) -> List[str]: - if not values: - return [] - normalized: List[str] = [] - for value in values: - token = str(value or "").strip().lower() - if token: - normalized.append(token) - return normalized - - -# [/DEF:_normalize_filter_values:Function] - - -# [DEF:_dashboard_git_filter_value:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Build comparable git status token for dashboards filtering. -# @PRE: dashboard payload may contain git_status or None. -# @POST: Returns one of ok|diff|no_repo|error|pending. -def _dashboard_git_filter_value(dashboard: Dict[str, Any]) -> str: - git_status = dashboard.get("git_status") or {} - sync_status = str(git_status.get("sync_status") or "").strip().upper() - has_repo = git_status.get("has_repo") - if has_repo is False or sync_status == "NO_REPO": - return "no_repo" - if sync_status == "DIFF": - return "diff" - if sync_status == "OK": - return "ok" - if sync_status == "ERROR": - return "error" - return "pending" - - -# [/DEF:_dashboard_git_filter_value:Function] - - -# [DEF:_normalize_actor_alias_token:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Normalize actor alias token to comparable trim+lower text. -# @PRE: value can be scalar/None. -# @POST: Returns normalized token or None. -def _normalize_actor_alias_token(value: Any) -> Optional[str]: - token = str(value or "").strip().lower() - return token or None - - -# [/DEF:_normalize_actor_alias_token:Function] - - -# [DEF:_normalize_owner_display_token:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Project owner payload value into stable display string for API response contracts. -# @PRE: owner can be scalar, dict or None. -# @POST: Returns trimmed non-empty owner display token or None. -def _normalize_owner_display_token(owner: Any) -> Optional[str]: - if owner is None: - return None - - if isinstance(owner, dict): - username = str( - owner.get("username") or owner.get("user_name") or owner.get("name") or "" - ).strip() - full_name = str(owner.get("full_name") or "").strip() - first_name = str(owner.get("first_name") or "").strip() - last_name = str(owner.get("last_name") or "").strip() - combined = " ".join(part for part in [first_name, last_name] if part).strip() - email = str(owner.get("email") or "").strip() - - for candidate in [username, full_name, combined, email]: - if candidate: - return candidate - return None - - normalized = str(owner).strip() - return normalized or None - - -# [/DEF:_normalize_owner_display_token:Function] - - -# [DEF:_normalize_dashboard_owner_values:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Normalize dashboard owners payload to optional list of display strings. -# @PRE: owners payload can be None, scalar, or list with mixed values. -# @POST: Returns deduplicated owner labels preserving order, or None when absent. -def _normalize_dashboard_owner_values(owners: Any) -> Optional[List[str]]: - if owners is None: - return None - - raw_items: List[Any] - if isinstance(owners, list): - raw_items = owners - else: - raw_items = [owners] - - normalized: List[str] = [] - for owner in raw_items: - token = _normalize_owner_display_token(owner) - if token and token not in normalized: - normalized.append(token) - - return normalized - - -# [/DEF:_normalize_dashboard_owner_values:Function] - - -# [DEF:_project_dashboard_response_items:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Project dashboard payloads to response-contract-safe shape. -# @PRE: dashboards is a list of dict-like dashboard payloads. -# @POST: Returned items satisfy DashboardItem owners=list[str]|None contract. -def _project_dashboard_response_items( - dashboards: List[Dict[str, Any]], -) -> List[Dict[str, Any]]: - projected: List[Dict[str, Any]] = [] - for dashboard in dashboards: - projected_dashboard = dict(dashboard) - projected_dashboard["owners"] = _normalize_dashboard_owner_values( - projected_dashboard.get("owners") - ) - projected.append(projected_dashboard) - return projected - - -# [/DEF:_project_dashboard_response_items:Function] - - -# [DEF:_get_profile_filter_binding:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve dashboard profile-filter binding through current or legacy profile service contracts. -# @PRE: profile_service implements get_dashboard_filter_binding or get_my_preference. -# @POST: Returns normalized binding payload with deterministic defaults. -def _get_profile_filter_binding( - profile_service: Any, current_user: User -) -> Dict[str, Any]: - def _read_optional_string(value: Any) -> Optional[str]: - return value if isinstance(value, str) else None - - def _read_bool(value: Any, default: bool) -> bool: - return value if isinstance(value, bool) else default - - if hasattr(profile_service, "get_dashboard_filter_binding"): - binding = profile_service.get_dashboard_filter_binding(current_user) - if isinstance(binding, dict): - return { - "superset_username": _read_optional_string( - binding.get("superset_username") - ), - "superset_username_normalized": _read_optional_string( - binding.get("superset_username_normalized") - ), - "show_only_my_dashboards": _read_bool( - binding.get("show_only_my_dashboards"), False - ), - "show_only_slug_dashboards": _read_bool( - binding.get("show_only_slug_dashboards"), False - ), - } - if hasattr(profile_service, "get_my_preference"): - response = profile_service.get_my_preference(current_user) - preference = getattr(response, "preference", None) - return { - "superset_username": _read_optional_string( - getattr(preference, "superset_username", None) - ), - "superset_username_normalized": _read_optional_string( - getattr(preference, "superset_username_normalized", None) - ), - "show_only_my_dashboards": _read_bool( - getattr(preference, "show_only_my_dashboards", False), False - ), - "show_only_slug_dashboards": _read_bool( - getattr(preference, "show_only_slug_dashboards", False), False - ), - } - return { - "superset_username": None, - "superset_username_normalized": None, - "show_only_my_dashboards": False, - "show_only_slug_dashboards": False, - } - - -# [/DEF:_get_profile_filter_binding:Function] - - -# [DEF:_resolve_profile_actor_aliases:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve stable actor aliases for profile filtering without per-dashboard detail fan-out. -# @PRE: bound username is available and env is valid. -# @POST: Returns at least normalized username; may include Superset display-name alias. -# @SIDE_EFFECT: Performs at most one Superset users-lookup request. -def _resolve_profile_actor_aliases(env: Any, bound_username: str) -> List[str]: - normalized_bound = _normalize_actor_alias_token(bound_username) - if not normalized_bound: - return [] - - aliases: List[str] = [normalized_bound] - try: - client = SupersetClient(env) - adapter = SupersetAccountLookupAdapter( - network_client=client.network, - environment_id=str(getattr(env, "id", "")), - ) - lookup_payload = adapter.get_users_page( - search=normalized_bound, - page_index=0, - page_size=20, - sort_column="username", - sort_order="asc", - ) - lookup_items = ( - lookup_payload.get("items", []) if isinstance(lookup_payload, dict) else [] - ) - - matched_item: Optional[Dict[str, Any]] = None - for item in lookup_items: - if not isinstance(item, dict): - continue - if _normalize_actor_alias_token(item.get("username")) == normalized_bound: - matched_item = item - break - - if matched_item is None: - for item in lookup_items: - if isinstance(item, dict): - matched_item = item - break - - display_alias = _normalize_actor_alias_token( - (matched_item or {}).get("display_name") - ) - if display_alias and display_alias not in aliases: - aliases.append(display_alias) - - logger.reflect( - "[REFLECT] Resolved profile actor aliases " - f"(env={getattr(env, 'id', None)}, bound_username={normalized_bound!r}, " - f"lookup_items={len(lookup_items)}, aliases={aliases!r})" - ) - except Exception as alias_error: - logger.explore( - "[EXPLORE] Failed to resolve profile actor aliases via Superset users lookup " - f"(env={getattr(env, 'id', None)}, bound_username={normalized_bound!r}): {alias_error}" - ) - return aliases - - -# [/DEF:_resolve_profile_actor_aliases:Function] - - -# [DEF:_matches_dashboard_actor_aliases:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Apply profile actor matching against multiple aliases (username + optional display name). -# @PRE: actor_aliases contains normalized non-empty tokens. -# @POST: Returns True when any alias matches owners OR modified_by. -def _matches_dashboard_actor_aliases( - profile_service: ProfileService, - actor_aliases: List[str], - owners: Optional[Any], - modified_by: Optional[str], -) -> bool: - for actor_alias in actor_aliases: - if profile_service.matches_dashboard_actor( - bound_username=actor_alias, - owners=owners, - modified_by=modified_by, - ): - return True - return False - - -# [/DEF:_matches_dashboard_actor_aliases:Function] - - -# [DEF:get_dashboards:Function] -# @COMPLEXITY: 3 -# @PURPOSE: Fetch list of dashboards from a specific environment with Git status and last task status -# @PRE: env_id must be a valid environment ID -# @PRE: page must be >= 1 if provided -# @PRE: page_size must be between 1 and 100 if provided -# @POST: Returns a list of dashboards with enhanced metadata and pagination info -# @POST: Response includes pagination metadata (page, page_size, total, total_pages) -# @POST: Response includes effective profile filter metadata for main dashboards page context -# @PARAM: env_id (str) - The environment ID to fetch dashboards from -# @PARAM: search (Optional[str]) - Filter by title/slug -# @PARAM: page (Optional[int]) - Page number (default: 1) -# @PARAM: page_size (Optional[int]) - Items per page (default: 10, max: 100) -# @RETURN: DashboardsResponse - List of dashboards with status metadata -# @RELATION: CALLS ->[get_dashboards_with_status] -@router.get("", response_model=DashboardsResponse) -async def get_dashboards( - env_id: str, - search: Optional[str] = None, - page: int = 1, - page_size: int = 10, - page_context: Literal["dashboards_main", "other"] = Query( - default="dashboards_main" - ), - apply_profile_default: bool = Query(default=True), - override_show_all: bool = Query(default=False), - filter_title: Optional[List[str]] = Query(default=None), - filter_git_status: Optional[List[str]] = Query(default=None), - filter_llm_status: Optional[List[str]] = Query(default=None), - filter_changed_on: Optional[List[str]] = Query(default=None), - filter_actor: Optional[List[str]] = Query(default=None), - config_manager=Depends(get_config_manager), - task_manager=Depends(get_task_manager), - resource_service=Depends(get_resource_service), - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), - _=Depends(has_permission("plugin:migration", "READ")), -): - with belief_scope( - "get_dashboards", - ( - f"env_id={env_id}, search={search}, page={page}, page_size={page_size}, " - f"page_context={page_context}, apply_profile_default={apply_profile_default}, " - f"override_show_all={override_show_all}" - ), - ): - if page < 1: - logger.error(f"[get_dashboards][Coherence:Failed] Invalid page: {page}") - raise HTTPException(status_code=400, detail="Page must be >= 1") - if page_size < 1 or page_size > 100: - logger.error( - f"[get_dashboards][Coherence:Failed] Invalid page_size: {page_size}" - ) - raise HTTPException( - status_code=400, detail="Page size must be between 1 and 100" - ) - - environments = config_manager.get_environments() - env = next((e for e in environments if e.id == env_id), None) - if not env: - logger.error( - f"[get_dashboards][Coherence:Failed] Environment not found: {env_id}" - ) - raise HTTPException(status_code=404, detail="Environment not found") - - bound_username: Optional[str] = None - can_apply_profile_filter = False - can_apply_slug_filter = False - effective_profile_filter = EffectiveProfileFilter( - applied=False, - source_page=page_context, - override_show_all=bool(override_show_all), - username=None, - match_logic=None, - ) - profile_service: Optional[ProfileService] = None - - try: - profile_service_module = getattr(ProfileService, "__module__", "") - is_mock_db = db.__class__.__module__.startswith("unittest.mock") - use_profile_service = (not is_mock_db) or profile_service_module.startswith( - "unittest.mock" - ) - if use_profile_service: - profile_service = ProfileService(db=db, config_manager=config_manager) - profile_preference = _get_profile_filter_binding( - profile_service, current_user - ) - normalized_username = ( - str(profile_preference.get("superset_username_normalized") or "") - .strip() - .lower() - ) - raw_username = ( - str(profile_preference.get("superset_username") or "") - .strip() - .lower() - ) - bound_username = normalized_username or raw_username or None - - can_apply_profile_filter = ( - page_context == "dashboards_main" - and bool(apply_profile_default) - and not bool(override_show_all) - and bool(profile_preference.get("show_only_my_dashboards", False)) - and bool(bound_username) - ) - can_apply_slug_filter = ( - page_context == "dashboards_main" - and bool(apply_profile_default) - and not bool(override_show_all) - and bool(profile_preference.get("show_only_slug_dashboards", True)) - ) - - profile_match_logic = None - if can_apply_profile_filter and can_apply_slug_filter: - profile_match_logic = "owners_or_modified_by+slug_only" - elif can_apply_profile_filter: - profile_match_logic = "owners_or_modified_by" - elif can_apply_slug_filter: - profile_match_logic = "slug_only" - - effective_profile_filter = EffectiveProfileFilter( - applied=bool(can_apply_profile_filter or can_apply_slug_filter), - source_page=page_context, - override_show_all=bool(override_show_all), - username=bound_username if can_apply_profile_filter else None, - match_logic=profile_match_logic, - ) - except Exception as profile_error: - logger.explore( - f"[EXPLORE] Profile preference unavailable; continuing without profile-default filter: {profile_error}" - ) - - try: - all_tasks = task_manager.get_all_tasks() - title_filters = _normalize_filter_values(filter_title) - git_filters = _normalize_filter_values(filter_git_status) - llm_filters = _normalize_filter_values(filter_llm_status) - changed_on_filters = _normalize_filter_values(filter_changed_on) - actor_filters = _normalize_filter_values(filter_actor) - has_column_filters = any( - ( - title_filters, - git_filters, - llm_filters, - changed_on_filters, - actor_filters, - ) - ) - needs_full_scan = ( - has_column_filters - or bool(can_apply_profile_filter) - or bool(can_apply_slug_filter) - ) - - if isinstance(resource_service, ResourceService) and not needs_full_scan: - try: - page_payload = ( - await resource_service.get_dashboards_page_with_status( - env, - all_tasks, - page=page, - page_size=page_size, - search=search, - include_git_status=False, - require_slug=bool(can_apply_slug_filter), - ) - ) - paginated_dashboards = page_payload["dashboards"] - total = page_payload["total"] - total_pages = page_payload["total_pages"] - except Exception as page_error: - logger.warning( - "[get_dashboards][Action] Page-based fetch failed; using compatibility fallback: %s", - page_error, - ) - if can_apply_slug_filter: - dashboards = await resource_service.get_dashboards_with_status( - env, - all_tasks, - include_git_status=False, - require_slug=True, - ) - else: - dashboards = await resource_service.get_dashboards_with_status( - env, - all_tasks, - include_git_status=False, - ) - - if search: - search_lower = search.lower() - dashboards = [ - d - for d in dashboards - if search_lower in d.get("title", "").lower() - or search_lower in d.get("slug", "").lower() - ] - - total = len(dashboards) - total_pages = ( - (total + page_size - 1) // page_size if total > 0 else 1 - ) - start_idx = (page - 1) * page_size - end_idx = start_idx + page_size - paginated_dashboards = dashboards[start_idx:end_idx] - else: - if can_apply_slug_filter: - dashboards = await resource_service.get_dashboards_with_status( - env, - all_tasks, - include_git_status=bool(git_filters), - require_slug=True, - ) - else: - dashboards = await resource_service.get_dashboards_with_status( - env, - all_tasks, - include_git_status=bool(git_filters), - ) - - if ( - can_apply_profile_filter - and bound_username - and profile_service is not None - ): - actor_aliases = _resolve_profile_actor_aliases(env, bound_username) - if not actor_aliases: - actor_aliases = [bound_username] - logger.reason( - "[REASON] Applying profile actor filter " - f"(env={env_id}, bound_username={bound_username}, actor_aliases={actor_aliases!r}, " - f"dashboards_before={len(dashboards)})" - ) - filtered_dashboards: List[Dict[str, Any]] = [] - max_actor_samples = 15 - for index, dashboard in enumerate(dashboards): - owners_value = dashboard.get("owners") - created_by_value = dashboard.get("created_by") - modified_by_value = dashboard.get("modified_by") - matches_actor = _matches_dashboard_actor_aliases( - profile_service=profile_service, - actor_aliases=actor_aliases, - owners=owners_value, - modified_by=modified_by_value, - ) - if index < max_actor_samples: - logger.reflect( - "[REFLECT] Profile actor filter sample " - f"(env={env_id}, dashboard_id={dashboard.get('id')}, " - f"bound_username={bound_username!r}, actor_aliases={actor_aliases!r}, " - f"owners={owners_value!r}, created_by={created_by_value!r}, " - f"modified_by={modified_by_value!r}, matches={matches_actor})" - ) - if matches_actor: - filtered_dashboards.append(dashboard) - - logger.reflect( - "[REFLECT] Profile actor filter summary " - f"(env={env_id}, bound_username={bound_username!r}, " - f"dashboards_before={len(dashboards)}, dashboards_after={len(filtered_dashboards)})" - ) - dashboards = filtered_dashboards - - if can_apply_slug_filter: - dashboards = [ - dashboard - for dashboard in dashboards - if str(dashboard.get("slug") or "").strip() - ] - - if search: - search_lower = search.lower() - dashboards = [ - d - for d in dashboards - if search_lower in d.get("title", "").lower() - or search_lower in d.get("slug", "").lower() - ] - - def _matches_dashboard_filters(dashboard: Dict[str, Any]) -> bool: - title_value = str(dashboard.get("title") or "").strip().lower() - if title_filters and title_value not in title_filters: - return False - - if git_filters: - git_value = _dashboard_git_filter_value(dashboard) - if git_value not in git_filters: - return False - - llm_value = ( - str( - ( - (dashboard.get("last_task") or {}).get( - "validation_status" - ) - ) - or "UNKNOWN" - ) - .strip() - .lower() - ) - if llm_filters and llm_value not in llm_filters: - return False - - changed_on_raw = ( - str(dashboard.get("last_modified") or "").strip().lower() - ) - changed_on_prefix = ( - changed_on_raw[:10] - if len(changed_on_raw) >= 10 - else changed_on_raw - ) - if ( - changed_on_filters - and changed_on_raw not in changed_on_filters - and changed_on_prefix not in changed_on_filters - ): - return False - - owners = dashboard.get("owners") or [] - if isinstance(owners, list): - actor_value = ", ".join( - str(item).strip() for item in owners if str(item).strip() - ).lower() - else: - actor_value = str(owners).strip().lower() - if not actor_value: - actor_value = "-" - if actor_filters and actor_value not in actor_filters: - return False - return True - - if has_column_filters: - dashboards = [ - d for d in dashboards if _matches_dashboard_filters(d) - ] - - total = len(dashboards) - total_pages = (total + page_size - 1) // page_size if total > 0 else 1 - start_idx = (page - 1) * page_size - end_idx = start_idx + page_size - paginated_dashboards = dashboards[start_idx:end_idx] - - logger.info( - f"[get_dashboards][Coherence:OK] Returning {len(paginated_dashboards)} dashboards " - f"(page {page}/{total_pages}, total: {total}, profile_filter_applied={effective_profile_filter.applied})" - ) - - response_dashboards = _project_dashboard_response_items( - paginated_dashboards - ) - - return DashboardsResponse( - dashboards=response_dashboards, - total=total, - page=page, - page_size=page_size, - total_pages=total_pages, - effective_profile_filter=effective_profile_filter, - ) - - except Exception as e: - logger.error( - f"[get_dashboards][Coherence:Failed] Failed to fetch dashboards: {e}" - ) - raise HTTPException( - status_code=503, detail=f"Failed to fetch dashboards: {str(e)}" - ) - - -# [/DEF:get_dashboards:Function] - - -# [DEF:get_database_mappings:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Get database mapping suggestions between source and target environments -# @PRE: User has permission plugin:migration:read -# @PRE: source_env_id and target_env_id are valid environment IDs -# @POST: Returns list of suggested database mappings with confidence scores -# @PARAM: source_env_id (str) - Source environment ID -# @PARAM: target_env_id (str) - Target environment ID -# @RETURN: DatabaseMappingsResponse - List of suggested mappings -# @RELATION: CALLS ->[MappingService:get_suggestions] -@router.get("/db-mappings", response_model=DatabaseMappingsResponse) -async def get_database_mappings( - source_env_id: str, - target_env_id: str, - config_manager=Depends(get_config_manager), - mapping_service=Depends(get_mapping_service), - _=Depends(has_permission("plugin:migration", "READ")), -): - with belief_scope( - "get_database_mappings", f"source={source_env_id}, target={target_env_id}" - ): - # Validate environments exist - environments = config_manager.get_environments() - source_env = next((e for e in environments if e.id == source_env_id), None) - target_env = next((e for e in environments if e.id == target_env_id), None) - - if not source_env: - logger.error( - f"[get_database_mappings][Coherence:Failed] Source environment not found: {source_env_id}" - ) - raise HTTPException(status_code=404, detail="Source environment not found") - if not target_env: - logger.error( - f"[get_database_mappings][Coherence:Failed] Target environment not found: {target_env_id}" - ) - raise HTTPException(status_code=404, detail="Target environment not found") - - try: - # Get mapping suggestions using MappingService - suggestions = await mapping_service.get_suggestions( - source_env_id, target_env_id - ) - - # Format suggestions as DatabaseMapping objects - mappings = [ - DatabaseMapping( - source_db=s.get("source_db", ""), - target_db=s.get("target_db", ""), - source_db_uuid=s.get("source_db_uuid"), - target_db_uuid=s.get("target_db_uuid"), - confidence=s.get("confidence", 0.0), - ) - for s in suggestions - ] - - logger.info( - f"[get_database_mappings][Coherence:OK] Returning {len(mappings)} database mapping suggestions" - ) - - return DatabaseMappingsResponse(mappings=mappings) - - except Exception as e: - logger.error( - f"[get_database_mappings][Coherence:Failed] Failed to get database mappings: {e}" - ) - raise HTTPException( - status_code=503, detail=f"Failed to get database mappings: {str(e)}" - ) - - -# [/DEF:get_database_mappings:Function] - - -# [DEF:get_dashboard_detail:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Fetch detailed dashboard info with related charts and datasets -# @PRE: env_id must be valid and dashboard ref (slug or id) must exist -# @POST: Returns dashboard detail payload for overview page -# @RELATION: CALLS ->[AsyncSupersetClient] -@router.get("/{dashboard_ref}", response_model=DashboardDetailResponse) -async def get_dashboard_detail( - dashboard_ref: str, - env_id: str, - config_manager=Depends(get_config_manager), - _=Depends(has_permission("plugin:migration", "READ")), -): - with belief_scope( - "get_dashboard_detail", f"dashboard_ref={dashboard_ref}, env_id={env_id}" - ): - environments = config_manager.get_environments() - env = next((e for e in environments if e.id == env_id), None) - if not env: - logger.error( - f"[get_dashboard_detail][Coherence:Failed] Environment not found: {env_id}" - ) - raise HTTPException(status_code=404, detail="Environment not found") - - try: - sync_client = SupersetClient(env) - dashboard_id = _resolve_dashboard_id_from_ref(dashboard_ref, sync_client) - detail = sync_client.get_dashboard_detail(dashboard_id) - logger.info( - f"[get_dashboard_detail][Coherence:OK] Dashboard ref={dashboard_ref} resolved_id={dashboard_id}: {detail.get('chart_count', 0)} charts, {detail.get('dataset_count', 0)} datasets" - ) - return DashboardDetailResponse(**detail) - except HTTPException: - raise - except Exception as e: - logger.error( - f"[get_dashboard_detail][Coherence:Failed] Failed to fetch dashboard detail: {e}" - ) - raise HTTPException( - status_code=503, detail=f"Failed to fetch dashboard detail: {str(e)}" - ) - - -# [/DEF:get_dashboard_detail:Function] - - -# [DEF:_task_matches_dashboard:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Checks whether task params are tied to a specific dashboard and environment. -# @PRE: task-like object exposes plugin_id and params fields. -# @POST: Returns True only for supported task plugins tied to dashboard_id (+optional env_id). -def _task_matches_dashboard( - task: Any, dashboard_id: int, env_id: Optional[str] -) -> bool: - plugin_id = getattr(task, "plugin_id", None) - if plugin_id not in {"superset-backup", "llm_dashboard_validation"}: - return False - - params = getattr(task, "params", {}) or {} - dashboard_id_str = str(dashboard_id) - - if plugin_id == "llm_dashboard_validation": - task_dashboard_id = params.get("dashboard_id") - if str(task_dashboard_id) != dashboard_id_str: - return False - if env_id: - task_env = params.get("environment_id") - return str(task_env) == str(env_id) - return True - - # superset-backup can pass dashboards as "dashboard_ids" or "dashboards" - dashboard_ids = params.get("dashboard_ids") or params.get("dashboards") or [] - normalized_ids = {str(item) for item in dashboard_ids} - if dashboard_id_str not in normalized_ids: - return False - if env_id: - task_env = params.get("environment_id") or params.get("env") - return str(task_env) == str(env_id) - return True - - -# [/DEF:_task_matches_dashboard:Function] - - -# [DEF:get_dashboard_tasks_history:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Returns history of backup and LLM validation tasks for a dashboard. -# @PRE: dashboard ref (slug or id) is valid. -# @POST: Response contains sorted task history (newest first). -@router.get("/{dashboard_ref}/tasks", response_model=DashboardTaskHistoryResponse) -async def get_dashboard_tasks_history( - dashboard_ref: str, - env_id: Optional[str] = None, - limit: int = Query(20, ge=1, le=100), - config_manager=Depends(get_config_manager), - task_manager=Depends(get_task_manager), - _=Depends(has_permission("tasks", "READ")), -): - with belief_scope( - "get_dashboard_tasks_history", - f"dashboard_ref={dashboard_ref}, env_id={env_id}, limit={limit}", - ): - dashboard_id: Optional[int] = None - client: Optional[AsyncSupersetClient] = None - try: - if dashboard_ref.isdigit(): - dashboard_id = int(dashboard_ref) - elif env_id: - environments = config_manager.get_environments() - env = next((e for e in environments if e.id == env_id), None) - if not env: - logger.error( - f"[get_dashboard_tasks_history][Coherence:Failed] Environment not found: {env_id}" - ) - raise HTTPException(status_code=404, detail="Environment not found") - client = AsyncSupersetClient(env) - dashboard_id = await _resolve_dashboard_id_from_ref_async( - dashboard_ref, client - ) - else: - logger.error( - "[get_dashboard_tasks_history][Coherence:Failed] Non-numeric dashboard ref requires env_id" - ) - raise HTTPException( - status_code=400, - detail="env_id is required when dashboard reference is a slug", - ) - - matching_tasks = [] - for task in task_manager.get_all_tasks(): - if _task_matches_dashboard(task, dashboard_id, env_id): - matching_tasks.append(task) - - def _sort_key(task_obj: Any) -> str: - return str(getattr(task_obj, "started_at", "") or "") or str( - getattr(task_obj, "finished_at", "") or "" - ) - - matching_tasks.sort(key=_sort_key, reverse=True) - selected = matching_tasks[:limit] - - items = [] - for task in selected: - result = getattr(task, "result", None) - summary = None - validation_status = None - if isinstance(result, dict): - raw_validation_status = result.get("status") - if raw_validation_status is not None: - validation_status = str(raw_validation_status) - summary = ( - result.get("summary") - or result.get("status") - or result.get("message") - ) - params = getattr(task, "params", {}) or {} - items.append( - DashboardTaskHistoryItem( - id=str(getattr(task, "id", "")), - plugin_id=str(getattr(task, "plugin_id", "")), - status=str(getattr(task, "status", "")), - validation_status=validation_status, - started_at=getattr(task, "started_at", None).isoformat() - if getattr(task, "started_at", None) - else None, - finished_at=getattr(task, "finished_at", None).isoformat() - if getattr(task, "finished_at", None) - else None, - env_id=str(params.get("environment_id") or params.get("env")) - if (params.get("environment_id") or params.get("env")) - else None, - summary=summary, - ) - ) - - logger.info( - f"[get_dashboard_tasks_history][Coherence:OK] Found {len(items)} tasks for dashboard_ref={dashboard_ref}, dashboard_id={dashboard_id}" - ) - return DashboardTaskHistoryResponse(dashboard_id=dashboard_id, items=items) - finally: - if client is not None: - await client.aclose() - - -# [/DEF:get_dashboard_tasks_history:Function] - - -# [DEF:get_dashboard_thumbnail:Function] -# @COMPLEXITY: 3 -# @PURPOSE: Proxies Superset dashboard thumbnail with cache support. -# @RELATION: CALLS ->[AsyncSupersetClient] -# @PRE: env_id must exist. -# @POST: Returns image bytes or 202 when thumbnail is being prepared by Superset. -@router.get("/{dashboard_ref}/thumbnail") -async def get_dashboard_thumbnail( - dashboard_ref: str, - env_id: str, - force: bool = Query(False), - config_manager=Depends(get_config_manager), - _=Depends(has_permission("plugin:migration", "READ")), -): - with belief_scope( - "get_dashboard_thumbnail", - f"dashboard_ref={dashboard_ref}, env_id={env_id}, force={force}", - ): - environments = config_manager.get_environments() - env = next((e for e in environments if e.id == env_id), None) - if not env: - logger.error( - f"[get_dashboard_thumbnail][Coherence:Failed] Environment not found: {env_id}" - ) - raise HTTPException(status_code=404, detail="Environment not found") - - try: - client = SupersetClient(env) - dashboard_id = _resolve_dashboard_id_from_ref(dashboard_ref, client) - digest = None - thumb_endpoint = None - - try: - screenshot_payload = client.network.request( - method="POST", - endpoint=f"/dashboard/{dashboard_id}/cache_dashboard_screenshot/", - json={"force": force}, - ) - payload = ( - screenshot_payload.get("result", screenshot_payload) - if isinstance(screenshot_payload, dict) - else {} - ) - image_url = ( - payload.get("image_url", "") if isinstance(payload, dict) else "" - ) - if isinstance(image_url, str) and image_url: - matched = re.search( - r"/dashboard/\d+/(?:thumbnail|screenshot)/([^/]+)/?$", image_url - ) - if matched: - digest = matched.group(1) - except DashboardNotFoundError: - logger.warning( - "[get_dashboard_thumbnail][Fallback] cache_dashboard_screenshot endpoint unavailable, fallback to dashboard.thumbnail_url" - ) - - if not digest: - dashboard_payload = client.network.request( - method="GET", - endpoint=f"/dashboard/{dashboard_id}", - ) - dashboard_data = ( - dashboard_payload.get("result", dashboard_payload) - if isinstance(dashboard_payload, dict) - else {} - ) - thumbnail_url = ( - dashboard_data.get("thumbnail_url", "") - if isinstance(dashboard_data, dict) - else "" - ) - if isinstance(thumbnail_url, str) and thumbnail_url: - parsed = urlparse(thumbnail_url) - parsed_path = parsed.path or thumbnail_url - if parsed_path.startswith("/api/v1/"): - parsed_path = parsed_path[len("/api/v1") :] - thumb_endpoint = parsed_path - matched = re.search( - r"/dashboard/\d+/(?:thumbnail|screenshot)/([^/]+)/?$", - parsed_path, - ) - if matched: - digest = matched.group(1) - - if not thumb_endpoint: - thumb_endpoint = ( - f"/dashboard/{dashboard_id}/thumbnail/{digest or 'latest'}/" - ) - - thumb_response = client.network.request( - method="GET", - endpoint=thumb_endpoint, - raw_response=True, - allow_redirects=True, - ) - - if thumb_response.status_code == 202: - payload_202: Dict[str, Any] = {} - try: - payload_202 = thumb_response.json() - except Exception: - payload_202 = {"message": "Thumbnail is being generated"} - return JSONResponse(status_code=202, content=payload_202) - - content_type = thumb_response.headers.get("Content-Type", "image/png") - return Response(content=thumb_response.content, media_type=content_type) - except DashboardNotFoundError as e: - logger.error( - f"[get_dashboard_thumbnail][Coherence:Failed] Dashboard not found for thumbnail: {e}" - ) - raise HTTPException(status_code=404, detail="Dashboard thumbnail not found") - except HTTPException: - raise - except Exception as e: - logger.error( - f"[get_dashboard_thumbnail][Coherence:Failed] Failed to fetch dashboard thumbnail: {e}" - ) - raise HTTPException( - status_code=503, detail=f"Failed to fetch dashboard thumbnail: {str(e)}" - ) - - -# [/DEF:get_dashboard_thumbnail:Function] - - -# [DEF:MigrateRequest:DataClass] -# @COMPLEXITY: 1 -# @PURPOSE: DTO for dashboard migration requests. -class MigrateRequest(BaseModel): - source_env_id: str = Field(..., description="Source environment ID") - target_env_id: str = Field(..., description="Target environment ID") - dashboard_ids: List[int] = Field( - ..., description="List of dashboard IDs to migrate" - ) - db_mappings: Optional[Dict[str, str]] = Field( - None, description="Database mappings for migration" - ) - replace_db_config: bool = Field(False, description="Replace database configuration") - - -# [/DEF:MigrateRequest:DataClass] - - -# [DEF:TaskResponse:DataClass] -# @COMPLEXITY: 1 -# @PURPOSE: DTO for async task ID return. -class TaskResponse(BaseModel): - task_id: str - - -# [/DEF:TaskResponse:DataClass] - - -# [DEF:migrate_dashboards:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Trigger bulk migration of dashboards from source to target environment -# @PRE: User has permission plugin:migration:execute -# @PRE: source_env_id and target_env_id are valid environment IDs -# @PRE: dashboard_ids is a non-empty list -# @POST: Returns task_id for tracking migration progress -# @POST: Task is created and queued for execution -# @PARAM: request (MigrateRequest) - Migration request with source, target, and dashboard IDs -# @RETURN: TaskResponse - Task ID for tracking -# @RELATION: DISPATCHES ->[MigrationPlugin:execute] -# @RELATION: CALLS ->[TaskManager] -@router.post("/migrate", response_model=TaskResponse) -async def migrate_dashboards( - request: MigrateRequest, - config_manager=Depends(get_config_manager), - task_manager=Depends(get_task_manager), - _=Depends(has_permission("plugin:migration", "EXECUTE")), -): - with belief_scope( - "migrate_dashboards", - f"source={request.source_env_id}, target={request.target_env_id}, count={len(request.dashboard_ids)}", - ): - # Validate request - if not request.dashboard_ids: - logger.error( - "[migrate_dashboards][Coherence:Failed] No dashboard IDs provided" - ) - raise HTTPException( - status_code=400, detail="At least one dashboard ID must be provided" - ) - - # Validate environments exist - environments = config_manager.get_environments() - source_env = next( - (e for e in environments if e.id == request.source_env_id), None - ) - target_env = next( - (e for e in environments if e.id == request.target_env_id), None - ) - - if not source_env: - logger.error( - f"[migrate_dashboards][Coherence:Failed] Source environment not found: {request.source_env_id}" - ) - raise HTTPException(status_code=404, detail="Source environment not found") - if not target_env: - logger.error( - f"[migrate_dashboards][Coherence:Failed] Target environment not found: {request.target_env_id}" - ) - raise HTTPException(status_code=404, detail="Target environment not found") - - try: - # Create migration task - task_params = { - "source_env_id": request.source_env_id, - "target_env_id": request.target_env_id, - "selected_ids": request.dashboard_ids, - "replace_db_config": request.replace_db_config, - "db_mappings": request.db_mappings or {}, - } - - task_obj = await task_manager.create_task( - plugin_id="superset-migration", params=task_params - ) - - logger.info( - f"[migrate_dashboards][Coherence:OK] Migration task created: {task_obj.id} for {len(request.dashboard_ids)} dashboards" - ) - - return TaskResponse(task_id=str(task_obj.id)) - - except Exception as e: - logger.error( - f"[migrate_dashboards][Coherence:Failed] Failed to create migration task: {e}" - ) - raise HTTPException( - status_code=503, detail=f"Failed to create migration task: {str(e)}" - ) - - -# [/DEF:migrate_dashboards:Function] - - -# [DEF:BackupRequest:DataClass] -# @COMPLEXITY: 1 -# @PURPOSE: DTO for dashboard backup requests. -class BackupRequest(BaseModel): - env_id: str = Field(..., description="Environment ID") - dashboard_ids: List[int] = Field(..., description="List of dashboard IDs to backup") - schedule: Optional[str] = Field( - None, description="Cron schedule for recurring backups (e.g., '0 0 * * *')" - ) - - -# [/DEF:BackupRequest:DataClass] - - -# [DEF:backup_dashboards:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Trigger bulk backup of dashboards with optional cron schedule -# @PRE: User has permission plugin:backup:execute -# @PRE: env_id is a valid environment ID -# @PRE: dashboard_ids is a non-empty list -# @POST: Returns task_id for tracking backup progress -# @POST: Task is created and queued for execution -# @POST: If schedule is provided, a scheduled task is created -# @PARAM: request (BackupRequest) - Backup request with environment and dashboard IDs -# @RETURN: TaskResponse - Task ID for tracking -# @RELATION: DISPATCHES ->[BackupPlugin:execute] -# @RELATION: CALLS ->[TaskManager] -@router.post("/backup", response_model=TaskResponse) -async def backup_dashboards( - request: BackupRequest, - config_manager=Depends(get_config_manager), - task_manager=Depends(get_task_manager), - _=Depends(has_permission("plugin:backup", "EXECUTE")), -): - with belief_scope( - "backup_dashboards", - f"env={request.env_id}, count={len(request.dashboard_ids)}, schedule={request.schedule}", - ): - # Validate request - if not request.dashboard_ids: - logger.error( - "[backup_dashboards][Coherence:Failed] No dashboard IDs provided" - ) - raise HTTPException( - status_code=400, detail="At least one dashboard ID must be provided" - ) - - # Validate environment exists - environments = config_manager.get_environments() - env = next((e for e in environments if e.id == request.env_id), None) - - if not env: - logger.error( - f"[backup_dashboards][Coherence:Failed] Environment not found: {request.env_id}" - ) - raise HTTPException(status_code=404, detail="Environment not found") - - try: - # Create backup task - task_params = { - "env": request.env_id, - "dashboards": request.dashboard_ids, - "schedule": request.schedule, - } - - task_obj = await task_manager.create_task( - plugin_id="superset-backup", params=task_params - ) - - logger.info( - f"[backup_dashboards][Coherence:OK] Backup task created: {task_obj.id} for {len(request.dashboard_ids)} dashboards" - ) - - return TaskResponse(task_id=str(task_obj.id)) - - except Exception as e: - logger.error( - f"[backup_dashboards][Coherence:Failed] Failed to create backup task: {e}" - ) - raise HTTPException( - status_code=503, detail=f"Failed to create backup task: {str(e)}" - ) - - -# [/DEF:backup_dashboards:Function] - -# [/DEF:DashboardsApi:Module] diff --git a/backend/src/api/routes/dashboards/__init__.py b/backend/src/api/routes/dashboards/__init__.py new file mode 100644 index 00000000..34536e07 --- /dev/null +++ b/backend/src/api/routes/dashboards/__init__.py @@ -0,0 +1,44 @@ +# [DEF:DashboardsApi:Module] +# +# @COMPLEXITY: 5 +# @SEMANTICS: api, dashboards, resources, hub +# @PURPOSE: API endpoints for the Dashboard Hub - listing dashboards with Git and task status +# @LAYER: API +# @RELATION: DEPENDS_ON ->[AppDependencies] +# @RELATION: DEPENDS_ON ->[ResourceService] +# @RELATION: DEPENDS_ON ->[SupersetClient] +# +# @INVARIANT: All dashboard responses include git_status and last_task metadata +# +# @PRE: Valid environment configurations exist in ConfigManager. +# @POST: Dashboard responses are projected into DashboardsResponse DTO. +# @SIDE_EFFECT: Performs external calls to Superset API and potentially Git providers. +# @DATA_CONTRACT: Input(env_id, filters) -> Output(DashboardsResponse) +# +# @TEST_CONTRACT: DashboardsAPI -> { +# required_fields: {env_id: string, page: integer, page_size: integer}, +# optional_fields: {search: string}, +# invariants: ["Pagination must be valid", "Environment must exist"] +# } +# +# @TEST_FIXTURE: dashboard_list_happy -> { +# "env_id": "prod", +# "expected_count": 1, +# "dashboards": [{"id": 1, "title": "Main Revenue"}] +# } +# +# @TEST_EDGE: pagination_zero_page -> {"env_id": "prod", "page": 0, "status": 400} +# @TEST_EDGE: pagination_oversize -> {"env_id": "prod", "page_size": 101, "status": 400} +# @TEST_EDGE: missing_env -> {"env_id": "ghost", "status": 404} +# @TEST_EDGE: empty_dashboards -> {"env_id": "empty_env", "expected_total": 0} +# @TEST_EDGE: external_superset_failure -> {"env_id": "bad_conn", "status": 503} +# +# @TEST_INVARIANT: metadata_consistency -> verifies: [dashboard_list_happy, empty_dashboards] + +from ._router import router +from ._schemas import * +from ._listing_routes import * +from ._detail_routes import * +from ._action_routes import * + +# [/DEF:DashboardsApi:Module] diff --git a/backend/src/api/routes/dashboards/_action_routes.py b/backend/src/api/routes/dashboards/_action_routes.py new file mode 100644 index 00000000..d824ed91 --- /dev/null +++ b/backend/src/api/routes/dashboards/_action_routes.py @@ -0,0 +1,176 @@ +# [DEF:DashboardActionRoutes:Module] +# @COMPLEXITY: 2 +# @SEMANTICS: api, dashboards, actions, routes +# @PURPOSE: Dashboard action route handlers — migrate, backup. +# @LAYER: API +# @RELATION: DEPENDS_ON -> [DashboardRouter] +# @RELATION: DEPENDS_ON -> [DashboardSchemas] + +# [SECTION: IMPORTS] +from fastapi import Depends, HTTPException + +from src.dependencies import ( + get_config_manager, + get_task_manager, + has_permission, +) +from src.core.logger import logger, belief_scope +from ._schemas import ( + MigrateRequest, + BackupRequest, + TaskResponse, +) +from ._router import router +# [/SECTION] + + +# [DEF:migrate_dashboards:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Trigger bulk migration of dashboards from source to target environment +# @PRE: User has permission plugin:migration:execute +# @PRE: source_env_id and target_env_id are valid environment IDs +# @PRE: dashboard_ids is a non-empty list +# @POST: Returns task_id for tracking migration progress +# @POST: Task is created and queued for execution +# @PARAM: request (MigrateRequest) - Migration request with source, target, and dashboard IDs +# @RETURN: TaskResponse - Task ID for tracking +# @RELATION: DISPATCHES ->[MigrationPlugin:execute] +# @RELATION: CALLS ->[TaskManager] +@router.post("/migrate", response_model=TaskResponse) +async def migrate_dashboards( + request: MigrateRequest, + config_manager=Depends(get_config_manager), + task_manager=Depends(get_task_manager), + _=Depends(has_permission("plugin:migration", "EXECUTE")), +): + with belief_scope( + "migrate_dashboards", + f"source={request.source_env_id}, target={request.target_env_id}, count={len(request.dashboard_ids)}", + ): + if not request.dashboard_ids: + logger.error( + "[migrate_dashboards][Coherence:Failed] No dashboard IDs provided" + ) + raise HTTPException( + status_code=400, detail="At least one dashboard ID must be provided" + ) + + environments = config_manager.get_environments() + source_env = next( + (e for e in environments if e.id == request.source_env_id), None + ) + target_env = next( + (e for e in environments if e.id == request.target_env_id), None + ) + + if not source_env: + logger.error( + f"[migrate_dashboards][Coherence:Failed] Source environment not found: {request.source_env_id}" + ) + raise HTTPException(status_code=404, detail="Source environment not found") + if not target_env: + logger.error( + f"[migrate_dashboards][Coherence:Failed] Target environment not found: {request.target_env_id}" + ) + raise HTTPException(status_code=404, detail="Target environment not found") + + try: + task_params = { + "source_env_id": request.source_env_id, + "target_env_id": request.target_env_id, + "selected_ids": request.dashboard_ids, + "replace_db_config": request.replace_db_config, + "db_mappings": request.db_mappings or {}, + } + + task_obj = await task_manager.create_task( + plugin_id="superset-migration", params=task_params + ) + + logger.info( + f"[migrate_dashboards][Coherence:OK] Migration task created: {task_obj.id} for {len(request.dashboard_ids)} dashboards" + ) + + return TaskResponse(task_id=str(task_obj.id)) + + except Exception as e: + logger.error( + f"[migrate_dashboards][Coherence:Failed] Failed to create migration task: {e}" + ) + raise HTTPException( + status_code=503, detail=f"Failed to create migration task: {str(e)}" + ) + + +# [/DEF:migrate_dashboards:Function] + + +# [DEF:backup_dashboards:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Trigger bulk backup of dashboards with optional cron schedule +# @PRE: User has permission plugin:backup:execute +# @PRE: env_id is a valid environment ID +# @PRE: dashboard_ids is a non-empty list +# @POST: Returns task_id for tracking backup progress +# @POST: Task is created and queued for execution +# @POST: If schedule is provided, a scheduled task is created +# @PARAM: request (BackupRequest) - Backup request with environment and dashboard IDs +# @RETURN: TaskResponse - Task ID for tracking +# @RELATION: DISPATCHES ->[BackupPlugin:execute] +# @RELATION: CALLS ->[TaskManager] +@router.post("/backup", response_model=TaskResponse) +async def backup_dashboards( + request: BackupRequest, + config_manager=Depends(get_config_manager), + task_manager=Depends(get_task_manager), + _=Depends(has_permission("plugin:backup", "EXECUTE")), +): + with belief_scope( + "backup_dashboards", + f"env={request.env_id}, count={len(request.dashboard_ids)}, schedule={request.schedule}", + ): + if not request.dashboard_ids: + logger.error( + "[backup_dashboards][Coherence:Failed] No dashboard IDs provided" + ) + raise HTTPException( + status_code=400, detail="At least one dashboard ID must be provided" + ) + + environments = config_manager.get_environments() + env = next((e for e in environments if e.id == request.env_id), None) + + if not env: + logger.error( + f"[backup_dashboards][Coherence:Failed] Environment not found: {request.env_id}" + ) + raise HTTPException(status_code=404, detail="Environment not found") + + try: + task_params = { + "env": request.env_id, + "dashboards": request.dashboard_ids, + "schedule": request.schedule, + } + + task_obj = await task_manager.create_task( + plugin_id="superset-backup", params=task_params + ) + + logger.info( + f"[backup_dashboards][Coherence:OK] Backup task created: {task_obj.id} for {len(request.dashboard_ids)} dashboards" + ) + + return TaskResponse(task_id=str(task_obj.id)) + + except Exception as e: + logger.error( + f"[backup_dashboards][Coherence:Failed] Failed to create backup task: {e}" + ) + raise HTTPException( + status_code=503, detail=f"Failed to create backup task: {str(e)}" + ) + + +# [/DEF:backup_dashboards:Function] +# [/DEF:DashboardActionRoutes:Module] diff --git a/backend/src/api/routes/dashboards/_detail_routes.py b/backend/src/api/routes/dashboards/_detail_routes.py new file mode 100644 index 00000000..71c689d4 --- /dev/null +++ b/backend/src/api/routes/dashboards/_detail_routes.py @@ -0,0 +1,392 @@ +# [DEF:DashboardDetailRoutes:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: api, dashboards, detail, routes +# @PURPOSE: Dashboard detail, db-mappings, task history, thumbnail route handlers. +# @LAYER: API +# @RELATION: DEPENDS_ON -> [DashboardRouter] +# @RELATION: DEPENDS_ON -> [DashboardSchemas] +# @RELATION: DEPENDS_ON -> [DashboardHelpers] +# @RELATION: DEPENDS_ON -> [DashboardProjection] + +# [SECTION: IMPORTS] +from typing import Optional, List, Dict, Any +import re +from urllib.parse import urlparse + +from fastapi import Depends, HTTPException, Query, Response +from fastapi.responses import JSONResponse + +from src.dependencies import ( + get_config_manager, + get_task_manager, + get_mapping_service, + has_permission, +) +from src.core.async_superset_client import AsyncSupersetClient +from src.core.superset_client import SupersetClient +from src.core.logger import logger, belief_scope +from src.core.utils.network import DashboardNotFoundError +from ._helpers import ( + _resolve_dashboard_id_from_ref, + _resolve_dashboard_id_from_ref_async, +) +from ._projection import ( + _task_matches_dashboard, +) +from ._schemas import ( + DashboardDetailResponse, + DashboardTaskHistoryItem, + DashboardTaskHistoryResponse, + DatabaseMapping, + DatabaseMappingsResponse, +) +from ._router import router +# [/SECTION] + + +# [DEF:get_database_mappings:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Get database mapping suggestions between source and target environments +# @PRE: User has permission plugin:migration:read +# @PRE: source_env_id and target_env_id are valid environment IDs +# @POST: Returns list of suggested database mappings with confidence scores +# @PARAM: source_env_id (str) - Source environment ID +# @PARAM: target_env_id (str) - Target environment ID +# @RETURN: DatabaseMappingsResponse - List of suggested mappings +# @RELATION: CALLS ->[MappingService:get_suggestions] +@router.get("/db-mappings", response_model=DatabaseMappingsResponse) +async def get_database_mappings( + source_env_id: str, + target_env_id: str, + config_manager=Depends(get_config_manager), + mapping_service=Depends(get_mapping_service), + _=Depends(has_permission("plugin:migration", "READ")), +): + with belief_scope( + "get_database_mappings", f"source={source_env_id}, target={target_env_id}" + ): + environments = config_manager.get_environments() + source_env = next((e for e in environments if e.id == source_env_id), None) + target_env = next((e for e in environments if e.id == target_env_id), None) + + if not source_env: + logger.error( + f"[get_database_mappings][Coherence:Failed] Source environment not found: {source_env_id}" + ) + raise HTTPException(status_code=404, detail="Source environment not found") + if not target_env: + logger.error( + f"[get_database_mappings][Coherence:Failed] Target environment not found: {target_env_id}" + ) + raise HTTPException(status_code=404, detail="Target environment not found") + + try: + suggestions = await mapping_service.get_suggestions( + source_env_id, target_env_id + ) + + mappings = [ + DatabaseMapping( + source_db=s.get("source_db", ""), + target_db=s.get("target_db", ""), + source_db_uuid=s.get("source_db_uuid"), + target_db_uuid=s.get("target_db_uuid"), + confidence=s.get("confidence", 0.0), + ) + for s in suggestions + ] + + logger.info( + f"[get_database_mappings][Coherence:OK] Returning {len(mappings)} database mapping suggestions" + ) + + return DatabaseMappingsResponse(mappings=mappings) + + except Exception as e: + logger.error( + f"[get_database_mappings][Coherence:Failed] Failed to get database mappings: {e}" + ) + raise HTTPException( + status_code=503, detail=f"Failed to get database mappings: {str(e)}" + ) + + +# [/DEF:get_database_mappings:Function] + + +# [DEF:get_dashboard_detail:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Fetch detailed dashboard info with related charts and datasets +# @PRE: env_id must be valid and dashboard ref (slug or id) must exist +# @POST: Returns dashboard detail payload for overview page +# @RELATION: CALLS ->[AsyncSupersetClient] +@router.get("/{dashboard_ref}", response_model=DashboardDetailResponse) +async def get_dashboard_detail( + dashboard_ref: str, + env_id: str, + config_manager=Depends(get_config_manager), + _=Depends(has_permission("plugin:migration", "READ")), +): + with belief_scope( + "get_dashboard_detail", f"dashboard_ref={dashboard_ref}, env_id={env_id}" + ): + environments = config_manager.get_environments() + env = next((e for e in environments if e.id == env_id), None) + if not env: + logger.error( + f"[get_dashboard_detail][Coherence:Failed] Environment not found: {env_id}" + ) + raise HTTPException(status_code=404, detail="Environment not found") + + try: + sync_client = SupersetClient(env) + dashboard_id = _resolve_dashboard_id_from_ref(dashboard_ref, sync_client) + detail = sync_client.get_dashboard_detail(dashboard_id) + logger.info( + f"[get_dashboard_detail][Coherence:OK] Dashboard ref={dashboard_ref} resolved_id={dashboard_id}: {detail.get('chart_count', 0)} charts, {detail.get('dataset_count', 0)} datasets" + ) + return DashboardDetailResponse(**detail) + except HTTPException: + raise + except Exception as e: + logger.error( + f"[get_dashboard_detail][Coherence:Failed] Failed to fetch dashboard detail: {e}" + ) + raise HTTPException( + status_code=503, detail=f"Failed to fetch dashboard detail: {str(e)}" + ) + + +# [/DEF:get_dashboard_detail:Function] + + +# [DEF:get_dashboard_tasks_history:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Returns history of backup and LLM validation tasks for a dashboard. +# @PRE: dashboard ref (slug or id) is valid. +# @POST: Response contains sorted task history (newest first). +@router.get("/{dashboard_ref}/tasks", response_model=DashboardTaskHistoryResponse) +async def get_dashboard_tasks_history( + dashboard_ref: str, + env_id: Optional[str] = None, + limit: int = Query(20, ge=1, le=100), + config_manager=Depends(get_config_manager), + task_manager=Depends(get_task_manager), + _=Depends(has_permission("tasks", "READ")), +): + with belief_scope( + "get_dashboard_tasks_history", + f"dashboard_ref={dashboard_ref}, env_id={env_id}, limit={limit}", + ): + dashboard_id: Optional[int] = None + client: Optional[AsyncSupersetClient] = None + try: + if dashboard_ref.isdigit(): + dashboard_id = int(dashboard_ref) + elif env_id: + environments = config_manager.get_environments() + env = next((e for e in environments if e.id == env_id), None) + if not env: + logger.error( + f"[get_dashboard_tasks_history][Coherence:Failed] Environment not found: {env_id}" + ) + raise HTTPException(status_code=404, detail="Environment not found") + client = AsyncSupersetClient(env) + dashboard_id = await _resolve_dashboard_id_from_ref_async( + dashboard_ref, client + ) + else: + logger.error( + "[get_dashboard_tasks_history][Coherence:Failed] Non-numeric dashboard ref requires env_id" + ) + raise HTTPException( + status_code=400, + detail="env_id is required when dashboard reference is a slug", + ) + + matching_tasks = [] + for task in task_manager.get_all_tasks(): + if _task_matches_dashboard(task, dashboard_id, env_id): + matching_tasks.append(task) + + def _sort_key(task_obj: Any) -> str: + return str(getattr(task_obj, "started_at", "") or "") or str( + getattr(task_obj, "finished_at", "") or "" + ) + + matching_tasks.sort(key=_sort_key, reverse=True) + selected = matching_tasks[:limit] + + items = [] + for task in selected: + result = getattr(task, "result", None) + summary = None + validation_status = None + if isinstance(result, dict): + raw_validation_status = result.get("status") + if raw_validation_status is not None: + validation_status = str(raw_validation_status) + summary = ( + result.get("summary") + or result.get("status") + or result.get("message") + ) + params = getattr(task, "params", {}) or {} + items.append( + DashboardTaskHistoryItem( + id=str(getattr(task, "id", "")), + plugin_id=str(getattr(task, "plugin_id", "")), + status=str(getattr(task, "status", "")), + validation_status=validation_status, + started_at=getattr(task, "started_at", None).isoformat() + if getattr(task, "started_at", None) + else None, + finished_at=getattr(task, "finished_at", None).isoformat() + if getattr(task, "finished_at", None) + else None, + env_id=str(params.get("environment_id") or params.get("env")) + if (params.get("environment_id") or params.get("env")) + else None, + summary=summary, + ) + ) + + logger.info( + f"[get_dashboard_tasks_history][Coherence:OK] Found {len(items)} tasks for dashboard_ref={dashboard_ref}, dashboard_id={dashboard_id}" + ) + return DashboardTaskHistoryResponse(dashboard_id=dashboard_id, items=items) + finally: + if client is not None: + await client.aclose() + + +# [/DEF:get_dashboard_tasks_history:Function] + + +# [DEF:get_dashboard_thumbnail:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Proxies Superset dashboard thumbnail with cache support. +# @RELATION: CALLS ->[AsyncSupersetClient] +# @PRE: env_id must exist. +# @POST: Returns image bytes or 202 when thumbnail is being prepared by Superset. +@router.get("/{dashboard_ref}/thumbnail") +async def get_dashboard_thumbnail( + dashboard_ref: str, + env_id: str, + force: bool = Query(False), + config_manager=Depends(get_config_manager), + _=Depends(has_permission("plugin:migration", "READ")), +): + with belief_scope( + "get_dashboard_thumbnail", + f"dashboard_ref={dashboard_ref}, env_id={env_id}, force={force}", + ): + environments = config_manager.get_environments() + env = next((e for e in environments if e.id == env_id), None) + if not env: + logger.error( + f"[get_dashboard_thumbnail][Coherence:Failed] Environment not found: {env_id}" + ) + raise HTTPException(status_code=404, detail="Environment not found") + + try: + client = SupersetClient(env) + dashboard_id = _resolve_dashboard_id_from_ref(dashboard_ref, client) + digest = None + thumb_endpoint = None + + try: + screenshot_payload = client.network.request( + method="POST", + endpoint=f"/dashboard/{dashboard_id}/cache_dashboard_screenshot/", + json={"force": force}, + ) + payload = ( + screenshot_payload.get("result", screenshot_payload) + if isinstance(screenshot_payload, dict) + else {} + ) + image_url = ( + payload.get("image_url", "") if isinstance(payload, dict) else "" + ) + if isinstance(image_url, str) and image_url: + matched = re.search( + r"/dashboard/\d+/(?:thumbnail|screenshot)/([^/]+)/?$", + image_url, + ) + if matched: + digest = matched.group(1) + except DashboardNotFoundError: + logger.warning( + "[get_dashboard_thumbnail][Fallback] cache_dashboard_screenshot endpoint unavailable, fallback to dashboard.thumbnail_url" + ) + + if not digest: + dashboard_payload = client.network.request( + method="GET", + endpoint=f"/dashboard/{dashboard_id}", + ) + dashboard_data = ( + dashboard_payload.get("result", dashboard_payload) + if isinstance(dashboard_payload, dict) + else {} + ) + thumbnail_url = ( + dashboard_data.get("thumbnail_url", "") + if isinstance(dashboard_data, dict) + else "" + ) + if isinstance(thumbnail_url, str) and thumbnail_url: + parsed = urlparse(thumbnail_url) + parsed_path = parsed.path or thumbnail_url + if parsed_path.startswith("/api/v1/"): + parsed_path = parsed_path[len("/api/v1") :] + thumb_endpoint = parsed_path + matched = re.search( + r"/dashboard/\d+/(?:thumbnail|screenshot)/([^/]+)/?$", + parsed_path, + ) + if matched: + digest = matched.group(1) + + if not thumb_endpoint: + thumb_endpoint = ( + f"/dashboard/{dashboard_id}/thumbnail/{digest or 'latest'}/" + ) + + thumb_response = client.network.request( + method="GET", + endpoint=thumb_endpoint, + raw_response=True, + allow_redirects=True, + ) + + if thumb_response.status_code == 202: + payload_202: Dict[str, Any] = {} + try: + payload_202 = thumb_response.json() + except Exception: + payload_202 = {"message": "Thumbnail is being generated"} + return JSONResponse(status_code=202, content=payload_202) + + content_type = thumb_response.headers.get("Content-Type", "image/png") + return Response(content=thumb_response.content, media_type=content_type) + except DashboardNotFoundError as e: + logger.error( + f"[get_dashboard_thumbnail][Coherence:Failed] Dashboard not found for thumbnail: {e}" + ) + raise HTTPException(status_code=404, detail="Dashboard thumbnail not found") + except HTTPException: + raise + except Exception as e: + logger.error( + f"[get_dashboard_thumbnail][Coherence:Failed] Failed to fetch dashboard thumbnail: {e}" + ) + raise HTTPException( + status_code=503, + detail=f"Failed to fetch dashboard thumbnail: {str(e)}", + ) + + +# [/DEF:get_dashboard_thumbnail:Function] +# [/DEF:DashboardDetailRoutes:Module] diff --git a/backend/src/api/routes/dashboards/_helpers.py b/backend/src/api/routes/dashboards/_helpers.py new file mode 100644 index 00000000..7a1cbf5c --- /dev/null +++ b/backend/src/api/routes/dashboards/_helpers.py @@ -0,0 +1,187 @@ +# [DEF:DashboardHelpers:Module] +# @COMPLEXITY: 2 +# @SEMANTICS: api, dashboards, helpers, resolution +# @PURPOSE: Basic helper functions for dashboard route handlers — slug resolution, filter normalization. +# @LAYER: Infra +# @RELATION: DEPENDS_ON -> [SupersetClient] +# @RELATION: DEPENDS_ON -> [AsyncSupersetClient] + +from typing import List, Optional, Dict, Any +from fastapi import HTTPException +from src.core.superset_client import SupersetClient +from src.core.async_superset_client import AsyncSupersetClient +from src.core.logger import logger + + +# [DEF:_find_dashboard_id_by_slug:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Resolve dashboard numeric ID by slug using Superset list endpoint. +# @PRE: `dashboard_slug` is non-empty. +# @POST: Returns dashboard ID when found, otherwise None. +def _find_dashboard_id_by_slug( + client: SupersetClient, + dashboard_slug: str, +) -> Optional[int]: + query_variants = [ + { + "filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}], + "page": 0, + "page_size": 1, + }, + { + "filters": [{"col": "slug", "op": "eq", "value": dashboard_slug}], + "page": 0, + "page_size": 1, + }, + ] + + for query in query_variants: + try: + _count, dashboards = client.get_dashboards_page(query=query) + if dashboards: + resolved_id = dashboards[0].get("id") + if resolved_id is not None: + return int(resolved_id) + except Exception: + continue + + return None + + +# [/DEF:_find_dashboard_id_by_slug:Function] + + +# [DEF:_resolve_dashboard_id_from_ref:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Resolve dashboard ID from slug-first reference with numeric fallback. +# @PRE: `dashboard_ref` is provided in route path. +# @POST: Returns a valid dashboard ID or raises HTTPException(404). +def _resolve_dashboard_id_from_ref( + dashboard_ref: str, + client: SupersetClient, +) -> int: + normalized_ref = str(dashboard_ref or "").strip() + if not normalized_ref: + raise HTTPException(status_code=404, detail="Dashboard not found") + + # Slug-first: even if ref looks numeric, try slug first. + slug_match_id = _find_dashboard_id_by_slug(client, normalized_ref) + if slug_match_id is not None: + return slug_match_id + + if normalized_ref.isdigit(): + return int(normalized_ref) + + raise HTTPException(status_code=404, detail="Dashboard not found") + + +# [/DEF:_resolve_dashboard_id_from_ref:Function] + + +# [DEF:_find_dashboard_id_by_slug_async:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Resolve dashboard numeric ID by slug using async Superset list endpoint. +# @PRE: dashboard_slug is non-empty. +# @POST: Returns dashboard ID when found, otherwise None. +async def _find_dashboard_id_by_slug_async( + client: AsyncSupersetClient, + dashboard_slug: str, +) -> Optional[int]: + query_variants = [ + { + "filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}], + "page": 0, + "page_size": 1, + }, + { + "filters": [{"col": "slug", "op": "eq", "value": dashboard_slug}], + "page": 0, + "page_size": 1, + }, + ] + + for query in query_variants: + try: + _count, dashboards = await client.get_dashboards_page_async(query=query) + if dashboards: + resolved_id = dashboards[0].get("id") + if resolved_id is not None: + return int(resolved_id) + except Exception: + continue + + return None + + +# [/DEF:_find_dashboard_id_by_slug_async:Function] + + +# [DEF:_resolve_dashboard_id_from_ref_async:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Resolve dashboard ID from slug-first reference using async Superset client. +# @PRE: dashboard_ref is provided in route path. +# @POST: Returns valid dashboard ID or raises HTTPException(404). +async def _resolve_dashboard_id_from_ref_async( + dashboard_ref: str, + client: AsyncSupersetClient, +) -> int: + normalized_ref = str(dashboard_ref or "").strip() + if not normalized_ref: + raise HTTPException(status_code=404, detail="Dashboard not found") + + slug_match_id = await _find_dashboard_id_by_slug_async(client, normalized_ref) + if slug_match_id is not None: + return slug_match_id + + if normalized_ref.isdigit(): + return int(normalized_ref) + + raise HTTPException(status_code=404, detail="Dashboard not found") + + +# [/DEF:_resolve_dashboard_id_from_ref_async:Function] + + +# [DEF:_normalize_filter_values:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Normalize query filter values to lower-cased non-empty tokens. +# @PRE: values may be None or list of strings. +# @POST: Returns trimmed normalized list preserving input order. +def _normalize_filter_values(values: Optional[List[str]]) -> List[str]: + if not values: + return [] + normalized: List[str] = [] + for value in values: + token = str(value or "").strip().lower() + if token: + normalized.append(token) + return normalized + + +# [/DEF:_normalize_filter_values:Function] + + +# [DEF:_dashboard_git_filter_value:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Build comparable git status token for dashboards filtering. +# @PRE: dashboard payload may contain git_status or None. +# @POST: Returns one of ok|diff|no_repo|error|pending. +def _dashboard_git_filter_value(dashboard: Dict[str, Any]) -> str: + git_status = dashboard.get("git_status") or {} + sync_status = str(git_status.get("sync_status") or "").strip().upper() + has_repo = git_status.get("has_repo") + if has_repo is False or sync_status == "NO_REPO": + return "no_repo" + if sync_status == "DIFF": + return "diff" + if sync_status == "OK": + return "ok" + if sync_status == "ERROR": + return "error" + return "pending" + + +# [/DEF:_dashboard_git_filter_value:Function] + + +# [/DEF:DashboardHelpers:Module] diff --git a/backend/src/api/routes/dashboards/_listing_routes.py b/backend/src/api/routes/dashboards/_listing_routes.py new file mode 100644 index 00000000..a89478e4 --- /dev/null +++ b/backend/src/api/routes/dashboards/_listing_routes.py @@ -0,0 +1,398 @@ +# [DEF:DashboardListingRoutes:Module] +# @COMPLEXITY: 4 +# @SEMANTICS: api, dashboards, listing, routes +# @PURPOSE: Dashboard listing route handler for Dashboard Hub. +# @LAYER: API +# @RELATION: DEPENDS_ON -> [DashboardRouter] +# @RELATION: DEPENDS_ON -> [DashboardSchemas] +# @RELATION: DEPENDS_ON -> [DashboardHelpers] +# @RELATION: DEPENDS_ON -> [DashboardProjection] + +# [SECTION: IMPORTS] +import os +from typing import List, Optional, Dict, Any, Literal +from fastapi import Depends, HTTPException, Query +from sqlalchemy.orm import Session +from src.dependencies import ( + get_config_manager, get_task_manager, get_resource_service, + get_current_user, has_permission, +) +from src.core.database import get_db +from src.core.logger import logger, belief_scope +from src.models.auth import User +from src.services.profile_service import ProfileService +from ._helpers import _normalize_filter_values, _dashboard_git_filter_value +from ._projection import ( + _project_dashboard_response_items, _get_profile_filter_binding, + _resolve_profile_actor_aliases, _matches_dashboard_actor_aliases, +) +from ._schemas import EffectiveProfileFilter, DashboardsResponse +from ._router import router +# [/SECTION] + + +# [DEF:get_dashboards:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Fetch list of dashboards from a specific environment with Git status and last task status +# @PRE: env_id must be a valid environment ID +# @PRE: page must be >= 1 if provided +# @PRE: page_size must be between 1 and 100 if provided +# @POST: Returns a list of dashboards with enhanced metadata and pagination info +# @POST: Response includes pagination metadata (page, page_size, total, total_pages) +# @POST: Response includes effective profile filter metadata for main dashboards page context +# @PARAM: env_id (str) - The environment ID to fetch dashboards from +# @PARAM: search (Optional[str]) - Filter by title/slug +# @PARAM: page (Optional[int]) - Page number (default: 1) +# @PARAM: page_size (Optional[int]) - Items per page (default: 10, max: 100) +# @RETURN: DashboardsResponse - List of dashboards with status metadata +# @RELATION: CALLS ->[get_dashboards_with_status] +@router.get("", response_model=DashboardsResponse) +async def get_dashboards( + env_id: str, + search: Optional[str] = None, + page: int = 1, + page_size: int = 10, + page_context: Literal["dashboards_main", "other"] = Query("dashboards_main"), + apply_profile_default: bool = Query(default=True), + override_show_all: bool = Query(default=False), + filter_title: Optional[List[str]] = Query(default=None), + filter_git_status: Optional[List[str]] = Query(default=None), + filter_llm_status: Optional[List[str]] = Query(default=None), + filter_changed_on: Optional[List[str]] = Query(default=None), + filter_actor: Optional[List[str]] = Query(default=None), + config_manager=Depends(get_config_manager), + task_manager=Depends(get_task_manager), + resource_service=Depends(get_resource_service), + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), + _=Depends(has_permission("plugin:migration", "READ")), +): + with belief_scope( + "get_dashboards", + f"env_id={env_id}, search={search}, page={page}, page_size={page_size}, " + f"page_context={page_context}, applyp={apply_profile_default}, overhide={override_show_all}", + ): + if page < 1: + logger.error(f"[get_dashboards][Coherence:Failed] Invalid page: {page}") + raise HTTPException(status_code=400, detail="Page must be >= 1") + if page_size < 1 or page_size > 100: + logger.error(f"[get_dashboards][Coherence:Failed] Invalid page_size: {page_size}") + raise HTTPException(status_code=400, detail="Page size must be between 1 and 100") + + environments = config_manager.get_environments() + env = next((e for e in environments if e.id == env_id), None) + if not env: + logger.error(f"[get_dashboards][Coherence:Failed] Environment not found: {env_id}") + raise HTTPException(status_code=404, detail="Environment not found") + + bound_username: Optional[str] = None + can_apply_profile_filter = False + can_apply_slug_filter = False + effective_profile_filter = EffectiveProfileFilter( + applied=False, + source_page=page_context, + override_show_all=bool(override_show_all), + username=None, + match_logic=None, + ) + profile_service: Optional[ProfileService] = None + + try: + profile_service_module = getattr(ProfileService, "__module__", "") + is_mock_db = db.__class__.__module__.startswith("unittest.mock") + use_profile_service = (not is_mock_db) or profile_service_module.startswith( + "unittest.mock" + ) + if use_profile_service: + profile_service = ProfileService(db=db, config_manager=config_manager) + profile_preference = _get_profile_filter_binding( + profile_service, current_user + ) + normalized_username = ( + str(profile_preference.get("superset_username_normalized") or "") + .strip() + .lower() + ) + raw_username = ( + str(profile_preference.get("superset_username") or "") + .strip() + .lower() + ) + bound_username = normalized_username or raw_username or None + + can_apply_profile_filter = ( + page_context == "dashboards_main" + and bool(apply_profile_default) + and not bool(override_show_all) + and bool(profile_preference.get("show_only_my_dashboards", False)) + and bool(bound_username) + ) + can_apply_slug_filter = ( + page_context == "dashboards_main" + and bool(apply_profile_default) + and not bool(override_show_all) + and bool(profile_preference.get("show_only_slug_dashboards", True)) + ) + + profile_match_logic = None + if can_apply_profile_filter and can_apply_slug_filter: + profile_match_logic = "owners_or_modified_by+slug_only" + elif can_apply_profile_filter: + profile_match_logic = "owners_or_modified_by" + elif can_apply_slug_filter: + profile_match_logic = "slug_only" + + effective_profile_filter = EffectiveProfileFilter( + applied=bool(can_apply_profile_filter or can_apply_slug_filter), + source_page=page_context, + override_show_all=bool(override_show_all), + username=bound_username if can_apply_profile_filter else None, + match_logic=profile_match_logic, + ) + except Exception as profile_error: + logger.explore( + f"[EXPLORE] Profile preference unavailable; continuing without profile-default filter: {profile_error}" + ) + + try: + all_tasks = task_manager.get_all_tasks() + title_filters = _normalize_filter_values(filter_title) + git_filters = _normalize_filter_values(filter_git_status) + llm_filters = _normalize_filter_values(filter_llm_status) + changed_on_filters = _normalize_filter_values(filter_changed_on) + actor_filters = _normalize_filter_values(filter_actor) + has_column_filters = any( + ( + title_filters, + git_filters, + llm_filters, + changed_on_filters, + actor_filters, + ) + ) + needs_full_scan = ( + has_column_filters + or bool(can_apply_profile_filter) + or bool(can_apply_slug_filter) + ) + + if not needs_full_scan: + try: + page_payload = ( + await resource_service.get_dashboards_page_with_status( + env, + all_tasks, + page=page, + page_size=page_size, + search=search, + include_git_status=False, + require_slug=bool(can_apply_slug_filter), + ) + ) + paginated_dashboards = page_payload["dashboards"] + total = page_payload["total"] + total_pages = page_payload["total_pages"] + except Exception as page_error: + logger.warning( + "[get_dashboards][Action] Page-based fetch failed; using compatibility fallback: %s", + page_error, + ) + if can_apply_slug_filter: + dashboards = await resource_service.get_dashboards_with_status( + env, + all_tasks, + include_git_status=False, + require_slug=True, + ) + else: + dashboards = await resource_service.get_dashboards_with_status( + env, + all_tasks, + include_git_status=False, + ) + + if search: + search_lower = search.lower() + dashboards = [ + d + for d in dashboards + if search_lower in d.get("title", "").lower() + or search_lower in d.get("slug", "").lower() + ] + + total = len(dashboards) + total_pages = ( + (total + page_size - 1) // page_size if total > 0 else 1 + ) + start_idx = (page - 1) * page_size + end_idx = start_idx + page_size + paginated_dashboards = dashboards[start_idx:end_idx] + else: + if can_apply_slug_filter: + dashboards = await resource_service.get_dashboards_with_status( + env, + all_tasks, + include_git_status=bool(git_filters), + require_slug=True, + ) + else: + dashboards = await resource_service.get_dashboards_with_status( + env, + all_tasks, + include_git_status=bool(git_filters), + ) + + if ( + can_apply_profile_filter + and bound_username + and profile_service is not None + ): + actor_aliases = _resolve_profile_actor_aliases(env, bound_username) + if not actor_aliases: + actor_aliases = [bound_username] + logger.reason( + "[REASON] Applying profile actor filter " + f"(env={env_id}, bound_username={bound_username}, actor_aliases={actor_aliases!r}, " + f"dashboards_before={len(dashboards)})" + ) + filtered_dashboards: List[Dict[str, Any]] = [] + max_actor_samples = 15 + for index, dashboard in enumerate(dashboards): + owners_value = dashboard.get("owners") + created_by_value = dashboard.get("created_by") + modified_by_value = dashboard.get("modified_by") + matches_actor = _matches_dashboard_actor_aliases( + profile_service=profile_service, + actor_aliases=actor_aliases, + owners=owners_value, + modified_by=modified_by_value, + ) + if index < max_actor_samples: + logger.reflect( + "[REFLECT] Profile actor filter sample " + f"(env={env_id}, dashboard_id={dashboard.get('id')}, " + f"bound_username={bound_username!r}, actor_aliases={actor_aliases!r}, " + f"owners={owners_value!r}, created_by={created_by_value!r}, " + f"modified_by={modified_by_value!r}, matches={matches_actor})" + ) + if matches_actor: + filtered_dashboards.append(dashboard) + + logger.reflect( + "[REFLECT] Profile actor filter summary " + f"(env={env_id}, bound_username={bound_username!r}, " + f"dashboards_before={len(dashboards)}, dashboards_after={len(filtered_dashboards)})" + ) + dashboards = filtered_dashboards + + if can_apply_slug_filter: + dashboards = [ + dashboard + for dashboard in dashboards + if str(dashboard.get("slug") or "").strip() + ] + + if search: + search_lower = search.lower() + dashboards = [ + d + for d in dashboards + if search_lower in d.get("title", "").lower() + or search_lower in d.get("slug", "").lower() + ] + + def _matches_dashboard_filters(dashboard: Dict[str, Any]) -> bool: + title_value = str(dashboard.get("title") or "").strip().lower() + if title_filters and title_value not in title_filters: + return False + + if git_filters: + git_value = _dashboard_git_filter_value(dashboard) + if git_value not in git_filters: + return False + + llm_value = ( + str( + ( + (dashboard.get("last_task") or {}).get( + "validation_status" + ) + ) + or "UNKNOWN" + ) + .strip() + .lower() + ) + if llm_filters and llm_value not in llm_filters: + return False + + changed_on_raw = ( + str(dashboard.get("last_modified") or "").strip().lower() + ) + changed_on_prefix = ( + changed_on_raw[:10] + if len(changed_on_raw) >= 10 + else changed_on_raw + ) + if ( + changed_on_filters + and changed_on_raw not in changed_on_filters + and changed_on_prefix not in changed_on_filters + ): + return False + + owners = dashboard.get("owners") or [] + if isinstance(owners, list): + actor_value = ", ".join( + str(item).strip() for item in owners if str(item).strip() + ).lower() + else: + actor_value = str(owners).strip().lower() + if not actor_value: + actor_value = "-" + if actor_filters and actor_value not in actor_filters: + return False + return True + + if has_column_filters: + dashboards = [ + d for d in dashboards if _matches_dashboard_filters(d) + ] + + total = len(dashboards) + total_pages = (total + page_size - 1) // page_size if total > 0 else 1 + start_idx = (page - 1) * page_size + end_idx = start_idx + page_size + paginated_dashboards = dashboards[start_idx:end_idx] + + logger.info( + f"[get_dashboards][Coherence:OK] Returning {len(paginated_dashboards)} dashboards " + f"(page {page}/{total_pages}, total: {total}, profile_filter_applied={effective_profile_filter.applied})" + ) + + response_dashboards = _project_dashboard_response_items( + paginated_dashboards + ) + + return DashboardsResponse( + dashboards=response_dashboards, + total=total, + page=page, + page_size=page_size, + total_pages=total_pages, + effective_profile_filter=effective_profile_filter, + ) + + except HTTPException: + raise + except Exception as e: + logger.error( + f"[get_dashboards][Coherence:Failed] Failed to fetch dashboards: {e}" + ) + raise HTTPException( + status_code=503, detail=f"Failed to fetch dashboards: {str(e)}" + ) + + +# [/DEF:get_dashboards:Function] +# [/DEF:DashboardListingRoutes:Module] diff --git a/backend/src/api/routes/dashboards/_projection.py b/backend/src/api/routes/dashboards/_projection.py new file mode 100644 index 00000000..6892d8c7 --- /dev/null +++ b/backend/src/api/routes/dashboards/_projection.py @@ -0,0 +1,260 @@ +# [DEF:DashboardProjection:Module] +# @COMPLEXITY: 2 +# @SEMANTICS: api, dashboards, projection, profile, filtering +# @PURPOSE: Dashboard response projection and profile-filter helpers for Dashboard Hub routes. +# @LAYER: Infra +# @RELATION: DEPENDS_ON -> [SupersetClient] +# @RELATION: DEPENDS_ON -> [ProfileService] + +from typing import Any, Dict, List, Optional + +from src.core.logger import logger, belief_scope +from src.core.superset_client import SupersetClient +from src.core.superset_profile_lookup import SupersetAccountLookupAdapter +from src.models.auth import User +from src.services.profile_service import ProfileService + + +# [DEF:_normalize_actor_alias_token:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Normalize actor alias token to comparable trim+lower text. +def _normalize_actor_alias_token(value: Any) -> Optional[str]: + if value is None: + return None + normalized = str(value).strip().lower() + return normalized if normalized else None + + +# [/DEF:_normalize_actor_alias_token:Function] + + +# [DEF:_normalize_owner_display_token:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Project owner payload value into stable display string for API response contracts. +def _normalize_owner_display_token(owner: Any) -> Optional[str]: + if owner is None: + return None + if isinstance(owner, dict): + for key in ("username", "full_name", "first_name", "email"): + candidate = owner.get(key) + if isinstance(candidate, str) and candidate.strip(): + return candidate.strip() + return None + if isinstance(owner, str): + return owner.strip() or None + return None + + +# [/DEF:_normalize_owner_display_token:Function] + + +# [DEF:_normalize_dashboard_owner_values:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Normalize dashboard owners payload to optional list of display strings. +def _normalize_dashboard_owner_values(owners: Any) -> Optional[List[str]]: + if owners is None: + return None + raw_items: List[Any] + if isinstance(owners, list): + raw_items = owners + else: + raw_items = [owners] + normalized: List[str] = [] + for owner in raw_items: + token = _normalize_owner_display_token(owner) + if token and token not in normalized: + normalized.append(token) + return normalized + + +# [/DEF:_normalize_dashboard_owner_values:Function] + + +# [DEF:_project_dashboard_response_items:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Project dashboard payloads to response-contract-safe shape. +def _project_dashboard_response_items( + dashboards: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + projected: List[Dict[str, Any]] = [] + for dashboard in dashboards: + projected_dashboard = dict(dashboard) + projected_dashboard["owners"] = _normalize_dashboard_owner_values( + projected_dashboard.get("owners") + ) + projected.append(projected_dashboard) + return projected + + +# [/DEF:_project_dashboard_response_items:Function] + + +# [DEF:_get_profile_filter_binding:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Resolve dashboard profile-filter binding through current or legacy profile service contracts. +def _get_profile_filter_binding( + profile_service: Any, current_user: User +) -> Dict[str, Any]: + + def _read_optional_string(value: Any) -> Optional[str]: + return value if isinstance(value, str) else None + + def _read_bool(value: Any, default: bool) -> bool: + return value if isinstance(value, bool) else default + + if hasattr(profile_service, "get_dashboard_filter_binding"): + binding = profile_service.get_dashboard_filter_binding(current_user) + if isinstance(binding, dict): + return { + "superset_username": _read_optional_string( + binding.get("superset_username") + ), + "superset_username_normalized": _read_optional_string( + binding.get("superset_username_normalized") + ), + "show_only_my_dashboards": _read_bool( + binding.get("show_only_my_dashboards"), False + ), + "show_only_slug_dashboards": _read_bool( + binding.get("show_only_slug_dashboards"), False + ), + } + if hasattr(profile_service, "get_my_preference"): + response = profile_service.get_my_preference(current_user) + preference = getattr(response, "preference", None) + return { + "superset_username": _read_optional_string( + getattr(preference, "superset_username", None) + ), + "superset_username_normalized": _read_optional_string( + getattr(preference, "superset_username_normalized", None) + ), + "show_only_my_dashboards": _read_bool( + getattr(preference, "show_only_my_dashboards", False), False + ), + "show_only_slug_dashboards": _read_bool( + getattr(preference, "show_only_slug_dashboards", False), False + ), + } + return { + "superset_username": None, + "superset_username_normalized": None, + "show_only_my_dashboards": False, + "show_only_slug_dashboards": False, + } + + +# [/DEF:_get_profile_filter_binding:Function] + + +# [DEF:_resolve_profile_actor_aliases:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Resolve stable actor aliases for profile filtering without per-dashboard detail fan-out. +# @SIDE_EFFECT: Performs at most one Superset users-lookup request. +def _resolve_profile_actor_aliases(env: Any, bound_username: str) -> List[str]: + normalized_bound = _normalize_actor_alias_token(bound_username) + if not normalized_bound: + return [] + aliases: List[str] = [normalized_bound] + try: + client = SupersetClient(env) + adapter = SupersetAccountLookupAdapter( + network_client=client.network, + environment_id=str(getattr(env, "id", "")), + ) + lookup_payload = adapter.get_users_page( + search=normalized_bound, + page_index=0, + page_size=20, + sort_column="username", + sort_order="asc", + ) + lookup_items = ( + lookup_payload.get("items", []) if isinstance(lookup_payload, dict) else [] + ) + matched_item: Optional[Dict[str, Any]] = None + for item in lookup_items: + if not isinstance(item, dict): + continue + if _normalize_actor_alias_token(item.get("username")) == normalized_bound: + matched_item = item + break + if matched_item is None: + for item in lookup_items: + if isinstance(item, dict): + matched_item = item + break + display_alias = _normalize_actor_alias_token( + (matched_item or {}).get("display_name") + ) + if display_alias and display_alias not in aliases: + aliases.append(display_alias) + logger.reflect( + "[REFLECT] Resolved profile actor aliases " + f"(env={getattr(env, 'id', None)}, bound_username={normalized_bound!r}, " + f"lookup_items={len(lookup_items)}, aliases={aliases!r})" + ) + except Exception as alias_error: + logger.explore( + "[EXPLORE] Failed to resolve profile actor aliases via Superset users lookup " + f"(env={getattr(env, 'id', None)}, bound_username={normalized_bound!r}): {alias_error}" + ) + return aliases + + +# [/DEF:_resolve_profile_actor_aliases:Function] + + +# [DEF:_matches_dashboard_actor_aliases:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Apply profile actor matching against multiple aliases (username + optional display name). +def _matches_dashboard_actor_aliases( + profile_service: ProfileService, + actor_aliases: List[str], + owners: Optional[Any], + modified_by: Optional[str], +) -> bool: + for actor_alias in actor_aliases: + if profile_service.matches_dashboard_actor( + bound_username=actor_alias, + owners=owners, + modified_by=modified_by, + ): + return True + return False + + +# [/DEF:_matches_dashboard_actor_aliases:Function] + + +# [DEF:_task_matches_dashboard:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Checks whether task params are tied to a specific dashboard and environment. +def _task_matches_dashboard( + task: Any, dashboard_id: int, env_id: Optional[str] +) -> bool: + plugin_id = getattr(task, "plugin_id", None) + if plugin_id not in {"superset-backup", "llm_dashboard_validation"}: + return False + params = getattr(task, "params", {}) or {} + dashboard_id_str = str(dashboard_id) + if plugin_id == "llm_dashboard_validation": + task_dashboard_id = params.get("dashboard_id") + if str(task_dashboard_id) != dashboard_id_str: + return False + if env_id: + task_env = params.get("environment_id") + return str(task_env) == str(env_id) + return True + dashboard_ids = params.get("dashboard_ids") or params.get("dashboards") or [] + normalized_ids = {str(item) for item in dashboard_ids} + if dashboard_id_str not in normalized_ids: + return False + if env_id: + task_env = params.get("environment_id") or params.get("env") + return str(task_env) == str(env_id) + return True + + +# [/DEF:_task_matches_dashboard:Function] +# [/DEF:DashboardProjection:Module] diff --git a/backend/src/api/routes/dashboards/_router.py b/backend/src/api/routes/dashboards/_router.py new file mode 100644 index 00000000..e1449166 --- /dev/null +++ b/backend/src/api/routes/dashboards/_router.py @@ -0,0 +1,8 @@ +# [DEF:DashboardsRouter:Block] +# @COMPLEXITY: 1 +# @PURPOSE: Single APIRouter instance for all dashboard endpoints. +# @RELATION: BINDS_TO -> [DashboardsApi] +from fastapi import APIRouter + +router = APIRouter(prefix="/api/dashboards", tags=["Dashboards"]) +# [/DEF:DashboardsRouter:Block] diff --git a/backend/src/api/routes/dashboards/_schemas.py b/backend/src/api/routes/dashboards/_schemas.py new file mode 100644 index 00000000..a9298ca3 --- /dev/null +++ b/backend/src/api/routes/dashboards/_schemas.py @@ -0,0 +1,232 @@ +# [DEF:DashboardSchemas:Module] +# @COMPLEXITY: 1 +# @SEMANTICS: api, dashboards, schemas, dto +# @PURPOSE: DTO classes for the Dashboard Hub API. +# @LAYER: Infra +# @RELATION: DEPENDS_ON -> [Pydantic] + +from typing import List, Optional, Dict, Any, Literal +from pydantic import BaseModel, Field + + +# [DEF:GitStatus:DataClass] +# @COMPLEXITY: 1 +# @PURPOSE: DTO for dashboard Git synchronization status. +class GitStatus(BaseModel): + branch: Optional[str] = None + sync_status: Optional[str] = Field(None, pattern="^OK|DIFF|NO_REPO|ERROR$") + has_repo: Optional[bool] = None + has_changes_for_commit: Optional[bool] = None + + +# [/DEF:GitStatus:DataClass] + + +# [DEF:LastTask:DataClass] +# @COMPLEXITY: 2 +# @PURPOSE: DTO for the most recent background task associated with a dashboard. +class LastTask(BaseModel): + task_id: Optional[str] = None + status: Optional[str] = Field( + None, + pattern="^PENDING|RUNNING|SUCCESS|FAILED|ERROR|AWAITING_INPUT|WAITING_INPUT|AWAITING_MAPPING$", + ) + validation_status: Optional[str] = Field(None, pattern="^PASS|FAIL|WARN|UNKNOWN$") + + +# [/DEF:LastTask:DataClass] + + +# [DEF:DashboardItem:DataClass] +# @COMPLEXITY: 1 +# @PURPOSE: DTO representing a single dashboard with projected metadata. +class DashboardItem(BaseModel): + id: int + title: str + slug: Optional[str] = None + url: Optional[str] = None + last_modified: Optional[str] = None + created_by: Optional[str] = None + modified_by: Optional[str] = None + owners: Optional[List[str]] = None + git_status: Optional[GitStatus] = None + last_task: Optional[LastTask] = None + + +# [/DEF:DashboardItem:DataClass] + + +# [DEF:EffectiveProfileFilter:DataClass] +# @COMPLEXITY: 1 +# @PURPOSE: Metadata about applied profile filters for UI context. +class EffectiveProfileFilter(BaseModel): + applied: bool + source_page: Literal["dashboards_main", "other"] = "dashboards_main" + override_show_all: bool = False + username: Optional[str] = None + match_logic: Optional[ + Literal["owners_or_modified_by", "slug_only", "owners_or_modified_by+slug_only"] + ] = None + + +# [/DEF:EffectiveProfileFilter:DataClass] + + +# [DEF:DashboardsResponse:DataClass] +# @COMPLEXITY: 1 +# @PURPOSE: Envelope DTO for paginated dashboards list. +class DashboardsResponse(BaseModel): + dashboards: List[DashboardItem] + total: int + page: int + page_size: int + total_pages: int + effective_profile_filter: Optional[EffectiveProfileFilter] = None + + +# [/DEF:DashboardsResponse:DataClass] + + +# [DEF:DashboardChartItem:DataClass] +# @COMPLEXITY: 1 +# @PURPOSE: DTO for a chart linked to a dashboard. +class DashboardChartItem(BaseModel): + id: int + title: str + viz_type: Optional[str] = None + dataset_id: Optional[int] = None + last_modified: Optional[str] = None + overview: Optional[str] = None + + +# [/DEF:DashboardChartItem:DataClass] + + +# [DEF:DashboardDatasetItem:DataClass] +# @COMPLEXITY: 1 +# @PURPOSE: DTO for a dataset associated with a dashboard. +class DashboardDatasetItem(BaseModel): + id: int + table_name: str + schema: Optional[str] = None + database: str + last_modified: Optional[str] = None + overview: Optional[str] = None + + +# [/DEF:DashboardDatasetItem:DataClass] + + +# [DEF:DashboardDetailResponse:DataClass] +# @COMPLEXITY: 1 +# @PURPOSE: Detailed dashboard metadata including children. +class DashboardDetailResponse(BaseModel): + id: int + title: str + slug: Optional[str] = None + url: Optional[str] = None + description: Optional[str] = None + last_modified: Optional[str] = None + published: Optional[bool] = None + charts: List[DashboardChartItem] + datasets: List[DashboardDatasetItem] + chart_count: int + dataset_count: int + + +# [/DEF:DashboardDetailResponse:DataClass] + + +# [DEF:DashboardTaskHistoryItem:DataClass] +# @COMPLEXITY: 1 +# @PURPOSE: Individual history record entry. +class DashboardTaskHistoryItem(BaseModel): + id: str + plugin_id: str + status: str + validation_status: Optional[str] = None + started_at: Optional[str] = None + finished_at: Optional[str] = None + env_id: Optional[str] = None + summary: Optional[str] = None + + +# [/DEF:DashboardTaskHistoryItem:DataClass] + + +# [DEF:DashboardTaskHistoryResponse:DataClass] +# @COMPLEXITY: 1 +# @PURPOSE: Collection DTO for task history. +class DashboardTaskHistoryResponse(BaseModel): + dashboard_id: int + items: List[DashboardTaskHistoryItem] + + +# [/DEF:DashboardTaskHistoryResponse:DataClass] + + +# [DEF:DatabaseMapping:DataClass] +# @COMPLEXITY: 2 +# @PURPOSE: DTO for cross-environment database ID mapping. +class DatabaseMapping(BaseModel): + source_db: str + target_db: str + source_db_uuid: Optional[str] = None + target_db_uuid: Optional[str] = None + confidence: float + + +# [/DEF:DatabaseMapping:DataClass] + + +# [DEF:DatabaseMappingsResponse:DataClass] +# @COMPLEXITY: 1 +# @PURPOSE: Wrapper for database mappings. +class DatabaseMappingsResponse(BaseModel): + mappings: List[DatabaseMapping] + + +# [/DEF:DatabaseMappingsResponse:DataClass] + + +# [DEF:MigrateRequest:DataClass] +# @COMPLEXITY: 1 +# @PURPOSE: DTO for dashboard migration requests. +class MigrateRequest(BaseModel): + source_env_id: str = Field(..., description="Source environment ID") + target_env_id: str = Field(..., description="Target environment ID") + dashboard_ids: List[int] = Field( + ..., description="List of dashboard IDs to migrate" + ) + db_mappings: Optional[Dict[str, str]] = Field( + None, description="Database mappings for migration" + ) + replace_db_config: bool = Field(False, description="Replace database configuration") + + +# [/DEF:MigrateRequest:DataClass] + + +# [DEF:TaskResponse:DataClass] +# @COMPLEXITY: 1 +# @PURPOSE: DTO for async task ID return. +class TaskResponse(BaseModel): + task_id: str + + +# [/DEF:TaskResponse:DataClass] + + +# [DEF:BackupRequest:DataClass] +# @COMPLEXITY: 1 +# @PURPOSE: DTO for dashboard backup requests. +class BackupRequest(BaseModel): + env_id: str = Field(..., description="Environment ID") + dashboard_ids: List[int] = Field(..., description="List of dashboard IDs to backup") + schedule: Optional[str] = Field( + None, description="Cron schedule for recurring backups (e.g., '0 0 * * *')" + ) + + +# [/DEF:BackupRequest:DataClass] +# [/DEF:DashboardSchemas:Module] diff --git a/backend/src/api/routes/git.py b/backend/src/api/routes/git.py deleted file mode 100644 index 5cd1fb9c..00000000 --- a/backend/src/api/routes/git.py +++ /dev/null @@ -1,1757 +0,0 @@ -# [DEF:GitApi:Module] -# -# @COMPLEXITY: 3 -# @SEMANTICS: git, routes, api, fastapi, repository, deployment -# @PURPOSE: Provides FastAPI endpoints for Git integration operations. -# @LAYER: API -# @RELATION: USES -> [GitService] -# @RELATION: USES -> [GitSchemas] -# @RELATION: USES -> [GitModels] -# -# @INVARIANT: All Git operations must be routed through GitService. - -from fastapi import APIRouter, Depends, HTTPException -from sqlalchemy.orm import Session -from typing import List, Optional -import typing -import os -from src.dependencies import get_config_manager, get_current_user, has_permission -from src.core.database import get_db -from src.models.auth import User -from src.models.git import GitServerConfig, GitRepository, GitProvider -from src.models.profile import UserDashboardPreference -from src.api.routes.git_schemas import ( - GitServerConfigSchema, - GitServerConfigCreate, - GitServerConfigUpdate, - BranchSchema, - BranchCreate, - BranchCheckout, - CommitSchema, - CommitCreate, - DeploymentEnvironmentSchema, - DeployRequest, - RepoInitRequest, - RepositoryBindingSchema, - RepoStatusBatchRequest, - RepoStatusBatchResponse, - GiteaRepoCreateRequest, - GiteaRepoSchema, - RemoteRepoCreateRequest, - RemoteRepoSchema, - PromoteRequest, - PromoteResponse, - MergeStatusSchema, - MergeConflictFileSchema, - MergeResolveRequest, - MergeContinueRequest, -) -from src.services.git_service import GitService -from src.core.async_superset_client import AsyncSupersetClient -from src.core.superset_client import SupersetClient -from src.core.logger import logger, belief_scope -from ...services.llm_prompt_templates import ( - DEFAULT_LLM_PROMPTS, - normalize_llm_settings, - resolve_bound_provider_id, -) - -router = APIRouter(tags=["git"]) -git_service = GitService() -MAX_REPOSITORY_STATUS_BATCH = 50 - - -# [DEF:_build_no_repo_status_payload:Function] -# @COMPLEXITY: 1 -# @PURPOSE: Build a consistent status payload for dashboards without initialized repositories. -# @PRE: None. -# @POST: Returns a stable payload compatible with frontend repository status parsing. -# @RETURN: dict -def _build_no_repo_status_payload() -> dict: - return { - "is_dirty": False, - "untracked_files": [], - "modified_files": [], - "staged_files": [], - "current_branch": None, - "upstream_branch": None, - "has_upstream": False, - "ahead_count": 0, - "behind_count": 0, - "is_diverged": False, - "sync_state": "NO_REPO", - "sync_status": "NO_REPO", - "has_repo": False, - } - - -# [/DEF:_build_no_repo_status_payload:Function] - - -# [DEF:_handle_unexpected_git_route_error:Function] -# @COMPLEXITY: 1 -# @PURPOSE: Convert unexpected route-level exceptions to stable 500 API responses. -# @PRE: `error` is a non-HTTPException instance. -# @POST: Raises HTTPException(500) with route-specific context. -# @PARAM: route_name (str) -# @PARAM: error (Exception) -def _handle_unexpected_git_route_error(route_name: str, error: Exception) -> None: - logger.error(f"[{route_name}][Coherence:Failed] {error}") - raise HTTPException(status_code=500, detail=f"{route_name} failed: {str(error)}") - - -# [/DEF:_handle_unexpected_git_route_error:Function] - - -# [DEF:_resolve_repository_status:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve repository status for one dashboard with graceful NO_REPO semantics. -# @PRE: `dashboard_id` is a valid integer. -# @POST: Returns standard status payload or `NO_REPO` payload when repository path is absent. -# @PARAM: dashboard_id (int) -# @RETURN: dict -def _resolve_repository_status(dashboard_id: int) -> dict: - repo_path = git_service._get_repo_path(dashboard_id) - if not os.path.exists(repo_path): - logger.debug( - f"[get_repository_status][Action] Repository is not initialized for dashboard {dashboard_id}" - ) - return _build_no_repo_status_payload() - - try: - return git_service.get_status(dashboard_id) - except HTTPException as e: - if e.status_code == 404: - logger.debug( - f"[get_repository_status][Action] Repository is not initialized for dashboard {dashboard_id}" - ) - return _build_no_repo_status_payload() - raise - - -# [/DEF:_resolve_repository_status:Function] - - -# [DEF:_get_git_config_or_404:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve GitServerConfig by id or raise 404. -# @PRE: db session is available. -# @POST: Returns GitServerConfig model. -def _get_git_config_or_404(db: Session, config_id: str) -> GitServerConfig: - config = db.query(GitServerConfig).filter(GitServerConfig.id == config_id).first() - if not config: - raise HTTPException(status_code=404, detail="Git configuration not found") - return config - - -# [/DEF:_get_git_config_or_404:Function] - - -# [DEF:_find_dashboard_id_by_slug:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve dashboard numeric ID by slug in a specific environment. -# @PRE: dashboard_slug is non-empty. -# @POST: Returns dashboard ID or None when not found. -def _find_dashboard_id_by_slug( - client: SupersetClient, - dashboard_slug: str, -) -> Optional[int]: - query_variants = [ - { - "filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}], - "page": 0, - "page_size": 1, - }, - { - "filters": [{"col": "slug", "op": "eq", "value": dashboard_slug}], - "page": 0, - "page_size": 1, - }, - ] - - for query in query_variants: - try: - _count, dashboards = client.get_dashboards_page(query=query) - if dashboards: - resolved_id = dashboards[0].get("id") - if resolved_id is not None: - return int(resolved_id) - except Exception: - continue - return None - - -# [/DEF:_find_dashboard_id_by_slug:Function] - - -# [DEF:_resolve_dashboard_id_from_ref:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve dashboard ID from slug-or-id reference for Git routes. -# @PRE: dashboard_ref is provided; env_id is required for slug values. -# @POST: Returns numeric dashboard ID or raises HTTPException. -def _resolve_dashboard_id_from_ref( - dashboard_ref: str, - config_manager, - env_id: Optional[str] = None, -) -> int: - normalized_ref = str(dashboard_ref or "").strip() - if not normalized_ref: - raise HTTPException(status_code=400, detail="dashboard_ref is required") - - if normalized_ref.isdigit(): - return int(normalized_ref) - - if not env_id: - raise HTTPException( - status_code=400, - detail="env_id is required for slug-based Git operations", - ) - - environments = config_manager.get_environments() - env = next((e for e in environments if e.id == env_id), None) - if not env: - raise HTTPException(status_code=404, detail="Environment not found") - - dashboard_id = _find_dashboard_id_by_slug(SupersetClient(env), normalized_ref) - if dashboard_id is None: - raise HTTPException( - status_code=404, detail=f"Dashboard slug '{normalized_ref}' not found" - ) - return dashboard_id - - -# [/DEF:_resolve_dashboard_id_from_ref:Function] - - -# [DEF:_find_dashboard_id_by_slug_async:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve dashboard numeric ID by slug asynchronously for hot-path Git routes. -# @PRE: dashboard_slug is non-empty. -# @POST: Returns dashboard ID or None when not found. -async def _find_dashboard_id_by_slug_async( - client: AsyncSupersetClient, - dashboard_slug: str, -) -> Optional[int]: - query_variants = [ - { - "filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}], - "page": 0, - "page_size": 1, - }, - { - "filters": [{"col": "slug", "op": "eq", "value": dashboard_slug}], - "page": 0, - "page_size": 1, - }, - ] - - for query in query_variants: - try: - _count, dashboards = await client.get_dashboards_page_async(query=query) - if dashboards: - resolved_id = dashboards[0].get("id") - if resolved_id is not None: - return int(resolved_id) - except Exception: - continue - return None - - -# [/DEF:_find_dashboard_id_by_slug_async:Function] - - -# [DEF:_resolve_dashboard_id_from_ref_async:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve dashboard ID asynchronously from slug-or-id reference for hot Git routes. -# @PRE: dashboard_ref is provided; env_id is required for slug values. -# @POST: Returns numeric dashboard ID or raises HTTPException. -async def _resolve_dashboard_id_from_ref_async( - dashboard_ref: str, - config_manager, - env_id: Optional[str] = None, -) -> int: - normalized_ref = str(dashboard_ref or "").strip() - if not normalized_ref: - raise HTTPException(status_code=400, detail="dashboard_ref is required") - - if normalized_ref.isdigit(): - return int(normalized_ref) - - if not env_id: - raise HTTPException( - status_code=400, - detail="env_id is required for slug-based Git operations", - ) - - environments = config_manager.get_environments() - env = next((e for e in environments if e.id == env_id), None) - if not env: - raise HTTPException(status_code=404, detail="Environment not found") - - client = AsyncSupersetClient(env) - try: - dashboard_id = await _find_dashboard_id_by_slug_async(client, normalized_ref) - if dashboard_id is None: - raise HTTPException( - status_code=404, detail=f"Dashboard slug '{normalized_ref}' not found" - ) - return dashboard_id - finally: - await client.aclose() - - -# [/DEF:_resolve_dashboard_id_from_ref_async:Function] - - -# [DEF:_resolve_repo_key_from_ref:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve repository folder key with slug-first strategy and deterministic fallback. -# @PRE: dashboard_id is resolved and valid. -# @POST: Returns safe key to be used in local repository path. -# @RETURN: str -def _resolve_repo_key_from_ref( - dashboard_ref: str, - dashboard_id: int, - config_manager, - env_id: Optional[str] = None, -) -> str: - normalized_ref = str(dashboard_ref or "").strip() - if normalized_ref and not normalized_ref.isdigit(): - return normalized_ref - - if env_id: - try: - environments = config_manager.get_environments() - env = next((e for e in environments if e.id == env_id), None) - if env: - payload = SupersetClient(env).get_dashboard(dashboard_id) - dashboard_data = ( - payload.get("result", payload) if isinstance(payload, dict) else {} - ) - dashboard_slug = dashboard_data.get("slug") - if dashboard_slug: - return str(dashboard_slug) - except Exception: - pass - - return f"dashboard-{dashboard_id}" - - -# [/DEF:_resolve_repo_key_from_ref:Function] - - -# [DEF:_sanitize_optional_identity_value:Function] -# @COMPLEXITY: 1 -# @PURPOSE: Normalize optional identity value into trimmed string or None. -# @PRE: value may be None or blank. -# @POST: Returns sanitized value suitable for git identity configuration. -# @RETURN: Optional[str] -def _sanitize_optional_identity_value(value: Optional[str]) -> Optional[str]: - normalized = str(value or "").strip() - if not normalized: - return None - return normalized - - -# [/DEF:_sanitize_optional_identity_value:Function] - - -# [DEF:_resolve_current_user_git_identity:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Resolve configured Git username/email from current user's profile preferences. -# @PRE: `db` may be stubbed in tests; `current_user` may be absent for direct handler invocations. -# @POST: Returns tuple(username, email) only when both values are configured. -# @RETURN: Optional[tuple[str, str]] -def _resolve_current_user_git_identity( - db: Session, - current_user: Optional[User], -) -> Optional[tuple[str, str]]: - if db is None or not hasattr(db, "query"): - return None - - user_id = _sanitize_optional_identity_value(getattr(current_user, "id", None)) - if not user_id: - return None - - try: - preference = ( - db.query(UserDashboardPreference) - .filter(UserDashboardPreference.user_id == user_id) - .first() - ) - except Exception as resolve_error: - logger.warning( - "[_resolve_current_user_git_identity][Action] Failed to load profile preference for user %s: %s", - user_id, - resolve_error, - ) - return None - - if not preference: - return None - - git_username = _sanitize_optional_identity_value( - getattr(preference, "git_username", None) - ) - git_email = _sanitize_optional_identity_value( - getattr(preference, "git_email", None) - ) - if not git_username or not git_email: - return None - return git_username, git_email - - -# [/DEF:_resolve_current_user_git_identity:Function] - - -# [DEF:_apply_git_identity_from_profile:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Apply user-scoped Git identity to repository-local config before write/pull operations. -# @PRE: dashboard_id is resolved; db/current_user may be missing in direct test invocation context. -# @POST: git_service.configure_identity is called only when identity and method are available. -# @RETURN: None -def _apply_git_identity_from_profile( - dashboard_id: int, - db: Session, - current_user: Optional[User], -) -> None: - identity = _resolve_current_user_git_identity(db, current_user) - if not identity: - return - - configure_identity = getattr(git_service, "configure_identity", None) - if not callable(configure_identity): - return - - git_username, git_email = identity - configure_identity(dashboard_id, git_username, git_email) - - -# [/DEF:_apply_git_identity_from_profile:Function] - - -# [DEF:get_git_configs:Function] -# @COMPLEXITY: 2 -# @PURPOSE: List all configured Git servers. -# @PRE: Database session `db` is available. -# @POST: Returns a list of all GitServerConfig objects from the database. -# @RETURN: List[GitServerConfigSchema] -@router.get("/config", response_model=List[GitServerConfigSchema]) -async def get_git_configs( - db: Session = Depends(get_db), _=Depends(has_permission("git_config", "READ")) -): - with belief_scope("get_git_configs"): - configs = db.query(GitServerConfig).all() - result = [] - for config in configs: - schema = GitServerConfigSchema.from_orm(config) - schema.pat = "********" - result.append(schema) - return result - - -# [/DEF:get_git_configs:Function] - - -# [DEF:create_git_config:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Register a new Git server configuration. -# @PRE: `config` contains valid GitServerConfigCreate data. -# @POST: A new GitServerConfig record is created in the database. -# @PARAM: config (GitServerConfigCreate) -# @RETURN: GitServerConfigSchema -@router.post("/config", response_model=GitServerConfigSchema) -async def create_git_config( - config: GitServerConfigCreate, - db: Session = Depends(get_db), - _=Depends(has_permission("admin:settings", "WRITE")), -): - with belief_scope("create_git_config"): - config_dict = config.dict(exclude={"config_id"}) - db_config = GitServerConfig(**config_dict) - db.add(db_config) - db.commit() - db.refresh(db_config) - return db_config - - -# [/DEF:create_git_config:Function] - - -# [DEF:update_git_config:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Update an existing Git server configuration. -# @PRE: `config_id` corresponds to an existing configuration. -# @POST: The configuration record is updated in the database. -# @PARAM: config_id (str) -# @PARAM: config_update (GitServerConfigUpdate) -# @RETURN: GitServerConfigSchema -@router.put("/config/{config_id}", response_model=GitServerConfigSchema) -async def update_git_config( - config_id: str, - config_update: GitServerConfigUpdate, - db: Session = Depends(get_db), - _=Depends(has_permission("admin:settings", "WRITE")), -): - with belief_scope("update_git_config"): - db_config = ( - db.query(GitServerConfig).filter(GitServerConfig.id == config_id).first() - ) - if not db_config: - raise HTTPException(status_code=404, detail="Configuration not found") - - update_data = config_update.dict(exclude_unset=True) - if update_data.get("pat") == "********": - update_data.pop("pat") - - for key, value in update_data.items(): - setattr(db_config, key, value) - - db.commit() - db.refresh(db_config) - - result_schema = GitServerConfigSchema.from_orm(db_config) - result_schema.pat = "********" - return result_schema - - -# [/DEF:update_git_config:Function] - - -# [DEF:delete_git_config:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Remove a Git server configuration. -# @PRE: `config_id` corresponds to an existing configuration. -# @POST: The configuration record is removed from the database. -# @PARAM: config_id (str) -@router.delete("/config/{config_id}") -async def delete_git_config( - config_id: str, - db: Session = Depends(get_db), - _=Depends(has_permission("admin:settings", "WRITE")), -): - with belief_scope("delete_git_config"): - db_config = ( - db.query(GitServerConfig).filter(GitServerConfig.id == config_id).first() - ) - if not db_config: - raise HTTPException(status_code=404, detail="Configuration not found") - - db.delete(db_config) - db.commit() - return {"status": "success", "message": "Configuration deleted"} - - -# [/DEF:delete_git_config:Function] - - -# [DEF:test_git_config:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Validate connection to a Git server using provided credentials. -# @PRE: `config` contains provider, url, and pat. -# @POST: Returns success if the connection is validated via GitService. -# @PARAM: config (GitServerConfigCreate) -@router.post("/config/test") -async def test_git_config( - config: GitServerConfigCreate, - db: Session = Depends(get_db), - _=Depends(has_permission("git_config", "READ")), -): - with belief_scope("test_git_config"): - pat_to_use = config.pat - if pat_to_use == "********": - if config.config_id: - db_config = ( - db.query(GitServerConfig) - .filter(GitServerConfig.id == config.config_id) - .first() - ) - if db_config: - pat_to_use = db_config.pat - else: - db_config = ( - db.query(GitServerConfig) - .filter(GitServerConfig.url == config.url) - .first() - ) - if db_config: - pat_to_use = db_config.pat - - success = await git_service.test_connection( - config.provider, config.url, pat_to_use - ) - if success: - return {"status": "success", "message": "Connection successful"} - else: - raise HTTPException(status_code=400, detail="Connection failed") - - -# [/DEF:test_git_config:Function] - - -# [DEF:list_gitea_repositories:Function] -# @COMPLEXITY: 2 -# @PURPOSE: List repositories in Gitea for a saved Gitea config. -# @PRE: config_id exists and provider is GITEA. -# @POST: Returns repositories visible to PAT user. -@router.get("/config/{config_id}/gitea/repos", response_model=List[GiteaRepoSchema]) -async def list_gitea_repositories( - config_id: str, - db: Session = Depends(get_db), - _=Depends(has_permission("git_config", "READ")), -): - with belief_scope("list_gitea_repositories"): - config = _get_git_config_or_404(db, config_id) - if config.provider != GitProvider.GITEA: - raise HTTPException( - status_code=400, detail="This endpoint supports GITEA provider only" - ) - repos = await git_service.list_gitea_repositories(config.url, config.pat) - return [ - GiteaRepoSchema( - name=repo.get("name", ""), - full_name=repo.get("full_name", ""), - private=bool(repo.get("private", False)), - clone_url=repo.get("clone_url"), - html_url=repo.get("html_url"), - ssh_url=repo.get("ssh_url"), - default_branch=repo.get("default_branch"), - ) - for repo in repos - ] - - -# [/DEF:list_gitea_repositories:Function] - - -# [DEF:create_gitea_repository:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Create a repository in Gitea for a saved Gitea config. -# @PRE: config_id exists and provider is GITEA. -# @POST: Returns created repository payload. -@router.post("/config/{config_id}/gitea/repos", response_model=GiteaRepoSchema) -async def create_gitea_repository( - config_id: str, - request: GiteaRepoCreateRequest, - db: Session = Depends(get_db), - _=Depends(has_permission("admin:settings", "WRITE")), -): - with belief_scope("create_gitea_repository"): - config = _get_git_config_or_404(db, config_id) - if config.provider != GitProvider.GITEA: - raise HTTPException( - status_code=400, detail="This endpoint supports GITEA provider only" - ) - repo = await git_service.create_gitea_repository( - server_url=config.url, - pat=config.pat, - name=request.name, - private=request.private, - description=request.description, - auto_init=request.auto_init, - default_branch=request.default_branch, - ) - return GiteaRepoSchema( - name=repo.get("name", ""), - full_name=repo.get("full_name", ""), - private=bool(repo.get("private", False)), - clone_url=repo.get("clone_url"), - html_url=repo.get("html_url"), - ssh_url=repo.get("ssh_url"), - default_branch=repo.get("default_branch"), - ) - - -# [/DEF:create_gitea_repository:Function] - - -# [DEF:create_remote_repository:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Create repository on remote Git server using selected provider config. -# @PRE: config_id exists and PAT has creation permissions. -# @POST: Returns normalized remote repository payload. -@router.post("/config/{config_id}/repositories", response_model=RemoteRepoSchema) -async def create_remote_repository( - config_id: str, - request: RemoteRepoCreateRequest, - db: Session = Depends(get_db), - _=Depends(has_permission("admin:settings", "WRITE")), -): - with belief_scope("create_remote_repository"): - config = _get_git_config_or_404(db, config_id) - - if config.provider == GitProvider.GITEA: - repo = await git_service.create_gitea_repository( - server_url=config.url, - pat=config.pat, - name=request.name, - private=request.private, - description=request.description, - auto_init=request.auto_init, - default_branch=request.default_branch, - ) - elif config.provider == GitProvider.GITHUB: - repo = await git_service.create_github_repository( - server_url=config.url, - pat=config.pat, - name=request.name, - private=request.private, - description=request.description, - auto_init=request.auto_init, - default_branch=request.default_branch, - ) - elif config.provider == GitProvider.GITLAB: - repo = await git_service.create_gitlab_repository( - server_url=config.url, - pat=config.pat, - name=request.name, - private=request.private, - description=request.description, - auto_init=request.auto_init, - default_branch=request.default_branch, - ) - else: - raise HTTPException( - status_code=501, detail=f"Provider {config.provider} is not supported" - ) - - return RemoteRepoSchema( - provider=config.provider, - name=repo.get("name", ""), - full_name=repo.get("full_name", repo.get("name", "")), - private=bool(repo.get("private", False)), - clone_url=repo.get("clone_url"), - html_url=repo.get("html_url"), - ssh_url=repo.get("ssh_url"), - default_branch=repo.get("default_branch"), - ) - - -# [/DEF:create_remote_repository:Function] - - -# [DEF:delete_gitea_repository:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Delete repository in Gitea for a saved Gitea config. -# @PRE: config_id exists and provider is GITEA. -# @POST: Target repository is deleted on Gitea. -@router.delete("/config/{config_id}/gitea/repos/{owner}/{repo_name}") -async def delete_gitea_repository( - config_id: str, - owner: str, - repo_name: str, - db: Session = Depends(get_db), - _=Depends(has_permission("admin:settings", "WRITE")), -): - with belief_scope("delete_gitea_repository"): - config = _get_git_config_or_404(db, config_id) - if config.provider != GitProvider.GITEA: - raise HTTPException( - status_code=400, detail="This endpoint supports GITEA provider only" - ) - await git_service.delete_gitea_repository( - server_url=config.url, - pat=config.pat, - owner=owner, - repo_name=repo_name, - ) - return {"status": "success", "message": "Repository deleted"} - - -# [/DEF:delete_gitea_repository:Function] - - -# [DEF:init_repository:Function] -# @COMPLEXITY: 3 -# @PURPOSE: Link a dashboard to a Git repository and perform initial clone/init. -# @PRE: `dashboard_ref` exists and `init_data` contains valid config_id and remote_url. -# @POST: Repository is initialized on disk and a GitRepository record is saved in DB. -# @PARAM: dashboard_ref (str) -# @PARAM: init_data (RepoInitRequest) -# @RELATION: CALLS -> [GitService] -@router.post("/repositories/{dashboard_ref}/init") -async def init_repository( - dashboard_ref: str, - init_data: RepoInitRequest, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - db: Session = Depends(get_db), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("init_repository"): - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - repo_key = _resolve_repo_key_from_ref( - dashboard_ref, dashboard_id, config_manager, env_id - ) - # 1. Get config - config = ( - db.query(GitServerConfig) - .filter(GitServerConfig.id == init_data.config_id) - .first() - ) - if not config: - raise HTTPException(status_code=404, detail="Git configuration not found") - - try: - # 2. Perform Git clone/init - logger.info( - f"[init_repository][Action] Initializing repo for dashboard {dashboard_id}" - ) - git_service.init_repo( - dashboard_id, - init_data.remote_url, - config.pat, - repo_key=repo_key, - default_branch=config.default_branch, - ) - - # 3. Save to DB - repo_path = git_service._get_repo_path(dashboard_id, repo_key=repo_key) - db_repo = ( - db.query(GitRepository) - .filter(GitRepository.dashboard_id == dashboard_id) - .first() - ) - if not db_repo: - db_repo = GitRepository( - dashboard_id=dashboard_id, - config_id=config.id, - remote_url=init_data.remote_url, - local_path=repo_path, - current_branch="dev", - ) - db.add(db_repo) - else: - db_repo.config_id = config.id - db_repo.remote_url = init_data.remote_url - db_repo.local_path = repo_path - db_repo.current_branch = "dev" - - db.commit() - logger.info( - f"[init_repository][Coherence:OK] Repository initialized for dashboard {dashboard_id}" - ) - return {"status": "success", "message": "Repository initialized"} - except Exception as e: - db.rollback() - logger.error( - f"[init_repository][Coherence:Failed] Failed to init repository: {e}" - ) - if isinstance(e, HTTPException): - raise - _handle_unexpected_git_route_error("init_repository", e) - - -# [/DEF:init_repository:Function] - - -# [DEF:get_repository_binding:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Return repository binding with provider metadata for selected dashboard. -# @PRE: `dashboard_ref` resolves to a valid dashboard and repository is initialized. -# @POST: Returns dashboard repository binding and linked provider. -# @PARAM: dashboard_ref (str) -# @RETURN: RepositoryBindingSchema -@router.get("/repositories/{dashboard_ref}", response_model=RepositoryBindingSchema) -async def get_repository_binding( - dashboard_ref: str, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - db: Session = Depends(get_db), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("get_repository_binding"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - db_repo = ( - db.query(GitRepository) - .filter(GitRepository.dashboard_id == dashboard_id) - .first() - ) - if not db_repo: - raise HTTPException( - status_code=404, detail="Repository not initialized" - ) - config = _get_git_config_or_404(db, db_repo.config_id) - return RepositoryBindingSchema( - dashboard_id=db_repo.dashboard_id, - config_id=db_repo.config_id, - provider=config.provider, - remote_url=db_repo.remote_url, - local_path=db_repo.local_path, - ) - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("get_repository_binding", e) - - -# [/DEF:get_repository_binding:Function] - - -# [DEF:delete_repository:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Delete local repository workspace and DB binding for selected dashboard. -# @PRE: `dashboard_ref` resolves to a valid dashboard. -# @POST: Repository files and binding record are removed when present. -# @PARAM: dashboard_ref (str) -# @RETURN: dict -@router.delete("/repositories/{dashboard_ref}") -async def delete_repository( - dashboard_ref: str, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("delete_repository"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - git_service.delete_repo(dashboard_id) - return {"status": "success"} - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("delete_repository", e) - - -# [/DEF:delete_repository:Function] - - -# [DEF:get_branches:Function] -# @COMPLEXITY: 2 -# @PURPOSE: List all branches for a dashboard's repository. -# @PRE: Repository for `dashboard_ref` is initialized. -# @POST: Returns a list of branches from the local repository. -# @PARAM: dashboard_ref (str) -# @RETURN: List[BranchSchema] -@router.get("/repositories/{dashboard_ref}/branches", response_model=List[BranchSchema]) -async def get_branches( - dashboard_ref: str, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("get_branches"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - return git_service.list_branches(dashboard_id) - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("get_branches", e) - - -# [/DEF:get_branches:Function] - - -# [DEF:create_branch:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Create a new branch in the dashboard's repository. -# @PRE: `dashboard_ref` repository exists and `branch_data` has name and from_branch. -# @POST: A new branch is created in the local repository. -# @PARAM: dashboard_ref (str) -# @PARAM: branch_data (BranchCreate) -@router.post("/repositories/{dashboard_ref}/branches") -async def create_branch( - dashboard_ref: str, - branch_data: BranchCreate, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("create_branch"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - _apply_git_identity_from_profile(dashboard_id, db, current_user) - git_service.create_branch( - dashboard_id, branch_data.name, branch_data.from_branch - ) - return {"status": "success"} - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("create_branch", e) - - -# [/DEF:create_branch:Function] - - -# [DEF:checkout_branch:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Switch the dashboard's repository to a specific branch. -# @PRE: `dashboard_ref` repository exists and branch `checkout_data.name` exists. -# @POST: The local repository HEAD is moved to the specified branch. -# @PARAM: dashboard_ref (str) -# @PARAM: checkout_data (BranchCheckout) -@router.post("/repositories/{dashboard_ref}/checkout") -async def checkout_branch( - dashboard_ref: str, - checkout_data: BranchCheckout, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("checkout_branch"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - git_service.checkout_branch(dashboard_id, checkout_data.name) - return {"status": "success"} - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("checkout_branch", e) - - -# [/DEF:checkout_branch:Function] - - -# [DEF:commit_changes:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Stage and commit changes in the dashboard's repository. -# @PRE: `dashboard_ref` repository exists and `commit_data` has message and files. -# @POST: Specified files are staged and a new commit is created. -# @PARAM: dashboard_ref (str) -# @PARAM: commit_data (CommitCreate) -@router.post("/repositories/{dashboard_ref}/commit") -async def commit_changes( - dashboard_ref: str, - commit_data: CommitCreate, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("commit_changes"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - _apply_git_identity_from_profile(dashboard_id, db, current_user) - git_service.commit_changes( - dashboard_id, commit_data.message, commit_data.files - ) - return {"status": "success"} - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("commit_changes", e) - - -# [/DEF:commit_changes:Function] - - -# [DEF:push_changes:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Push local commits to the remote repository. -# @PRE: `dashboard_ref` repository exists and has a remote configured. -# @POST: Local commits are pushed to the remote repository. -# @PARAM: dashboard_ref (str) -@router.post("/repositories/{dashboard_ref}/push") -async def push_changes( - dashboard_ref: str, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("push_changes"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - git_service.push_changes(dashboard_id) - return {"status": "success"} - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("push_changes", e) - - -# [/DEF:push_changes:Function] - - -# [DEF:pull_changes:Function] -# @COMPLEXITY: 3 -# @PURPOSE: Pull changes from the remote repository. -# @PRE: `dashboard_ref` repository exists and has a remote configured. -# @POST: Remote changes are fetched and merged into the local branch. -# @PARAM: dashboard_ref (str) -# @RELATION: CALLS -> [GitService] -@router.post("/repositories/{dashboard_ref}/pull") -async def pull_changes( - dashboard_ref: str, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("pull_changes"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - db_repo = None - config_url = None - config_provider = None - try: - db_repo_candidate = ( - db.query(GitRepository) - .filter(GitRepository.dashboard_id == dashboard_id) - .first() - ) - if getattr(db_repo_candidate, "config_id", None): - db_repo = db_repo_candidate - config_row = ( - db.query(GitServerConfig) - .filter(GitServerConfig.id == db_repo.config_id) - .first() - ) - if config_row: - config_url = config_row.url - config_provider = config_row.provider - except Exception as diagnostics_error: - logger.warning( - "[pull_changes][Action] Failed to load repository binding diagnostics for dashboard %s: %s", - dashboard_id, - diagnostics_error, - ) - logger.info( - "[pull_changes][Action] Route diagnostics dashboard_ref=%s env_id=%s resolved_dashboard_id=%s " - "binding_exists=%s binding_local_path=%s binding_remote_url=%s binding_config_id=%s config_provider=%s config_url=%s", - dashboard_ref, - env_id, - dashboard_id, - bool(db_repo), - (db_repo.local_path if db_repo else None), - (db_repo.remote_url if db_repo else None), - (db_repo.config_id if db_repo else None), - config_provider, - config_url, - ) - _apply_git_identity_from_profile(dashboard_id, db, current_user) - git_service.pull_changes(dashboard_id) - return {"status": "success"} - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("pull_changes", e) - - -# [/DEF:pull_changes:Function] - - -# [DEF:get_merge_status:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Return unfinished-merge status for repository (web-only recovery support). -# @PRE: `dashboard_ref` resolves to a valid dashboard repository. -# @POST: Returns merge status payload. -@router.get( - "/repositories/{dashboard_ref}/merge/status", response_model=MergeStatusSchema -) -async def get_merge_status( - dashboard_ref: str, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("get_merge_status"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - return git_service.get_merge_status(dashboard_id) - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("get_merge_status", e) - - -# [/DEF:get_merge_status:Function] - - -# [DEF:get_merge_conflicts:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Return conflicted files with mine/theirs previews for web conflict resolver. -# @PRE: `dashboard_ref` resolves to a valid dashboard repository. -# @POST: Returns conflict file list. -@router.get( - "/repositories/{dashboard_ref}/merge/conflicts", - response_model=List[MergeConflictFileSchema], -) -async def get_merge_conflicts( - dashboard_ref: str, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("get_merge_conflicts"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - return git_service.get_merge_conflicts(dashboard_id) - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("get_merge_conflicts", e) - - -# [/DEF:get_merge_conflicts:Function] - - -# [DEF:resolve_merge_conflicts:Function] -# @COMPLEXITY: 3 -# @PURPOSE: Apply mine/theirs/manual conflict resolutions from WebUI and stage files. -# @PRE: `dashboard_ref` resolves; request contains at least one resolution item. -# @POST: Resolved files are staged in index. -# @RELATION: CALLS -> [GitService] -@router.post("/repositories/{dashboard_ref}/merge/resolve") -async def resolve_merge_conflicts( - dashboard_ref: str, - resolve_data: MergeResolveRequest, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("resolve_merge_conflicts"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - resolved_files = git_service.resolve_merge_conflicts( - dashboard_id, - [item.dict() for item in resolve_data.resolutions], - ) - return {"status": "success", "resolved_files": resolved_files} - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("resolve_merge_conflicts", e) - - -# [/DEF:resolve_merge_conflicts:Function] - - -# [DEF:abort_merge:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Abort unfinished merge from WebUI flow. -# @PRE: `dashboard_ref` resolves to repository. -# @POST: Merge operation is aborted or reports no active merge. -@router.post("/repositories/{dashboard_ref}/merge/abort") -async def abort_merge( - dashboard_ref: str, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("abort_merge"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - return git_service.abort_merge(dashboard_id) - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("abort_merge", e) - - -# [/DEF:abort_merge:Function] - - -# [DEF:continue_merge:Function] -# @COMPLEXITY: 3 -# @PURPOSE: Finalize unfinished merge from WebUI flow. -# @PRE: All conflicts are resolved and staged. -# @POST: Merge commit is created. -# @RELATION: CALLS -> [GitService] -@router.post("/repositories/{dashboard_ref}/merge/continue") -async def continue_merge( - dashboard_ref: str, - continue_data: MergeContinueRequest, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("continue_merge"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - return git_service.continue_merge(dashboard_id, continue_data.message) - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("continue_merge", e) - - -# [/DEF:continue_merge:Function] - - -# [DEF:sync_dashboard:Function] -# @COMPLEXITY: 3 -# @PURPOSE: Sync dashboard state from Superset to Git using the GitPlugin. -# @PRE: `dashboard_ref` is valid; GitPlugin is available. -# @POST: Dashboard YAMLs are exported from Superset and committed to Git. -# @PARAM: dashboard_ref (str) -# @PARAM: source_env_id (Optional[str]) -# @RELATION: CALLS -> [GitPlugin] -@router.post("/repositories/{dashboard_ref}/sync") -async def sync_dashboard( - dashboard_ref: str, - env_id: Optional[str] = None, - source_env_id: typing.Optional[str] = None, - config_manager=Depends(get_config_manager), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("sync_dashboard"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - from src.plugins.git_plugin import GitPlugin - - plugin = GitPlugin() - return await plugin.execute( - { - "operation": "sync", - "dashboard_id": dashboard_id, - "source_env_id": source_env_id, - } - ) - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("sync_dashboard", e) - - -# [/DEF:sync_dashboard:Function] - - -# [DEF:promote_dashboard:Function] -# @COMPLEXITY: 3 -# @PURPOSE: Promote changes between branches via MR or direct merge. -# @PRE: dashboard repository is initialized and Git config is valid. -# @POST: Returns promotion result metadata. -# @RELATION: CALLS -> [GitPlugin] -@router.post("/repositories/{dashboard_ref}/promote", response_model=PromoteResponse) -async def promote_dashboard( - dashboard_ref: str, - payload: PromoteRequest, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("promote_dashboard"): - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - db_repo = ( - db.query(GitRepository) - .filter(GitRepository.dashboard_id == dashboard_id) - .first() - ) - if not db_repo: - raise HTTPException( - status_code=404, - detail=f"Repository for dashboard {dashboard_ref} is not initialized", - ) - config = _get_git_config_or_404(db, db_repo.config_id) - - from_branch = payload.from_branch.strip() - to_branch = payload.to_branch.strip() - if not from_branch or not to_branch: - raise HTTPException( - status_code=400, detail="from_branch and to_branch are required" - ) - if from_branch == to_branch: - raise HTTPException( - status_code=400, detail="from_branch and to_branch must be different" - ) - - mode = (payload.mode or "mr").strip().lower() - if mode == "direct": - reason = (payload.reason or "").strip() - if not reason: - raise HTTPException( - status_code=400, detail="Direct promote requires non-empty reason" - ) - logger.warning( - "[promote_dashboard][PolicyViolation] Direct promote without MR by actor=unknown dashboard_ref=%s from=%s to=%s reason=%s", - dashboard_ref, - from_branch, - to_branch, - reason, - ) - _apply_git_identity_from_profile(dashboard_id, db, current_user) - result = git_service.promote_direct_merge( - dashboard_id=dashboard_id, - from_branch=from_branch, - to_branch=to_branch, - ) - return PromoteResponse( - mode="direct", - from_branch=from_branch, - to_branch=to_branch, - status=result.get("status", "merged"), - policy_violation=True, - ) - - title = (payload.title or "").strip() or f"Promote {from_branch} -> {to_branch}" - description = payload.description - if config.provider == GitProvider.GITEA: - pr = await git_service.create_gitea_pull_request( - server_url=config.url, - pat=config.pat, - remote_url=db_repo.remote_url, - from_branch=from_branch, - to_branch=to_branch, - title=title, - description=description, - ) - elif config.provider == GitProvider.GITHUB: - pr = await git_service.create_github_pull_request( - server_url=config.url, - pat=config.pat, - remote_url=db_repo.remote_url, - from_branch=from_branch, - to_branch=to_branch, - title=title, - description=description, - draft=payload.draft, - ) - elif config.provider == GitProvider.GITLAB: - pr = await git_service.create_gitlab_merge_request( - server_url=config.url, - pat=config.pat, - remote_url=db_repo.remote_url, - from_branch=from_branch, - to_branch=to_branch, - title=title, - description=description, - remove_source_branch=payload.remove_source_branch, - ) - else: - raise HTTPException( - status_code=501, - detail=f"Provider {config.provider} does not support promotion API", - ) - - return PromoteResponse( - mode="mr", - from_branch=from_branch, - to_branch=to_branch, - status=pr.get("status", "opened"), - url=pr.get("url"), - reference_id=str(pr.get("id")) if pr.get("id") is not None else None, - policy_violation=False, - ) - - -# [/DEF:promote_dashboard:Function] - - -# [DEF:get_environments:Function] -# @COMPLEXITY: 2 -# @PURPOSE: List all deployment environments. -# @PRE: Config manager is accessible. -# @POST: Returns a list of DeploymentEnvironmentSchema objects. -# @RETURN: List[DeploymentEnvironmentSchema] -@router.get("/environments", response_model=List[DeploymentEnvironmentSchema]) -async def get_environments( - config_manager=Depends(get_config_manager), - _=Depends(has_permission("environments", "READ")), -): - with belief_scope("get_environments"): - envs = config_manager.get_environments() - return [ - DeploymentEnvironmentSchema( - id=e.id, name=e.name, superset_url=e.url, is_active=True - ) - for e in envs - ] - - -# [/DEF:get_environments:Function] - - -# [DEF:deploy_dashboard:Function] -# @COMPLEXITY: 3 -# @PURPOSE: Deploy dashboard from Git to a target environment. -# @PRE: `dashboard_ref` and `deploy_data.environment_id` are valid. -# @POST: Dashboard YAMLs are read from Git and imported into the target Superset. -# @PARAM: dashboard_ref (str) -# @PARAM: deploy_data (DeployRequest) -# @RELATION: CALLS -> [GitPlugin] -@router.post("/repositories/{dashboard_ref}/deploy") -async def deploy_dashboard( - dashboard_ref: str, - deploy_data: DeployRequest, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("deploy_dashboard"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - from src.plugins.git_plugin import GitPlugin - - plugin = GitPlugin() - return await plugin.execute( - { - "operation": "deploy", - "dashboard_id": dashboard_id, - "environment_id": deploy_data.environment_id, - } - ) - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("deploy_dashboard", e) - - -# [/DEF:deploy_dashboard:Function] - - -# [DEF:get_history:Function] -# @COMPLEXITY: 2 -# @PURPOSE: View commit history for a dashboard's repository. -# @PRE: `dashboard_ref` repository exists. -# @POST: Returns a list of recent commits from the repository. -# @PARAM: dashboard_ref (str) -# @PARAM: limit (int) -# @RETURN: List[CommitSchema] -@router.get("/repositories/{dashboard_ref}/history", response_model=List[CommitSchema]) -async def get_history( - dashboard_ref: str, - limit: int = 50, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("get_history"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - return git_service.get_commit_history(dashboard_id, limit) - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("get_history", e) - - -# [/DEF:get_history:Function] - - -# [DEF:get_repository_status:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Get current Git status for a dashboard repository. -# @PRE: `dashboard_ref` resolves to a valid dashboard. -# @POST: Returns repository status; if repo is not initialized, returns `NO_REPO` payload. -# @PARAM: dashboard_ref (str) -# @RETURN: dict -@router.get("/repositories/{dashboard_ref}/status") -async def get_repository_status( - dashboard_ref: str, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("get_repository_status"): - try: - dashboard_id = await _resolve_dashboard_id_from_ref_async( - dashboard_ref, config_manager, env_id - ) - return _resolve_repository_status(dashboard_id) - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("get_repository_status", e) - - -# [/DEF:get_repository_status:Function] - - -# [DEF:get_repository_status_batch:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Get Git statuses for multiple dashboard repositories in one request. -# @PRE: `request.dashboard_ids` is provided. -# @POST: Returns `statuses` map where each key is dashboard ID and value is repository status payload. -# @PARAM: request (RepoStatusBatchRequest) -# @RETURN: RepoStatusBatchResponse -@router.post("/repositories/status/batch", response_model=RepoStatusBatchResponse) -async def get_repository_status_batch( - request: RepoStatusBatchRequest, _=Depends(has_permission("plugin:git", "EXECUTE")) -): - with belief_scope("get_repository_status_batch"): - dashboard_ids = list(dict.fromkeys(request.dashboard_ids)) - if len(dashboard_ids) > MAX_REPOSITORY_STATUS_BATCH: - logger.warning( - "[get_repository_status_batch][Action] Batch size %s exceeds limit %s. Truncating request.", - len(dashboard_ids), - MAX_REPOSITORY_STATUS_BATCH, - ) - dashboard_ids = dashboard_ids[:MAX_REPOSITORY_STATUS_BATCH] - - statuses = {} - for dashboard_id in dashboard_ids: - try: - statuses[str(dashboard_id)] = _resolve_repository_status(dashboard_id) - except HTTPException: - statuses[str(dashboard_id)] = { - **_build_no_repo_status_payload(), - "sync_state": "ERROR", - "sync_status": "ERROR", - } - except Exception as e: - logger.error( - f"[get_repository_status_batch][Coherence:Failed] Failed for dashboard {dashboard_id}: {e}" - ) - statuses[str(dashboard_id)] = { - **_build_no_repo_status_payload(), - "sync_state": "ERROR", - "sync_status": "ERROR", - } - return RepoStatusBatchResponse(statuses=statuses) - - -# [/DEF:get_repository_status_batch:Function] - - -# [DEF:get_repository_diff:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Get Git diff for a dashboard repository. -# @PRE: `dashboard_ref` repository exists. -# @POST: Returns the diff text for the specified file or all changes. -# @PARAM: dashboard_ref (str) -# @PARAM: file_path (Optional[str]) -# @PARAM: staged (bool) -# @RETURN: str -@router.get("/repositories/{dashboard_ref}/diff") -async def get_repository_diff( - dashboard_ref: str, - file_path: Optional[str] = None, - staged: bool = False, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("get_repository_diff"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - diff_text = git_service.get_diff(dashboard_id, file_path, staged) - return diff_text - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("get_repository_diff", e) - - -# [/DEF:get_repository_diff:Function] - - -# [DEF:generate_commit_message:Function] -# @COMPLEXITY: 3 -# @PURPOSE: Generate a suggested commit message using LLM. -# @PRE: Repository for `dashboard_ref` is initialized. -# @POST: Returns a suggested commit message string. -# @RELATION: CALLS -> [GitService] -@router.post("/repositories/{dashboard_ref}/generate-message") -async def generate_commit_message( - dashboard_ref: str, - env_id: Optional[str] = None, - config_manager=Depends(get_config_manager), - db: Session = Depends(get_db), - _=Depends(has_permission("plugin:git", "EXECUTE")), -): - with belief_scope("generate_commit_message"): - try: - dashboard_id = _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - # 1. Get Diff - diff = git_service.get_diff(dashboard_id, staged=True) - if not diff: - diff = git_service.get_diff(dashboard_id, staged=False) - - if not diff: - return {"message": "No changes detected"} - - # 2. Get History - history_objs = git_service.get_commit_history(dashboard_id, limit=5) - history = [h.message for h in history_objs if hasattr(h, "message")] - - # 3. Get LLM Client - from ...services.llm_provider import LLMProviderService - from ...plugins.llm_analysis.service import LLMClient - from ...plugins.llm_analysis.models import LLMProviderType - - llm_service = LLMProviderService(db) - providers = llm_service.get_all_providers() - llm_settings = normalize_llm_settings( - config_manager.get_config().settings.llm - ) - bound_provider_id = resolve_bound_provider_id(llm_settings, "git_commit") - provider = next((p for p in providers if p.id == bound_provider_id), None) - if not provider: - provider = next((p for p in providers if p.is_active), None) - - if not provider: - raise HTTPException( - status_code=400, detail="No active LLM provider found" - ) - - api_key = llm_service.get_decrypted_api_key(provider.id) - client = LLMClient( - provider_type=LLMProviderType(provider.provider_type), - api_key=api_key, - base_url=provider.base_url, - default_model=provider.default_model, - ) - - # 4. Generate Message - from ...plugins.git.llm_extension import GitLLMExtension - - extension = GitLLMExtension(client) - git_prompt = llm_settings["prompts"].get( - "git_commit_prompt", - DEFAULT_LLM_PROMPTS["git_commit_prompt"], - ) - message = await extension.suggest_commit_message( - diff, - history, - prompt_template=git_prompt, - ) - - return {"message": message} - except HTTPException: - raise - except Exception as e: - _handle_unexpected_git_route_error("generate_commit_message", e) - - -# [/DEF:generate_commit_message:Function] - -# [/DEF:GitApi:Module] diff --git a/backend/src/api/routes/git/__init__.py b/backend/src/api/routes/git/__init__.py new file mode 100644 index 00000000..0c8390f0 --- /dev/null +++ b/backend/src/api/routes/git/__init__.py @@ -0,0 +1,44 @@ +# [DEF:GitPackage:Module] +# @COMPLEXITY: 3 +# @PURPOSE: Package root for decomposed git routes. Re-exports all public symbols from submodules. +# @LAYER: API +# @SEMANTICS: git, package, re-exports +# @RELATION: USES -> [GitRouter, GitDeps, GitHelpers, GitConfigRoutes, GitGiteaRoutes, +# GitRepoRoutes, GitRepoOperationsRoutes, GitRepoLifecycleRoutes, +# GitMergeRoutes, GitEnvironmentRoutes] +# @INVARIANT: git_service and os are module-level attributes for test monkeypatch compatibility. +# All route functions are re-exported for direct access via `from src.api.routes import git`. + +import os + +from src.services.git_service import GitService + +from ._router import router +from ._deps import MAX_REPOSITORY_STATUS_BATCH + +# -- git_service singleton (canonical instance; tests may monkeypatch git_routes.git_service) -- +git_service = GitService() + +# -- Config routes -- +from ._config_routes import get_git_configs, create_git_config, update_git_config, delete_git_config, test_git_config # noqa: E501, F401 + +# -- Gitea routes -- +from ._gitea_routes import list_gitea_repositories, create_gitea_repository, delete_gitea_repository, create_remote_repository # noqa: E501, F401 + +# -- Repo routes (core) -- +from ._repo_routes import init_repository, get_repository_binding, delete_repository, get_branches, create_branch, checkout_branch # noqa: E501, F401 +from ._helpers import _resolve_dashboard_id_from_ref, _resolve_dashboard_id_from_ref_async, _resolve_repo_key_from_ref # noqa: E501, F401 — re-exported for test monkeypatch compatibility + +# -- Repo operations routes (commit, push, pull, status, diff, history, generate-message) -- +from ._repo_operations_routes import commit_changes, push_changes, pull_changes, get_repository_status, get_repository_status_batch, get_repository_diff, get_history, generate_commit_message # noqa: E501, F401 + +# -- Repo lifecycle routes (sync, promote, deploy) -- +from ._repo_lifecycle_routes import sync_dashboard, promote_dashboard, deploy_dashboard # noqa: E501, F401 + +# -- Merge routes -- +from ._merge_routes import get_merge_status, get_merge_conflicts, resolve_merge_conflicts, abort_merge, continue_merge # noqa: E501, F401 + +# -- Environment routes -- +from ._environment_routes import get_environments # noqa: E501, F401 + +# [/DEF:GitPackage:Module] diff --git a/backend/src/api/routes/git/_config_routes.py b/backend/src/api/routes/git/_config_routes.py new file mode 100644 index 00000000..6b757ddd --- /dev/null +++ b/backend/src/api/routes/git/_config_routes.py @@ -0,0 +1,159 @@ +# [DEF:GitConfigRoutes:Module] +# @COMPLEXITY: 2 +# @PURPOSE: FastAPI endpoints for Git server configuration CRUD and connection testing. +# @LAYER: API +# @SEMANTICS: git, config, routes, crud + +from typing import List + +from fastapi import Depends, HTTPException +from sqlalchemy.orm import Session + +from src.core.database import get_db +from src.core.logger import belief_scope, logger +from src.dependencies import has_permission +from src.models.git import GitServerConfig + +from src.api.routes.git_schemas import ( + GitServerConfigCreate, + GitServerConfigSchema, + GitServerConfigUpdate, +) + +from ._router import router +from ._helpers import _get_git_config_or_404 +from ._deps import get_git_service + + +# [DEF:get_git_configs:Function] +# @COMPLEXITY: 2 +# @PURPOSE: List all configured Git servers. +@router.get("/config", response_model=List[GitServerConfigSchema]) +async def get_git_configs( + db: Session = Depends(get_db), + _=Depends(has_permission("git_config", "READ")), +): + with belief_scope("get_git_configs"): + configs = db.query(GitServerConfig).all() + result = [] + for config in configs: + schema = GitServerConfigSchema.from_orm(config) + schema.pat = "********" + result.append(schema) + return result +# [/DEF:get_git_configs:Function] + + +# [DEF:create_git_config:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Register a new Git server configuration. +@router.post("/config", response_model=GitServerConfigSchema) +async def create_git_config( + config: GitServerConfigCreate, + db: Session = Depends(get_db), + _=Depends(has_permission("admin:settings", "WRITE")), +): + with belief_scope("create_git_config"): + config_dict = config.dict(exclude={"config_id"}) + db_config = GitServerConfig(**config_dict) + db.add(db_config) + db.commit() + db.refresh(db_config) + return db_config +# [/DEF:create_git_config:Function] + + +# [DEF:update_git_config:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Update an existing Git server configuration. +@router.put("/config/{config_id}", response_model=GitServerConfigSchema) +async def update_git_config( + config_id: str, + config_update: GitServerConfigUpdate, + db: Session = Depends(get_db), + _=Depends(has_permission("admin:settings", "WRITE")), +): + with belief_scope("update_git_config"): + db_config = ( + db.query(GitServerConfig).filter(GitServerConfig.id == config_id).first() + ) + if not db_config: + raise HTTPException(status_code=404, detail="Configuration not found") + + update_data = config_update.dict(exclude_unset=True) + if update_data.get("pat") == "********": + update_data.pop("pat") + + for key, value in update_data.items(): + setattr(db_config, key, value) + + db.commit() + db.refresh(db_config) + + result_schema = GitServerConfigSchema.from_orm(db_config) + result_schema.pat = "********" + return result_schema +# [/DEF:update_git_config:Function] + + +# [DEF:delete_git_config:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Remove a Git server configuration. +@router.delete("/config/{config_id}") +async def delete_git_config( + config_id: str, + db: Session = Depends(get_db), + _=Depends(has_permission("admin:settings", "WRITE")), +): + with belief_scope("delete_git_config"): + db_config = ( + db.query(GitServerConfig).filter(GitServerConfig.id == config_id).first() + ) + if not db_config: + raise HTTPException(status_code=404, detail="Configuration not found") + + db.delete(db_config) + db.commit() + return {"status": "success", "message": "Configuration deleted"} +# [/DEF:delete_git_config:Function] + + +# [DEF:test_git_config:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Validate connection to a Git server using provided credentials. +@router.post("/config/test") +async def test_git_config( + config: GitServerConfigCreate, + db: Session = Depends(get_db), + _=Depends(has_permission("git_config", "READ")), +): + with belief_scope("test_git_config"): + _git_service = get_git_service() + pat_to_use = config.pat + if pat_to_use == "********": + if config.config_id: + db_config = ( + db.query(GitServerConfig) + .filter(GitServerConfig.id == config.config_id) + .first() + ) + if db_config: + pat_to_use = db_config.pat + else: + db_config = ( + db.query(GitServerConfig) + .filter(GitServerConfig.url == config.url) + .first() + ) + if db_config: + pat_to_use = db_config.pat + + success = await _git_service.test_connection( + config.provider, config.url, pat_to_use + ) + if success: + return {"status": "success", "message": "Connection successful"} + else: + raise HTTPException(status_code=400, detail="Connection failed") +# [/DEF:test_git_config:Function] +# [/DEF:GitConfigRoutes:Module] diff --git a/backend/src/api/routes/git/_deps.py b/backend/src/api/routes/git/_deps.py new file mode 100644 index 00000000..d8ea712c --- /dev/null +++ b/backend/src/api/routes/git/_deps.py @@ -0,0 +1,24 @@ +# [DEF:GitDeps:Module] +# @COMPLEXITY: 1 +# @PURPOSE: Shared dependency wiring for monkeypatch-safe git_service access and constants. +# @LAYER: API +# @SEMANTICS: git, deps, service-locator +# @INVARIANT: get_git_service() resolves from sys.modules at call time so test monkeypatching +# of git_routes.git_service (the __init__ attribute) takes effect across all submodules. + +import sys + +# Batch limit for repository status queries +MAX_REPOSITORY_STATUS_BATCH: int = 50 + + +def get_git_service(): + """Resolve git_service from the git package root at call time. + + This indirection ensures that test monkeypatching via + ``monkeypatch.setattr(git_routes, 'git_service', mock)`` + is visible to every submodule function, because each call + re-resolves ``sys.modules['src.api.routes.git'].git_service``. + """ + return sys.modules["src.api.routes.git"].git_service +# [/DEF:GitDeps:Module] diff --git a/backend/src/api/routes/git/_environment_routes.py b/backend/src/api/routes/git/_environment_routes.py new file mode 100644 index 00000000..385693e7 --- /dev/null +++ b/backend/src/api/routes/git/_environment_routes.py @@ -0,0 +1,36 @@ +# [DEF:GitEnvironmentRoutes:Module] +# @COMPLEXITY: 2 +# @PURPOSE: FastAPI endpoint for listing deployment environments. +# @LAYER: API +# @SEMANTICS: git, environments, routes + +from typing import List + +from fastapi import Depends + +from src.core.logger import belief_scope +from src.dependencies import get_config_manager, has_permission + +from src.api.routes.git_schemas import DeploymentEnvironmentSchema + +from ._router import router + + +# [DEF:get_environments:Function] +# @COMPLEXITY: 2 +# @PURPOSE: List all deployment environments. +@router.get("/environments", response_model=List[DeploymentEnvironmentSchema]) +async def get_environments( + config_manager=Depends(get_config_manager), + _=Depends(has_permission("environments", "READ")), +): + with belief_scope("get_environments"): + envs = config_manager.get_environments() + return [ + DeploymentEnvironmentSchema( + id=e.id, name=e.name, superset_url=e.url, is_active=True + ) + for e in envs + ] +# [/DEF:get_environments:Function] +# [/DEF:GitEnvironmentRoutes:Module] diff --git a/backend/src/api/routes/git/_gitea_routes.py b/backend/src/api/routes/git/_gitea_routes.py new file mode 100644 index 00000000..ceac24aa --- /dev/null +++ b/backend/src/api/routes/git/_gitea_routes.py @@ -0,0 +1,188 @@ +# [DEF:GitGiteaRoutes:Module] +# @COMPLEXITY: 2 +# @PURPOSE: FastAPI endpoints for Gitea-specific repository operations. +# @LAYER: API +# @SEMANTICS: git, gitea, routes, repositories + +from typing import List + +from fastapi import Depends, HTTPException +from sqlalchemy.orm import Session + +from src.core.database import get_db +from src.core.logger import belief_scope +from src.dependencies import has_permission +from src.models.git import GitProvider + +from src.api.routes.git_schemas import ( + GiteaRepoCreateRequest, + GiteaRepoSchema, + RemoteRepoCreateRequest, + RemoteRepoSchema, +) + +from ._router import router +from ._helpers import _get_git_config_or_404 +from ._deps import get_git_service + + +# [DEF:list_gitea_repositories:Function] +# @COMPLEXITY: 2 +# @PURPOSE: List repositories in Gitea for a saved Gitea config. +@router.get("/config/{config_id}/gitea/repos", response_model=List[GiteaRepoSchema]) +async def list_gitea_repositories( + config_id: str, + db: Session = Depends(get_db), + _=Depends(has_permission("git_config", "READ")), +): + _gs = get_git_service() + with belief_scope("list_gitea_repositories"): + config = _get_git_config_or_404(db, config_id) + if config.provider != GitProvider.GITEA: + raise HTTPException( + status_code=400, detail="This endpoint supports GITEA provider only" + ) + repos = await _gs.list_gitea_repositories(config.url, config.pat) + return [ + GiteaRepoSchema( + name=repo.get("name", ""), + full_name=repo.get("full_name", ""), + private=bool(repo.get("private", False)), + clone_url=repo.get("clone_url"), + html_url=repo.get("html_url"), + ssh_url=repo.get("ssh_url"), + default_branch=repo.get("default_branch"), + ) + for repo in repos + ] +# [/DEF:list_gitea_repositories:Function] + + +# [DEF:create_gitea_repository:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Create a repository in Gitea for a saved Gitea config. +@router.post("/config/{config_id}/gitea/repos", response_model=GiteaRepoSchema) +async def create_gitea_repository( + config_id: str, + request: GiteaRepoCreateRequest, + db: Session = Depends(get_db), + _=Depends(has_permission("admin:settings", "WRITE")), +): + _gs = get_git_service() + with belief_scope("create_gitea_repository"): + config = _get_git_config_or_404(db, config_id) + if config.provider != GitProvider.GITEA: + raise HTTPException( + status_code=400, detail="This endpoint supports GITEA provider only" + ) + repo = await _gs.create_gitea_repository( + server_url=config.url, + pat=config.pat, + name=request.name, + private=request.private, + description=request.description, + auto_init=request.auto_init, + default_branch=request.default_branch, + ) + return GiteaRepoSchema( + name=repo.get("name", ""), + full_name=repo.get("full_name", ""), + private=bool(repo.get("private", False)), + clone_url=repo.get("clone_url"), + html_url=repo.get("html_url"), + ssh_url=repo.get("ssh_url"), + default_branch=repo.get("default_branch"), + ) +# [/DEF:create_gitea_repository:Function] + + +# [DEF:delete_gitea_repository:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Delete repository in Gitea for a saved Gitea config. +@router.delete("/config/{config_id}/gitea/repos/{owner}/{repo_name}") +async def delete_gitea_repository( + config_id: str, + owner: str, + repo_name: str, + db: Session = Depends(get_db), + _=Depends(has_permission("admin:settings", "WRITE")), +): + _gs = get_git_service() + with belief_scope("delete_gitea_repository"): + config = _get_git_config_or_404(db, config_id) + if config.provider != GitProvider.GITEA: + raise HTTPException( + status_code=400, detail="This endpoint supports GITEA provider only" + ) + await _gs.delete_gitea_repository( + server_url=config.url, + pat=config.pat, + owner=owner, + repo_name=repo_name, + ) + return {"status": "success", "message": "Repository deleted"} +# [/DEF:delete_gitea_repository:Function] + + +# [DEF:create_remote_repository:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Create repository on remote Git server using selected provider config. +@router.post("/config/{config_id}/repositories", response_model=RemoteRepoSchema) +async def create_remote_repository( + config_id: str, + request: RemoteRepoCreateRequest, + db: Session = Depends(get_db), + _=Depends(has_permission("admin:settings", "WRITE")), +): + _gs = get_git_service() + with belief_scope("create_remote_repository"): + config = _get_git_config_or_404(db, config_id) + + if config.provider == GitProvider.GITEA: + repo = await _gs.create_gitea_repository( + server_url=config.url, + pat=config.pat, + name=request.name, + private=request.private, + description=request.description, + auto_init=request.auto_init, + default_branch=request.default_branch, + ) + elif config.provider == GitProvider.GITHUB: + repo = await _gs.create_github_repository( + server_url=config.url, + pat=config.pat, + name=request.name, + private=request.private, + description=request.description, + auto_init=request.auto_init, + default_branch=request.default_branch, + ) + elif config.provider == GitProvider.GITLAB: + repo = await _gs.create_gitlab_repository( + server_url=config.url, + pat=config.pat, + name=request.name, + private=request.private, + description=request.description, + auto_init=request.auto_init, + default_branch=request.default_branch, + ) + else: + raise HTTPException( + status_code=501, + detail=f"Provider {config.provider} is not supported", + ) + + return RemoteRepoSchema( + provider=config.provider, + name=repo.get("name", ""), + full_name=repo.get("full_name", repo.get("name", "")), + private=bool(repo.get("private", False)), + clone_url=repo.get("clone_url"), + html_url=repo.get("html_url"), + ssh_url=repo.get("ssh_url"), + default_branch=repo.get("default_branch"), + ) +# [/DEF:create_remote_repository:Function] +# [/DEF:GitGiteaRoutes:Module] diff --git a/backend/src/api/routes/git/_helpers.py b/backend/src/api/routes/git/_helpers.py new file mode 100644 index 00000000..9bf90e36 --- /dev/null +++ b/backend/src/api/routes/git/_helpers.py @@ -0,0 +1,345 @@ +# [DEF:GitHelpers:Module] +# @COMPLEXITY: 3 +# @PURPOSE: Shared helper functions for Git route modules. +# @LAYER: API +# @SEMANTICS: git, helpers, resolution, identity +# @RELATION USES -> [GitDeps] +# @RELATION USES -> [SupersetClient] +# @RELATION USES -> [UserDashboardPreference] + +import os +from typing import Optional + +from fastapi import HTTPException +from sqlalchemy.orm import Session + +from src.core.logger import logger +from src.core.superset_client import SupersetClient +from src.dependencies import get_config_manager +from src.models.auth import User +from src.models.git import GitServerConfig +from src.models.profile import UserDashboardPreference + +from ._deps import get_git_service, MAX_REPOSITORY_STATUS_BATCH + + +# [DEF:_build_no_repo_status_payload:Function] +# @COMPLEXITY: 1 +# @PURPOSE: Build a consistent status payload for dashboards without initialized repositories. +# @POST: Returns a stable payload compatible with frontend repository status parsing. +def _build_no_repo_status_payload() -> dict: + return { + "is_dirty": False, + "untracked_files": [], + "modified_files": [], + "staged_files": [], + "current_branch": None, + "upstream_branch": None, + "has_upstream": False, + "ahead_count": 0, + "behind_count": 0, + "is_diverged": False, + "sync_state": "NO_REPO", + "sync_status": "NO_REPO", + "has_repo": False, + } +# [/DEF:_build_no_repo_status_payload:Function] + + +# [DEF:_handle_unexpected_git_route_error:Function] +# @COMPLEXITY: 1 +# @PURPOSE: Convert unexpected route-level exceptions to stable 500 API responses. +# @PRE: `error` is a non-HTTPException instance. +# @POST: Raises HTTPException(500) with route-specific context. +def _handle_unexpected_git_route_error(route_name: str, error: Exception) -> None: + logger.error(f"[{route_name}][Coherence:Failed] {error}") + raise HTTPException(status_code=500, detail=f"{route_name} failed: {str(error)}") +# [/DEF:_handle_unexpected_git_route_error:Function] + + +# [DEF:_resolve_repository_status:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Resolve repository status for one dashboard with graceful NO_REPO semantics. +# @PRE: `dashboard_id` is a valid integer. +# @POST: Returns standard status payload or `NO_REPO` payload when repository path is absent. +def _resolve_repository_status(dashboard_id: int) -> dict: + git_service = get_git_service() + repo_path = git_service._get_repo_path(dashboard_id) + if not os.path.exists(repo_path): + logger.debug( + f"[get_repository_status][Action] Repository is not initialized for dashboard {dashboard_id}" + ) + return _build_no_repo_status_payload() + + try: + return git_service.get_status(dashboard_id) + except HTTPException as e: + if e.status_code == 404: + logger.debug( + f"[get_repository_status][Action] Repository is not initialized for dashboard {dashboard_id}" + ) + return _build_no_repo_status_payload() + raise +# [/DEF:_resolve_repository_status:Function] + + +# [DEF:_get_git_config_or_404:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Resolve GitServerConfig by id or raise 404. +def _get_git_config_or_404(db: Session, config_id: str) -> GitServerConfig: + config = db.query(GitServerConfig).filter(GitServerConfig.id == config_id).first() + if not config: + raise HTTPException(status_code=404, detail="Git configuration not found") + return config +# [/DEF:_get_git_config_or_404:Function] + + +# [DEF:_find_dashboard_id_by_slug:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Resolve dashboard numeric ID by slug in a specific environment. +def _find_dashboard_id_by_slug( + client: SupersetClient, + dashboard_slug: str, +) -> Optional[int]: + query_variants = [ + { + "filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}], + "page": 0, + "page_size": 1, + }, + { + "filters": [{"col": "slug", "op": "eq", "value": dashboard_slug}], + "page": 0, + "page_size": 1, + }, + ] + + for query in query_variants: + try: + _count, dashboards = client.get_dashboards_page(query=query) + if dashboards: + resolved_id = dashboards[0].get("id") + if resolved_id is not None: + return int(resolved_id) + except Exception: + continue + return None +# [/DEF:_find_dashboard_id_by_slug:Function] + + +# [DEF:_resolve_dashboard_id_from_ref:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Resolve dashboard ID from slug-or-id reference for Git routes. +def _resolve_dashboard_id_from_ref( + dashboard_ref: str, + config_manager, + env_id: Optional[str] = None, +) -> int: + normalized_ref = str(dashboard_ref or "").strip() + if not normalized_ref: + raise HTTPException(status_code=400, detail="dashboard_ref is required") + + if normalized_ref.isdigit(): + return int(normalized_ref) + + if not env_id: + raise HTTPException( + status_code=400, + detail="env_id is required for slug-based Git operations", + ) + + environments = config_manager.get_environments() + env = next((e for e in environments if e.id == env_id), None) + if not env: + raise HTTPException(status_code=404, detail="Environment not found") + + dashboard_id = _find_dashboard_id_by_slug(SupersetClient(env), normalized_ref) + if dashboard_id is None: + raise HTTPException( + status_code=404, detail=f"Dashboard slug '{normalized_ref}' not found" + ) + return dashboard_id +# [/DEF:_resolve_dashboard_id_from_ref:Function] + + +# [DEF:_find_dashboard_id_by_slug_async:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Resolve dashboard numeric ID by slug asynchronously for hot-path Git routes. +async def _find_dashboard_id_by_slug_async( + client: "AsyncSupersetClient", + dashboard_slug: str, +) -> Optional[int]: + query_variants = [ + { + "filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}], + "page": 0, + "page_size": 1, + }, + { + "filters": [{"col": "slug", "op": "eq", "value": dashboard_slug}], + "page": 0, + "page_size": 1, + }, + ] + + for query in query_variants: + try: + _count, dashboards = await client.get_dashboards_page_async(query=query) + if dashboards: + resolved_id = dashboards[0].get("id") + if resolved_id is not None: + return int(resolved_id) + except Exception: + continue + return None +# [/DEF:_find_dashboard_id_by_slug_async:Function] + + +# [DEF:_resolve_dashboard_id_from_ref_async:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Resolve dashboard ID asynchronously from slug-or-id reference for hot Git routes. +async def _resolve_dashboard_id_from_ref_async( + dashboard_ref: str, + config_manager, + env_id: Optional[str] = None, +) -> int: + normalized_ref = str(dashboard_ref or "").strip() + if not normalized_ref: + raise HTTPException(status_code=400, detail="dashboard_ref is required") + + if normalized_ref.isdigit(): + return int(normalized_ref) + + if not env_id: + raise HTTPException( + status_code=400, + detail="env_id is required for slug-based Git operations", + ) + + environments = config_manager.get_environments() + env = next((e for e in environments if e.id == env_id), None) + if not env: + raise HTTPException(status_code=404, detail="Environment not found") + + from src.core.async_superset_client import AsyncSupersetClient + + client = AsyncSupersetClient(env) + try: + dashboard_id = await _find_dashboard_id_by_slug_async(client, normalized_ref) + if dashboard_id is None: + raise HTTPException( + status_code=404, detail=f"Dashboard slug '{normalized_ref}' not found" + ) + return dashboard_id + finally: + await client.aclose() +# [/DEF:_resolve_dashboard_id_from_ref_async:Function] + + +# [DEF:_resolve_repo_key_from_ref:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Resolve repository folder key with slug-first strategy and deterministic fallback. +def _resolve_repo_key_from_ref( + dashboard_ref: str, + dashboard_id: int, + config_manager, + env_id: Optional[str] = None, +) -> str: + normalized_ref = str(dashboard_ref or "").strip() + if normalized_ref and not normalized_ref.isdigit(): + return normalized_ref + + if env_id: + try: + environments = config_manager.get_environments() + env = next((e for e in environments if e.id == env_id), None) + if env: + payload = SupersetClient(env).get_dashboard(dashboard_id) + dashboard_data = ( + payload.get("result", payload) if isinstance(payload, dict) else {} + ) + dashboard_slug = dashboard_data.get("slug") + if dashboard_slug: + return str(dashboard_slug) + except Exception: + pass + + return f"dashboard-{dashboard_id}" +# [/DEF:_resolve_repo_key_from_ref:Function] + + +# [DEF:_sanitize_optional_identity_value:Function] +# @COMPLEXITY: 1 +# @PURPOSE: Normalize optional identity value into trimmed string or None. +def _sanitize_optional_identity_value(value: Optional[str]) -> Optional[str]: + normalized = str(value or "").strip() + if not normalized: + return None + return normalized +# [/DEF:_sanitize_optional_identity_value:Function] + + +# [DEF:_resolve_current_user_git_identity:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Resolve configured Git username/email from current user's profile preferences. +def _resolve_current_user_git_identity( + db: Session, + current_user: Optional[User], +) -> Optional[tuple[str, str]]: + if db is None or not hasattr(db, "query"): + return None + + user_id = _sanitize_optional_identity_value(getattr(current_user, "id", None)) + if not user_id: + return None + + try: + preference = ( + db.query(UserDashboardPreference) + .filter(UserDashboardPreference.user_id == user_id) + .first() + ) + except Exception as resolve_error: + logger.warning( + "[_resolve_current_user_git_identity][Action] Failed to load profile preference for user %s: %s", + user_id, + resolve_error, + ) + return None + + if not preference: + return None + + git_username = _sanitize_optional_identity_value( + getattr(preference, "git_username", None) + ) + git_email = _sanitize_optional_identity_value( + getattr(preference, "git_email", None) + ) + if not git_username or not git_email: + return None + return git_username, git_email +# [/DEF:_resolve_current_user_git_identity:Function] + + +# [DEF:_apply_git_identity_from_profile:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Apply user-scoped Git identity to repository-local config before write/pull operations. +def _apply_git_identity_from_profile( + dashboard_id: int, + db: Session, + current_user: Optional[User], +) -> None: + identity = _resolve_current_user_git_identity(db, current_user) + if not identity: + return + + git_service = get_git_service() + configure_identity = getattr(git_service, "configure_identity", None) + if not callable(configure_identity): + return + + git_username, git_email = identity + configure_identity(dashboard_id, git_username, git_email) +# [/DEF:_apply_git_identity_from_profile:Function] +# [/DEF:GitHelpers:Module] diff --git a/backend/src/api/routes/git/_merge_routes.py b/backend/src/api/routes/git/_merge_routes.py new file mode 100644 index 00000000..92dd415a --- /dev/null +++ b/backend/src/api/routes/git/_merge_routes.py @@ -0,0 +1,169 @@ +# [DEF:GitMergeRoutes:Module] +# @COMPLEXITY: 3 +# @PURPOSE: FastAPI endpoints for merge operations (status, conflicts, resolve, abort, continue). +# @LAYER: API +# @SEMANTICS: git, merge, routes, conflicts + +from typing import List, Optional + +from fastapi import Depends, HTTPException + +from src.core.logger import belief_scope +from src.dependencies import get_config_manager, has_permission + +from src.api.routes.git_schemas import ( + MergeConflictFileSchema, + MergeContinueRequest, + MergeResolveRequest, + MergeStatusSchema, +) + +from ._router import router +from ._helpers import ( + _handle_unexpected_git_route_error, +) +from ._deps import get_git_service + + +# [DEF:get_merge_status:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Return unfinished-merge status for repository (web-only recovery support). +@router.get( + "/repositories/{dashboard_ref}/merge/status", response_model=MergeStatusSchema +) +async def get_merge_status( + dashboard_ref: str, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("get_merge_status"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + return _gs.get_merge_status(dashboard_id) + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("get_merge_status", e) +# [/DEF:get_merge_status:Function] + + +# [DEF:get_merge_conflicts:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Return conflicted files with mine/theirs previews for web conflict resolver. +@router.get( + "/repositories/{dashboard_ref}/merge/conflicts", + response_model=List[MergeConflictFileSchema], +) +async def get_merge_conflicts( + dashboard_ref: str, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("get_merge_conflicts"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + return _gs.get_merge_conflicts(dashboard_id) + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("get_merge_conflicts", e) +# [/DEF:get_merge_conflicts:Function] + + +# [DEF:resolve_merge_conflicts:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Apply mine/theirs/manual conflict resolutions from WebUI and stage files. +# @RELATION: CALLS -> [GitService] +@router.post("/repositories/{dashboard_ref}/merge/resolve") +async def resolve_merge_conflicts( + dashboard_ref: str, + resolve_data: MergeResolveRequest, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("resolve_merge_conflicts"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + resolved_files = _gs.resolve_merge_conflicts( + dashboard_id, + [item.dict() for item in resolve_data.resolutions], + ) + return {"status": "success", "resolved_files": resolved_files} + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("resolve_merge_conflicts", e) +# [/DEF:resolve_merge_conflicts:Function] + + +# [DEF:abort_merge:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Abort unfinished merge from WebUI flow. +@router.post("/repositories/{dashboard_ref}/merge/abort") +async def abort_merge( + dashboard_ref: str, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("abort_merge"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + return _gs.abort_merge(dashboard_id) + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("abort_merge", e) +# [/DEF:abort_merge:Function] + + +# [DEF:continue_merge:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Finalize unfinished merge from WebUI flow. +# @RELATION: CALLS -> [GitService] +@router.post("/repositories/{dashboard_ref}/merge/continue") +async def continue_merge( + dashboard_ref: str, + continue_data: MergeContinueRequest, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("continue_merge"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + return _gs.continue_merge(dashboard_id, continue_data.message) + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("continue_merge", e) +# [/DEF:continue_merge:Function] +# [/DEF:GitMergeRoutes:Module] diff --git a/backend/src/api/routes/git/_repo_lifecycle_routes.py b/backend/src/api/routes/git/_repo_lifecycle_routes.py new file mode 100644 index 00000000..ba26d0d8 --- /dev/null +++ b/backend/src/api/routes/git/_repo_lifecycle_routes.py @@ -0,0 +1,228 @@ +# [DEF:GitRepoLifecycleRoutes:Module] +# @COMPLEXITY: 3 +# @PURPOSE: FastAPI endpoints for Git lifecycle operations (sync, promote, deploy). +# @LAYER: API +# @SEMANTICS: git, lifecycle, sync, promote, deploy + +import typing +from typing import Optional + +from fastapi import Depends, HTTPException +from sqlalchemy.orm import Session + +from src.core.database import get_db +from src.core.logger import belief_scope, logger +from src.dependencies import get_config_manager, get_current_user, has_permission +from src.models.auth import User +from src.models.git import GitProvider, GitRepository + +from src.api.routes.git_schemas import ( + DeployRequest, + PromoteRequest, + PromoteResponse, +) + +from ._router import router +from ._helpers import ( + _apply_git_identity_from_profile, + _get_git_config_or_404, + _handle_unexpected_git_route_error, +) +from ._deps import get_git_service + + +# [DEF:sync_dashboard:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Sync dashboard state from Superset to Git using the GitPlugin. +# @RELATION: CALLS -> [GitPlugin] +@router.post("/repositories/{dashboard_ref}/sync") +async def sync_dashboard( + dashboard_ref: str, + env_id: Optional[str] = None, + source_env_id: typing.Optional[str] = None, + config_manager=Depends(get_config_manager), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + with belief_scope("sync_dashboard"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + from src.plugins.git_plugin import GitPlugin + + plugin = GitPlugin() + return await plugin.execute( + { + "operation": "sync", + "dashboard_id": dashboard_id, + "source_env_id": source_env_id, + } + ) + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("sync_dashboard", e) +# [/DEF:sync_dashboard:Function] + + +# [DEF:promote_dashboard:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Promote changes between branches via MR or direct merge. +# @RELATION: CALLS -> [GitPlugin] +@router.post("/repositories/{dashboard_ref}/promote", response_model=PromoteResponse) +async def promote_dashboard( + dashboard_ref: str, + payload: PromoteRequest, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("promote_dashboard"): + from . import _resolve_dashboard_id_from_ref + + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + db_repo = ( + db.query(GitRepository) + .filter(GitRepository.dashboard_id == dashboard_id) + .first() + ) + if not db_repo: + raise HTTPException( + status_code=404, + detail=f"Repository for dashboard {dashboard_ref} is not initialized", + ) + config = _get_git_config_or_404(db, db_repo.config_id) + + from_branch = payload.from_branch.strip() + to_branch = payload.to_branch.strip() + if not from_branch or not to_branch: + raise HTTPException( + status_code=400, detail="from_branch and to_branch are required" + ) + if from_branch == to_branch: + raise HTTPException( + status_code=400, detail="from_branch and to_branch must be different" + ) + + mode = (payload.mode or "mr").strip().lower() + if mode == "direct": + reason = (payload.reason or "").strip() + if not reason: + raise HTTPException( + status_code=400, detail="Direct promote requires non-empty reason" + ) + logger.warning( + "[promote_dashboard][PolicyViolation] Direct promote without MR by actor=unknown dashboard_ref=%s from=%s to=%s reason=%s", + dashboard_ref, + from_branch, + to_branch, + reason, + ) + _apply_git_identity_from_profile(dashboard_id, db, current_user) + result = _gs.promote_direct_merge( + dashboard_id=dashboard_id, + from_branch=from_branch, + to_branch=to_branch, + ) + return PromoteResponse( + mode="direct", + from_branch=from_branch, + to_branch=to_branch, + status=result.get("status", "merged"), + policy_violation=True, + ) + + title = (payload.title or "").strip() or f"Promote {from_branch} -> {to_branch}" + description = payload.description + if config.provider == GitProvider.GITEA: + pr = await _gs.create_gitea_pull_request( + server_url=config.url, + pat=config.pat, + remote_url=db_repo.remote_url, + from_branch=from_branch, + to_branch=to_branch, + title=title, + description=description, + ) + elif config.provider == GitProvider.GITHUB: + pr = await _gs.create_github_pull_request( + server_url=config.url, + pat=config.pat, + remote_url=db_repo.remote_url, + from_branch=from_branch, + to_branch=to_branch, + title=title, + description=description, + draft=payload.draft, + ) + elif config.provider == GitProvider.GITLAB: + pr = await _gs.create_gitlab_merge_request( + server_url=config.url, + pat=config.pat, + remote_url=db_repo.remote_url, + from_branch=from_branch, + to_branch=to_branch, + title=title, + description=description, + remove_source_branch=payload.remove_source_branch, + ) + else: + raise HTTPException( + status_code=501, + detail=f"Provider {config.provider} does not support promotion API", + ) + + return PromoteResponse( + mode="mr", + from_branch=from_branch, + to_branch=to_branch, + status=pr.get("status", "opened"), + url=pr.get("url"), + reference_id=str(pr.get("id")) if pr.get("id") is not None else None, + policy_violation=False, + ) +# [/DEF:promote_dashboard:Function] + + +# [DEF:deploy_dashboard:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Deploy dashboard from Git to a target environment. +# @RELATION: CALLS -> [GitPlugin] +@router.post("/repositories/{dashboard_ref}/deploy") +async def deploy_dashboard( + dashboard_ref: str, + deploy_data: DeployRequest, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + with belief_scope("deploy_dashboard"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + from src.plugins.git_plugin import GitPlugin + + plugin = GitPlugin() + return await plugin.execute( + { + "operation": "deploy", + "dashboard_id": dashboard_id, + "environment_id": deploy_data.environment_id, + } + ) + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("deploy_dashboard", e) +# [/DEF:deploy_dashboard:Function] +# [/DEF:GitRepoLifecycleRoutes:Module] diff --git a/backend/src/api/routes/git/_repo_operations_routes.py b/backend/src/api/routes/git/_repo_operations_routes.py new file mode 100644 index 00000000..6059d043 --- /dev/null +++ b/backend/src/api/routes/git/_repo_operations_routes.py @@ -0,0 +1,365 @@ +# [DEF:GitRepoOperationsRoutes:Module] +# @COMPLEXITY: 3 +# @PURPOSE: FastAPI endpoints for Git repository operations (commit, push, pull, status, diff, history). +# @LAYER: API +# @SEMANTICS: git, repository, operations, commit, push, pull, status + +from typing import List, Optional + +from fastapi import Depends, HTTPException +from sqlalchemy.orm import Session + +from src.core.database import get_db +from src.core.logger import belief_scope, logger +from src.dependencies import get_config_manager, get_current_user, has_permission +from src.models.auth import User +from src.models.git import GitRepository, GitServerConfig + +from src.api.routes.git_schemas import ( + CommitCreate, + CommitSchema, + RepoStatusBatchRequest, + RepoStatusBatchResponse, +) + +from ._router import router +from ._helpers import ( + _apply_git_identity_from_profile, + _build_no_repo_status_payload, + _handle_unexpected_git_route_error, + _resolve_repository_status, +) +from ._deps import get_git_service, MAX_REPOSITORY_STATUS_BATCH + + +# [DEF:commit_changes:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Stage and commit changes in the dashboard's repository. +@router.post("/repositories/{dashboard_ref}/commit") +async def commit_changes( + dashboard_ref: str, + commit_data: CommitCreate, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("commit_changes"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + _apply_git_identity_from_profile(dashboard_id, db, current_user) + _gs.commit_changes( + dashboard_id, commit_data.message, commit_data.files + ) + return {"status": "success"} + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("commit_changes", e) +# [/DEF:commit_changes:Function] + + +# [DEF:push_changes:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Push local commits to the remote repository. +@router.post("/repositories/{dashboard_ref}/push") +async def push_changes( + dashboard_ref: str, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("push_changes"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + _gs.push_changes(dashboard_id) + return {"status": "success"} + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("push_changes", e) +# [/DEF:push_changes:Function] + + +# [DEF:pull_changes:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Pull changes from the remote repository. +# @RELATION: CALLS -> [GitService] +@router.post("/repositories/{dashboard_ref}/pull") +async def pull_changes( + dashboard_ref: str, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("pull_changes"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + db_repo = None + config_url = None + config_provider = None + try: + db_repo_candidate = ( + db.query(GitRepository) + .filter(GitRepository.dashboard_id == dashboard_id) + .first() + ) + if getattr(db_repo_candidate, "config_id", None): + db_repo = db_repo_candidate + config_row = ( + db.query(GitServerConfig) + .filter(GitServerConfig.id == db_repo.config_id) + .first() + ) + if config_row: + config_url = config_row.url + config_provider = config_row.provider + except Exception as diagnostics_error: + logger.warning( + "[pull_changes][Action] Failed to load repository binding diagnostics for dashboard %s: %s", + dashboard_id, + diagnostics_error, + ) + logger.info( + "[pull_changes][Action] Route diagnostics dashboard_ref=%s env_id=%s resolved_dashboard_id=%s " + "binding_exists=%s binding_local_path=%s binding_remote_url=%s binding_config_id=%s config_provider=%s config_url=%s", + dashboard_ref, + env_id, + dashboard_id, + bool(db_repo), + (db_repo.local_path if db_repo else None), + (db_repo.remote_url if db_repo else None), + (db_repo.config_id if db_repo else None), + config_provider, + config_url, + ) + _apply_git_identity_from_profile(dashboard_id, db, current_user) + _gs.pull_changes(dashboard_id) + return {"status": "success"} + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("pull_changes", e) +# [/DEF:pull_changes:Function] + + +# [DEF:get_repository_status:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Get current Git status for a dashboard repository. +@router.get("/repositories/{dashboard_ref}/status") +async def get_repository_status( + dashboard_ref: str, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + with belief_scope("get_repository_status"): + from . import _resolve_dashboard_id_from_ref_async + + try: + dashboard_id = await _resolve_dashboard_id_from_ref_async( + dashboard_ref, config_manager, env_id + ) + return _resolve_repository_status(dashboard_id) + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("get_repository_status", e) +# [/DEF:get_repository_status:Function] + + +# [DEF:get_repository_status_batch:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Get Git statuses for multiple dashboard repositories in one request. +@router.post("/repositories/status/batch", response_model=RepoStatusBatchResponse) +async def get_repository_status_batch( + request: RepoStatusBatchRequest, + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + with belief_scope("get_repository_status_batch"): + dashboard_ids = list(dict.fromkeys(request.dashboard_ids)) + if len(dashboard_ids) > MAX_REPOSITORY_STATUS_BATCH: + logger.warning( + "[get_repository_status_batch][Action] Batch size %s exceeds limit %s. Truncating request.", + len(dashboard_ids), + MAX_REPOSITORY_STATUS_BATCH, + ) + dashboard_ids = dashboard_ids[:MAX_REPOSITORY_STATUS_BATCH] + + statuses = {} + for dashboard_id in dashboard_ids: + try: + statuses[str(dashboard_id)] = _resolve_repository_status(dashboard_id) + except HTTPException: + statuses[str(dashboard_id)] = { + **_build_no_repo_status_payload(), + "sync_state": "ERROR", + "sync_status": "ERROR", + } + except Exception as e: + logger.error( + f"[get_repository_status_batch][Coherence:Failed] Failed for dashboard {dashboard_id}: {e}" + ) + statuses[str(dashboard_id)] = { + **_build_no_repo_status_payload(), + "sync_state": "ERROR", + "sync_status": "ERROR", + } + return RepoStatusBatchResponse(statuses=statuses) +# [/DEF:get_repository_status_batch:Function] + + +# [DEF:get_repository_diff:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Get Git diff for a dashboard repository. +@router.get("/repositories/{dashboard_ref}/diff") +async def get_repository_diff( + dashboard_ref: str, + file_path: Optional[str] = None, + staged: bool = False, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("get_repository_diff"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + return _gs.get_diff(dashboard_id, file_path, staged) + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("get_repository_diff", e) +# [/DEF:get_repository_diff:Function] + + +# [DEF:get_history:Function] +# @COMPLEXITY: 2 +# @PURPOSE: View commit history for a dashboard's repository. +@router.get("/repositories/{dashboard_ref}/history", response_model=List[CommitSchema]) +async def get_history( + dashboard_ref: str, + limit: int = 50, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("get_history"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + return _gs.get_commit_history(dashboard_id, limit) + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("get_history", e) +# [/DEF:get_history:Function] + + +# [DEF:generate_commit_message:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Generate a suggested commit message using LLM. +# @RELATION: CALLS -> [GitService] +@router.post("/repositories/{dashboard_ref}/generate-message") +async def generate_commit_message( + dashboard_ref: str, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + db: Session = Depends(get_db), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("generate_commit_message"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + diff = _gs.get_diff(dashboard_id, staged=True) + if not diff: + diff = _gs.get_diff(dashboard_id, staged=False) + + if not diff: + return {"message": "No changes detected"} + + history_objs = _gs.get_commit_history(dashboard_id, limit=5) + history = [h.message for h in history_objs if hasattr(h, "message")] + + from ...services.llm_provider import LLMProviderService + from ...plugins.llm_analysis.service import LLMClient + from ...plugins.llm_analysis.models import LLMProviderType + from ...services.llm_prompt_templates import ( + DEFAULT_LLM_PROMPTS, + normalize_llm_settings, + resolve_bound_provider_id, + ) + + llm_service = LLMProviderService(db) + providers = llm_service.get_all_providers() + llm_settings = normalize_llm_settings( + config_manager.get_config().settings.llm + ) + bound_provider_id = resolve_bound_provider_id(llm_settings, "git_commit") + provider = next((p for p in providers if p.id == bound_provider_id), None) + if not provider: + provider = next((p for p in providers if p.is_active), None) + + if not provider: + raise HTTPException( + status_code=400, detail="No active LLM provider found" + ) + + api_key = llm_service.get_decrypted_api_key(provider.id) + client = LLMClient( + provider_type=LLMProviderType(provider.provider_type), + api_key=api_key, + base_url=provider.base_url, + default_model=provider.default_model, + ) + + from ...plugins.git.llm_extension import GitLLMExtension + + extension = GitLLMExtension(client) + git_prompt = llm_settings["prompts"].get( + "git_commit_prompt", + DEFAULT_LLM_PROMPTS["git_commit_prompt"], + ) + message = await extension.suggest_commit_message( + diff, + history, + prompt_template=git_prompt, + ) + return {"message": message} + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("generate_commit_message", e) +# [/DEF:generate_commit_message:Function] +# [/DEF:GitRepoOperationsRoutes:Module] diff --git a/backend/src/api/routes/git/_repo_routes.py b/backend/src/api/routes/git/_repo_routes.py new file mode 100644 index 00000000..3ccb6e98 --- /dev/null +++ b/backend/src/api/routes/git/_repo_routes.py @@ -0,0 +1,269 @@ +# [DEF:GitRepoRoutes:Module] +# @COMPLEXITY: 3 +# @PURPOSE: FastAPI endpoints for core Git repository operations (init, binding, branches, checkout). +# @LAYER: API +# @SEMANTICS: git, repository, routes, init, branches + +from typing import List, Optional + +from fastapi import Depends, HTTPException +from sqlalchemy.orm import Session + +from src.core.database import get_db +from src.core.logger import belief_scope, logger +from src.dependencies import get_config_manager, get_current_user, has_permission +from src.models.auth import User +from src.models.git import GitRepository, GitServerConfig + +from src.api.routes.git_schemas import ( + BranchCheckout, + BranchCreate, + BranchSchema, + RepoInitRequest, + RepositoryBindingSchema, +) + +from ._router import router +from ._helpers import ( + _apply_git_identity_from_profile, + _get_git_config_or_404, + _handle_unexpected_git_route_error, +) +from ._deps import get_git_service + + +# [DEF:init_repository:Function] +# @COMPLEXITY: 3 +# @PURPOSE: Link a dashboard to a Git repository and perform initial clone/init. +# @RELATION: CALLS -> [GitService] +@router.post("/repositories/{dashboard_ref}/init") +async def init_repository( + dashboard_ref: str, + init_data: RepoInitRequest, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + db: Session = Depends(get_db), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("init_repository"): + from . import _resolve_dashboard_id_from_ref, _resolve_repo_key_from_ref + + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + repo_key = _resolve_repo_key_from_ref( + dashboard_ref, dashboard_id, config_manager, env_id + ) + config = ( + db.query(GitServerConfig) + .filter(GitServerConfig.id == init_data.config_id) + .first() + ) + if not config: + raise HTTPException(status_code=404, detail="Git configuration not found") + + try: + logger.info( + f"[init_repository][Action] Initializing repo for dashboard {dashboard_id}" + ) + _gs.init_repo( + dashboard_id, + init_data.remote_url, + config.pat, + repo_key=repo_key, + default_branch=config.default_branch, + ) + + repo_path = _gs._get_repo_path(dashboard_id, repo_key=repo_key) + db_repo = ( + db.query(GitRepository) + .filter(GitRepository.dashboard_id == dashboard_id) + .first() + ) + if not db_repo: + db_repo = GitRepository( + dashboard_id=dashboard_id, + config_id=config.id, + remote_url=init_data.remote_url, + local_path=repo_path, + current_branch="dev", + ) + db.add(db_repo) + else: + db_repo.config_id = config.id + db_repo.remote_url = init_data.remote_url + db_repo.local_path = repo_path + db_repo.current_branch = "dev" + + db.commit() + logger.info( + f"[init_repository][Coherence:OK] Repository initialized for dashboard {dashboard_id}" + ) + return {"status": "success", "message": "Repository initialized"} + except Exception as e: + db.rollback() + logger.error( + f"[init_repository][Coherence:Failed] Failed to init repository: {e}" + ) + if isinstance(e, HTTPException): + raise + _handle_unexpected_git_route_error("init_repository", e) +# [/DEF:init_repository:Function] + + +# [DEF:get_repository_binding:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Return repository binding with provider metadata for selected dashboard. +@router.get("/repositories/{dashboard_ref}", response_model=RepositoryBindingSchema) +async def get_repository_binding( + dashboard_ref: str, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + db: Session = Depends(get_db), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + with belief_scope("get_repository_binding"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + db_repo = ( + db.query(GitRepository) + .filter(GitRepository.dashboard_id == dashboard_id) + .first() + ) + if not db_repo: + raise HTTPException( + status_code=404, detail="Repository not initialized" + ) + config = _get_git_config_or_404(db, db_repo.config_id) + return RepositoryBindingSchema( + dashboard_id=db_repo.dashboard_id, + config_id=db_repo.config_id, + provider=config.provider, + remote_url=db_repo.remote_url, + local_path=db_repo.local_path, + ) + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("get_repository_binding", e) +# [/DEF:get_repository_binding:Function] + + +# [DEF:delete_repository:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Delete local repository workspace and DB binding for selected dashboard. +@router.delete("/repositories/{dashboard_ref}") +async def delete_repository( + dashboard_ref: str, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("delete_repository"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + _gs.delete_repo(dashboard_id) + return {"status": "success"} + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("delete_repository", e) +# [/DEF:delete_repository:Function] + + +# [DEF:get_branches:Function] +# @COMPLEXITY: 2 +# @PURPOSE: List all branches for a dashboard's repository. +@router.get("/repositories/{dashboard_ref}/branches", response_model=List[BranchSchema]) +async def get_branches( + dashboard_ref: str, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("get_branches"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + return _gs.list_branches(dashboard_id) + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("get_branches", e) +# [/DEF:get_branches:Function] + + +# [DEF:create_branch:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Create a new branch in the dashboard's repository. +@router.post("/repositories/{dashboard_ref}/branches") +async def create_branch( + dashboard_ref: str, + branch_data: BranchCreate, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("create_branch"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + _apply_git_identity_from_profile(dashboard_id, db, current_user) + _gs.create_branch( + dashboard_id, branch_data.name, branch_data.from_branch + ) + return {"status": "success"} + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("create_branch", e) +# [/DEF:create_branch:Function] + + +# [DEF:checkout_branch:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Switch the dashboard's repository to a specific branch. +@router.post("/repositories/{dashboard_ref}/checkout") +async def checkout_branch( + dashboard_ref: str, + checkout_data: BranchCheckout, + env_id: Optional[str] = None, + config_manager=Depends(get_config_manager), + _=Depends(has_permission("plugin:git", "EXECUTE")), +): + _gs = get_git_service() + with belief_scope("checkout_branch"): + from . import _resolve_dashboard_id_from_ref + + try: + dashboard_id = _resolve_dashboard_id_from_ref( + dashboard_ref, config_manager, env_id + ) + _gs.checkout_branch(dashboard_id, checkout_data.name) + return {"status": "success"} + except HTTPException: + raise + except Exception as e: + _handle_unexpected_git_route_error("checkout_branch", e) +# [/DEF:checkout_branch:Function] +# [/DEF:GitRepoRoutes:Module] diff --git a/backend/src/api/routes/git/_router.py b/backend/src/api/routes/git/_router.py new file mode 100644 index 00000000..f720fda8 --- /dev/null +++ b/backend/src/api/routes/git/_router.py @@ -0,0 +1,10 @@ +# [DEF:GitRouter:Module] +# @COMPLEXITY: 1 +# @PURPOSE: Shared APIRouter for all Git route modules. +# @LAYER: API +# @SEMANTICS: git, routes, fastapi, router + +from fastapi import APIRouter + +router = APIRouter(tags=["git"]) +# [/DEF:GitRouter:Module] diff --git a/backend/src/api/routes/translate/__init__.py b/backend/src/api/routes/translate/__init__.py new file mode 100644 index 00000000..e50ce3b5 --- /dev/null +++ b/backend/src/api/routes/translate/__init__.py @@ -0,0 +1,37 @@ +# [DEF:TranslateRoutes:Module] +# @COMPLEXITY: 4 +# @SEMANTICS: api, routes, translate +# @PURPOSE: API routes for LLM-based SQL/dashboard translation management including terminology dictionary CRUD and import. +# @LAYER: UI (API) +# @RELATION: DEPENDS_ON -> [TranslateJobResponse] +# @RELATION: DEPENDS_ON -> [DictionaryManager:Class] +# @RELATION: DEPENDS_ON -> [get_current_user] +# @RELATION: DEPENDS_ON -> [get_db] +# @RELATION: DEPENDS_ON -> [has_permission] +# @RELATION: DEPENDS_ON -> [ConfigManager] +# @RELATION: DEPENDS_ON -> [SupersetClient] +# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only. +# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity. + +""" +Translate routes package. + +Re-exports the shared router and all route handler modules. +All sub-modules register their handlers on the router at import time. +""" + +from ._router import router +from . import _job_routes # noqa: F401 — registers job handlers +from . import _run_routes # noqa: F401 — registers run handlers +from . import _run_list_routes # noqa: F401 — registers run list/detail/csv handlers +from . import _preview_routes # noqa: F401 — registers preview handlers +from . import _dictionary_routes # noqa: F401 — registers dictionary handlers +from . import _schedule_routes # noqa: F401 — registers schedule handlers +from . import _metrics_routes # noqa: F401 — registers metrics handlers +from . import _correction_routes # noqa: F401 — registers correction handlers + +__all__ = [ + "router", +] + +# [/DEF:TranslateRoutes:Module] diff --git a/backend/src/api/routes/translate/_correction_routes.py b/backend/src/api/routes/translate/_correction_routes.py new file mode 100644 index 00000000..59593be2 --- /dev/null +++ b/backend/src/api/routes/translate/_correction_routes.py @@ -0,0 +1,97 @@ +# [DEF:TranslateCorrectionRoutesModule:Module] +# @COMPLEXITY: 2 +# @PURPOSE: Term correction submission endpoints. +# @LAYER: API +# @SEMANTICS: api, routes, translate, corrections + +from fastapi import APIRouter, Depends, HTTPException, status, Query +from typing import Any, Dict, List, Optional +from sqlalchemy.orm import Session + +from ....core.database import get_db +from ....core.logger import logger, belief_scope +from ....schemas.auth import User +from ....dependencies import get_current_user, has_permission +from ....plugins.translate.dictionary import DictionaryManager +from ....schemas.translate import ( + TermCorrectionSubmit, + TermCorrectionBulkSubmit, + CorrectionSubmitResponse, +) + +from ._router import router + + +# ============================================================ +# Corrections +# ============================================================ + +# [DEF:submit_correction:Function] +# @PURPOSE: Submit a term correction from a run result review. +# @PRE: User has translate.dictionary.edit permission. +# @POST: Correction applied with conflict detection. +@router.post("/corrections") +async def submit_correction( + payload: TermCorrectionSubmit, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.dictionary", "EDIT")), + db: Session = Depends(get_db), +): + """Submit a term correction from a run result review.""" + logger.info(f"[translate_routes][submit_correction] User: {current_user.username}") + if not payload.dictionary_id: + raise HTTPException(status_code=422, detail="dictionary_id is required") + try: + result = DictionaryManager.submit_correction( + db, + dict_id=payload.dictionary_id, + source_term=payload.source_term, + incorrect_target_term=payload.incorrect_target_term, + corrected_target_term=payload.corrected_target_term, + origin_run_id=payload.origin_run_id, + origin_row_key=payload.origin_row_key, + origin_user_id=current_user.username, + ) + return result + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# [/DEF:submit_correction:Function] + + +# [DEF:submit_bulk_corrections:Function] +# @PURPOSE: Submit multiple term corrections atomically. +# @PRE: User has translate.dictionary.edit permission. +# @POST: All corrections applied or none with conflict list. +@router.post("/corrections/bulk") +async def submit_bulk_corrections( + payload: TermCorrectionBulkSubmit, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.dictionary", "EDIT")), + db: Session = Depends(get_db), +): + """Submit multiple term corrections atomically.""" + logger.info(f"[translate_routes][submit_bulk_corrections] User: {current_user.username}") + try: + corr_list = [] + for c in payload.corrections: + corr_list.append({ + "source_term": c.source_term, + "incorrect_target_term": c.incorrect_target_term, + "corrected_target_term": c.corrected_target_term, + "origin_run_id": c.origin_run_id, + "origin_row_key": c.origin_row_key, + }) + result = DictionaryManager.submit_bulk_corrections( + db, + dict_id=payload.dictionary_id, + corrections=corr_list, + origin_user_id=current_user.username, + ) + if result.get("status") == "conflicts": + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=result) + return result + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# [/DEF:submit_bulk_corrections:Function] + +# [/DEF:TranslateCorrectionRoutesModule:Module] diff --git a/backend/src/api/routes/translate/_dictionary_routes.py b/backend/src/api/routes/translate/_dictionary_routes.py new file mode 100644 index 00000000..be3f139d --- /dev/null +++ b/backend/src/api/routes/translate/_dictionary_routes.py @@ -0,0 +1,356 @@ +# [DEF:TranslateDictionaryRoutesModule:Module] +# @COMPLEXITY: 3 +# @PURPOSE: Terminology Dictionary CRUD, entries management, and import routes. +# @LAYER: API +# @SEMANTICS: api, routes, translate, dictionaries + +from fastapi import APIRouter, Depends, HTTPException, status, Query +from typing import Any, Dict, List, Optional +from sqlalchemy.orm import Session + +from ....core.database import get_db +from ....core.logger import logger, belief_scope +from ....schemas.auth import User +from ....dependencies import get_current_user, has_permission +from ....plugins.translate.dictionary import DictionaryManager +from ....schemas.translate import ( + DictionaryCreate, + DictionaryEntryCreate, + DictionaryEntryResponse, + DictionaryImport, + DictionaryResponse, +) + +from ._router import router +from ._helpers import _dict_to_response, _get_dictionary_entry_counts + + +# ============================================================ +# Terminology Dictionaries +# ============================================================ + +# [DEF:list_dictionaries:Function] +# @PURPOSE: List all terminology dictionaries. +# @PRE: User has translate.dictionary.view permission. +# @POST: Returns list of dictionaries with total count. +@router.get("/dictionaries") +async def list_dictionaries( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.dictionary", "VIEW")), + db: Session = Depends(get_db), +): + """List all terminology dictionaries.""" + with belief_scope("list_dictionaries"): + logger.reason(f"User: {current_user.username}") + dicts, total = DictionaryManager.list_dictionaries(db, page=page, page_size=page_size) + dict_ids = [d.id for d in dicts] + counts = _get_dictionary_entry_counts(db, dict_ids) if dict_ids else {} + items = [_dict_to_response(d, counts.get(d.id, 0)) for d in dicts] + logger.reflect("Dictionaries listed", {"total": total, "returned": len(items)}) + return {"items": items, "total": total, "page": page, "page_size": page_size} +# [/DEF:list_dictionaries:Function] + + +# [DEF:get_dictionary:Function] +# @PURPOSE: Get a single terminology dictionary by ID. +# @PRE: User has translate.dictionary.view permission. +# @POST: Returns the dictionary with entry_count. +@router.get("/dictionaries/{dictionary_id}") +async def get_dictionary( + dictionary_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.dictionary", "VIEW")), + db: Session = Depends(get_db), +): + """Get a terminology dictionary by ID.""" + with belief_scope("get_dictionary"): + logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") + try: + d = DictionaryManager.get_dictionary(db, dictionary_id) + from ...models.translate import DictionaryEntry + entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count() + logger.reflect("Dictionary found", {"id": dictionary_id}) + return _dict_to_response(d, entry_count) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:get_dictionary:Function] + + +# [DEF:create_dictionary:Function] +# @PURPOSE: Create a new terminology dictionary. +# @PRE: User has translate.dictionary.create permission. +# @POST: Returns the created dictionary. +@router.post("/dictionaries", status_code=status.HTTP_201_CREATED) +async def create_dictionary( + payload: DictionaryCreate, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.dictionary", "CREATE")), + db: Session = Depends(get_db), +): + """Create a new terminology dictionary.""" + with belief_scope("create_dictionary"): + logger.reason(f"User: {current_user.username}") + d = DictionaryManager.create_dictionary( + db, + name=payload.name, + source_dialect=payload.source_dialect, + target_dialect=payload.target_dialect, + created_by=current_user.username, + description=payload.description, + is_active=payload.is_active, + ) + logger.reflect("Dictionary created", {"id": d.id}) + return _dict_to_response(d, 0) +# [/DEF:create_dictionary:Function] + + +# [DEF:update_dictionary:Function] +# @PURPOSE: Update an existing terminology dictionary. +# @PRE: User has translate.dictionary.edit permission. +# @POST: Returns the updated dictionary. +@router.put("/dictionaries/{dictionary_id}") +async def update_dictionary( + dictionary_id: str, + payload: DictionaryCreate, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.dictionary", "EDIT")), + db: Session = Depends(get_db), +): + """Update a terminology dictionary.""" + with belief_scope("update_dictionary"): + logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") + try: + d = DictionaryManager.update_dictionary( + db, + dict_id=dictionary_id, + name=payload.name, + description=payload.description, + source_dialect=payload.source_dialect, + target_dialect=payload.target_dialect, + is_active=payload.is_active, + ) + from ...models.translate import DictionaryEntry + entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count() + logger.reflect("Dictionary updated", {"id": dictionary_id}) + return _dict_to_response(d, entry_count) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:update_dictionary:Function] + + +# [DEF:delete_dictionary:Function] +# @PURPOSE: Delete a terminology dictionary, blocked if attached to active/scheduled jobs. +# @PRE: User has translate.dictionary.delete permission. +# @POST: Dictionary is deleted. +@router.delete("/dictionaries/{dictionary_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_dictionary( + dictionary_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.dictionary", "DELETE")), + db: Session = Depends(get_db), +): + """Delete a terminology dictionary.""" + with belief_scope("delete_dictionary"): + logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") + try: + DictionaryManager.delete_dictionary(db, dictionary_id) + logger.reflect("Dictionary deleted", {"id": dictionary_id}) + return None + except ValueError as e: + if "active/scheduled" in str(e): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e)) + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:delete_dictionary:Function] + + +# ============================================================ +# Dictionary Entries CRUD +# ============================================================ + +# [DEF:list_dictionary_entries:Function] +# @PURPOSE: List entries for a dictionary. +# @PRE: User has translate.dictionary.view permission. +# @POST: Returns paginated list of entries. +@router.get("/dictionaries/{dictionary_id}/entries") +async def list_dictionary_entries( + dictionary_id: str, + page: int = Query(1, ge=1), + page_size: int = Query(50, ge=1, le=500), + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.dictionary", "VIEW")), + db: Session = Depends(get_db), +): + """List entries for a dictionary.""" + with belief_scope("list_dictionary_entries"): + logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") + try: + entries, total = DictionaryManager.list_entries(db, dictionary_id, page=page, page_size=page_size) + items = [ + { + "id": e.id, + "dictionary_id": e.dictionary_id, + "source_term": e.source_term, + "source_term_normalized": e.source_term_normalized, + "target_term": e.target_term, + "context_notes": e.context_notes, + "created_at": e.created_at, + "updated_at": e.updated_at, + } + for e in entries + ] + logger.reflect("Entries listed", {"dict_id": dictionary_id, "total": total}) + return {"items": items, "total": total, "page": page, "page_size": page_size} + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:list_dictionary_entries:Function] + + +# [DEF:add_dictionary_entry:Function] +# @PURPOSE: Add a new entry to a dictionary. +# @PRE: User has translate.dictionary.edit permission. +# @POST: Entry is created. +@router.post("/dictionaries/{dictionary_id}/entries", status_code=status.HTTP_201_CREATED) +async def add_dictionary_entry( + dictionary_id: str, + payload: DictionaryEntryCreate, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.dictionary", "EDIT")), + db: Session = Depends(get_db), +): + """Add a new entry to a dictionary.""" + with belief_scope("add_dictionary_entry"): + logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") + try: + entry = DictionaryManager.add_entry( + db, dictionary_id, + source_term=payload.source_term, + target_term=payload.target_term, + context_notes=payload.context_notes, + ) + logger.reflect("Entry added", {"entry_id": entry.id}) + return { + "id": entry.id, + "dictionary_id": entry.dictionary_id, + "source_term": entry.source_term, + "source_term_normalized": entry.source_term_normalized, + "target_term": entry.target_term, + "context_notes": entry.context_notes, + "created_at": entry.created_at, + "updated_at": entry.updated_at, + } + except ValueError as e: + if "already exists" in str(e): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e)) + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:add_dictionary_entry:Function] + + +# [DEF:edit_dictionary_entry:Function] +# @PURPOSE: Update an existing dictionary entry. +# @PRE: User has translate.dictionary.edit permission. +# @POST: Entry is updated. +@router.put("/dictionaries/{dictionary_id}/entries/{entry_id}") +async def edit_dictionary_entry( + dictionary_id: str, + entry_id: str, + payload: DictionaryEntryCreate, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.dictionary", "EDIT")), + db: Session = Depends(get_db), +): + """Update an existing dictionary entry.""" + with belief_scope("edit_dictionary_entry"): + logger.reason(f"Entry: {entry_id}, User: {current_user.username}") + try: + entry = DictionaryManager.edit_entry( + db, entry_id, + source_term=payload.source_term, + target_term=payload.target_term, + context_notes=payload.context_notes, + ) + logger.reflect("Entry updated", {"entry_id": entry_id}) + return { + "id": entry.id, + "dictionary_id": entry.dictionary_id, + "source_term": entry.source_term, + "source_term_normalized": entry.source_term_normalized, + "target_term": entry.target_term, + "context_notes": entry.context_notes, + "created_at": entry.created_at, + "updated_at": entry.updated_at, + } + except ValueError as e: + if "already exists" in str(e): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e)) + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:edit_dictionary_entry:Function] + + +# [DEF:delete_dictionary_entry:Function] +# @PURPOSE: Delete a dictionary entry. +# @PRE: User has translate.dictionary.edit permission. +# @POST: Entry is deleted. +@router.delete("/dictionaries/{dictionary_id}/entries/{entry_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_dictionary_entry( + dictionary_id: str, + entry_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.dictionary", "EDIT")), + db: Session = Depends(get_db), +): + """Delete a dictionary entry.""" + with belief_scope("delete_dictionary_entry"): + logger.reason(f"Entry: {entry_id}, User: {current_user.username}") + try: + DictionaryManager.delete_entry(db, entry_id) + logger.reflect("Entry deleted", {"entry_id": entry_id}) + return None + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:delete_dictionary_entry:Function] + + +# [DEF:import_dictionary_entries:Function] +# @PURPOSE: Import entries into a terminology dictionary from CSV/TSV content. +# @PRE: User has translate.dictionary.edit permission. +# @POST: Entries are imported per conflict mode. +@router.post("/dictionaries/{dictionary_id}/import") +async def import_dictionary_entries( + dictionary_id: str, + payload: DictionaryImport, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.dictionary", "EDIT")), + db: Session = Depends(get_db), +): + """Import entries into a terminology dictionary from CSV/TSV content.""" + with belief_scope("import_dictionary_entries"): + logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") + + if payload.on_conflict not in ("overwrite", "keep_existing", "cancel"): + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="on_conflict must be 'overwrite', 'keep_existing', or 'cancel'", + ) + + try: + result = DictionaryManager.import_entries( + db, + dict_id=dictionary_id, + content=payload.content, + delimiter=payload.delimiter, + on_conflict=payload.on_conflict, + preview_only=payload.preview_only, + ) + logger.reflect("Import completed", { + "dict_id": dictionary_id, + "total": result["total"], + "errors": len(result["errors"]), + }) + return result + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# [/DEF:import_dictionary_entries:Function] + +# [/DEF:TranslateDictionaryRoutesModule:Module] diff --git a/backend/src/api/routes/translate/_helpers.py b/backend/src/api/routes/translate/_helpers.py new file mode 100644 index 00000000..6fb28faa --- /dev/null +++ b/backend/src/api/routes/translate/_helpers.py @@ -0,0 +1,75 @@ +# [DEF:TranslateHelpersModule:Module] +# @COMPLEXITY: 2 +# @PURPOSE: Shared helper functions for translate route handlers. +# @LAYER: API +# @SEMANTICS: api, routes, translate, helpers +# @RELATION: DEPENDS_ON -> [models.translate] + +from typing import Any, Dict, List +from sqlalchemy.orm import Session + + +# [DEF:_run_to_response:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Convert TranslationRun ORM to response dict. +def _run_to_response(run: Any) -> dict: + return { + "id": run.id, + "job_id": run.job_id, + "status": run.status, + "trigger_type": run.trigger_type, + "started_at": run.started_at.isoformat() if run.started_at else None, + "completed_at": run.completed_at.isoformat() if run.completed_at else None, + "error_message": run.error_message, + "total_records": run.total_records or 0, + "successful_records": run.successful_records or 0, + "failed_records": run.failed_records or 0, + "skipped_records": run.skipped_records or 0, + "insert_status": run.insert_status, + "superset_execution_id": run.superset_execution_id, + "config_snapshot": run.config_snapshot, + "key_hash": run.key_hash, + "config_hash": run.config_hash, + "dict_snapshot_hash": run.dict_snapshot_hash, + "created_by": run.created_by, + "created_at": run.created_at.isoformat() if run.created_at else None, + } +# [/DEF:_run_to_response:Function] + + +# [DEF:_dict_to_response:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Convert a TerminologyDictionary ORM model to a response dict with computed entry_count. +@staticmethod +def _dict_to_response(d: Any, entry_count: int = 0) -> dict: + return { + "id": d.id, + "name": d.name, + "description": d.description, + "source_dialect": d.source_dialect, + "target_dialect": d.target_dialect, + "is_active": d.is_active, + "created_by": d.created_by, + "created_at": d.created_at, + "updated_at": d.updated_at, + "entry_count": entry_count, + } +# [/DEF:_dict_to_response:Function] + + +# [DEF:_get_dictionary_entry_counts:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Get entry counts for a list of dictionary IDs. +def _get_dictionary_entry_counts(db: Session, dict_ids: List[str]) -> Dict[str, int]: + from sqlalchemy import func + from ...models.translate import DictionaryEntry + counts = ( + db.query(DictionaryEntry.dictionary_id, func.count(DictionaryEntry.id)) + .filter(DictionaryEntry.dictionary_id.in_(dict_ids)) + .group_by(DictionaryEntry.dictionary_id) + .all() + ) + return {row[0]: row[1] for row in counts} +# [/DEF:_get_dictionary_entry_counts:Function] + +# [/DEF:TranslateHelpersModule:Module] diff --git a/backend/src/api/routes/translate/_job_routes.py b/backend/src/api/routes/translate/_job_routes.py new file mode 100644 index 00000000..fdeafde5 --- /dev/null +++ b/backend/src/api/routes/translate/_job_routes.py @@ -0,0 +1,250 @@ +# [DEF:TranslateJobRoutesModule:Module] +# @COMPLEXITY: 3 +# @PURPOSE: Translation Job CRUD and datasource column routes. +# @LAYER: API +# @SEMANTICS: api, routes, translate, jobs + +from fastapi import APIRouter, Depends, HTTPException, status, Query +from typing import Any, Dict, List, Optional +from sqlalchemy.orm import Session + +from ....core.database import get_db +from ....core.logger import logger, belief_scope +from ....schemas.auth import User +from ....dependencies import get_current_user, has_permission, get_config_manager +from ....core.config_manager import ConfigManager +from ....plugins.translate.service import ( + TranslateJobService, + job_to_response, + get_datasource_columns as fetch_datasource_columns_service, +) +from ....schemas.translate import ( + TranslateJobCreate, + TranslateJobUpdate, + TranslateJobResponse, + DatasourceColumnsResponse, + DuplicateJobResponse, +) + +from ._router import router + + +# ============================================================ +# Translation Job CRUD +# ============================================================ + +# [DEF:list_jobs:Function] +# @PURPOSE: List all translation jobs. +# @PRE: User has translate.job.view permission. +# @POST: Returns list of translation jobs. +@router.get("/jobs", response_model=List[TranslateJobResponse]) +async def list_jobs( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + status: Optional[str] = Query(None), + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "VIEW")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """List all translation jobs.""" + logger.info(f"[translate_routes][list_jobs] User: {current_user.username}") + try: + service = TranslateJobService(db, config_manager, current_user.username) + total, jobs = service.list_jobs( + page=page, + page_size=page_size, + status_filter=status, + ) + results = [] + for job in jobs: + dict_ids = service.get_job_dictionary_ids(job.id) + results.append(job_to_response(job, dict_ids)) + return results + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# [/DEF:list_jobs:Function] + + +# [DEF:get_job:Function] +# @PURPOSE: Get a single translation job by ID. +# @PRE: User has translate.job.view permission. +# @POST: Returns the translation job. +@router.get("/jobs/{job_id}", response_model=TranslateJobResponse) +async def get_job( + job_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "VIEW")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Get a translation job by ID.""" + logger.info(f"[translate_routes][get_job] Job: {job_id}, User: {current_user.username}") + try: + service = TranslateJobService(db, config_manager, current_user.username) + job = service.get_job(job_id) + dict_ids = service.get_job_dictionary_ids(job.id) + return job_to_response(job, dict_ids) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:get_job:Function] + + +# [DEF:create_job:Function] +# @PURPOSE: Create a new translation job. +# @PRE: User has translate.job.create permission. +# @POST: Returns the created translation job. +# @SIDE_EFFECT: Validates columns via SupersetClient; caches database_dialect. +@router.post("/jobs", response_model=TranslateJobResponse, status_code=status.HTTP_201_CREATED) +async def create_job( + payload: TranslateJobCreate, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "CREATE")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Create a new translation job.""" + logger.info(f"[translate_routes][create_job] User: {current_user.username}") + try: + service = TranslateJobService(db, config_manager, current_user.username) + job = service.create_job(payload) + dict_ids = service.get_job_dictionary_ids(job.id) + return job_to_response(job, dict_ids) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)) +# [/DEF:create_job:Function] + + +# [DEF:update_job:Function] +# @PURPOSE: Update an existing translation job. +# @PRE: User has translate.job.edit permission. +# @POST: Returns the updated translation job. +# @SIDE_EFFECT: Re-detects database_dialect if datasource changed. +@router.put("/jobs/{job_id}", response_model=TranslateJobResponse) +async def update_job( + job_id: str, + payload: TranslateJobUpdate, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "EDIT")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Update a translation job.""" + logger.info(f"[translate_routes][update_job] Job: {job_id}, User: {current_user.username}") + try: + service = TranslateJobService(db, config_manager, current_user.username) + job = service.update_job(job_id, payload) + dict_ids = service.get_job_dictionary_ids(job.id) + return job_to_response(job, dict_ids) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:update_job:Function] + + +# [DEF:delete_job:Function] +# @PURPOSE: Delete a translation job. +# @PRE: User has translate.job.delete permission. +# @POST: Job is deleted. +@router.delete("/jobs/{job_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_job( + job_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "DELETE")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Delete a translation job.""" + logger.info(f"[translate_routes][delete_job] Job: {job_id}, User: {current_user.username}") + try: + service = TranslateJobService(db, config_manager, current_user.username) + service.delete_job(job_id) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:delete_job:Function] + + +# [DEF:duplicate_job:Function] +# @PURPOSE: Duplicate a translation job. +# @PRE: User has translate.job.create permission. +# @POST: Returns the duplicated job with status DRAFT. +@router.post("/jobs/{job_id}/duplicate", response_model=DuplicateJobResponse, status_code=status.HTTP_201_CREATED) +async def duplicate_job( + job_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "CREATE")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Duplicate a translation job.""" + logger.info(f"[translate_routes][duplicate_job] Job: {job_id}, User: {current_user.username}") + try: + service = TranslateJobService(db, config_manager, current_user.username) + new_job = service.duplicate_job(job_id) + return DuplicateJobResponse( + id=new_job.id, + name=new_job.name, + message="Job duplicated successfully", + ) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:duplicate_job:Function] + + +# [DEF:get_datasource_columns:Function] +# @PURPOSE: Get column metadata and database dialect for a Superset datasource. +# @PRE: User has translate.job.view permission. +# @POST: Returns column list with metadata and database_dialect. +# @SIDE_EFFECT: Queries Superset API for dataset detail and database info. +# @RATIONALE: Dialect detection from Superset connection cached on TranslationJob at save time. +# @REJECTED: Hardcoding dialect list per database — would drift from actual Superset config. + +@router.get("/datasources") +async def list_datasources( + env_id: str = Query(..., description="Superset environment ID"), + search: Optional[str] = Query(None, description="Search by table name"), + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "VIEW")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """List all Superset datasets available for translation jobs.""" + with belief_scope("list_datasources"): + try: + service = TranslateJobService(db, config_manager) + datasets = service.fetch_available_datasources(env_id, search) + return datasets + except Exception as e: + logger.error(f"[translate_routes][list_datasources] Error: {e}") + raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(e)) + +@router.get("/datasources/{datasource_id}/columns", response_model=DatasourceColumnsResponse) +async def get_datasource_columns( + datasource_id: int, + env_id: str = Query(..., description="Superset environment ID"), + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "VIEW")), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Get column metadata and database dialect for a Superset datasource.""" + logger.info( + f"[translate_routes][get_datasource_columns] " + f"Datasource: {datasource_id}, Env: {env_id}, User: {current_user.username}" + ) + try: + result = fetch_datasource_columns_service( + datasource_id=datasource_id, + env_id=env_id, + config_manager=config_manager, + ) + return result + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + except Exception as e: + logger.error(f"[translate_routes][get_datasource_columns] Error: {e}") + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=f"Failed to fetch datasource columns from Superset: {e}", + ) +# [/DEF:get_datasource_columns:Function] + +# [/DEF:TranslateJobRoutesModule:Module] diff --git a/backend/src/api/routes/translate/_metrics_routes.py b/backend/src/api/routes/translate/_metrics_routes.py new file mode 100644 index 00000000..94b39fc6 --- /dev/null +++ b/backend/src/api/routes/translate/_metrics_routes.py @@ -0,0 +1,67 @@ +# [DEF:TranslateMetricsRoutesModule:Module] +# @COMPLEXITY: 2 +# @PURPOSE: Translation Metrics endpoints. +# @LAYER: API +# @SEMANTICS: api, routes, translate, metrics + +from fastapi import APIRouter, Depends, HTTPException, status, Query +from typing import Any, Dict, List, Optional +from sqlalchemy.orm import Session + +from ....core.database import get_db +from ....core.logger import logger, belief_scope +from ....schemas.auth import User +from ....dependencies import get_current_user, has_permission +from ....plugins.translate.metrics import TranslationMetrics + +from ._router import router + + +# ============================================================ +# Metrics +# ============================================================ + +# [DEF:get_metrics:Function] +# @PURPOSE: Get translation metrics, optionally filtered by job. +# @PRE: User has translate.metrics.view permission. +# @POST: Returns metrics data. +@router.get("/metrics") +async def get_metrics( + job_id: Optional[str] = Query(None), + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.metrics", "VIEW")), + db: Session = Depends(get_db), +): + """Get translation metrics.""" + logger.info(f"[translate_routes][get_metrics] User: {current_user.username}") + try: + metrics_svc = TranslationMetrics(db) + if job_id: + return metrics_svc.get_job_metrics(job_id) + return metrics_svc.get_all_metrics() + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:get_metrics:Function] + + +# [DEF:get_job_metrics:Function] +# @PURPOSE: Get aggregated metrics for a specific job. +# @PRE: User has translate.metrics.view permission. +# @POST: Returns metrics dict. +@router.get("/jobs/{job_id}/metrics") +async def get_job_metrics( + job_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.metrics", "VIEW")), + db: Session = Depends(get_db), +): + """Get aggregated metrics for a specific job.""" + logger.info(f"[translate_routes][get_job_metrics] Job: {job_id}, User: {current_user.username}") + try: + metrics_svc = TranslationMetrics(db) + return metrics_svc.get_job_metrics(job_id) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:get_job_metrics:Function] + +# [/DEF:TranslateMetricsRoutesModule:Module] diff --git a/backend/src/api/routes/translate/_preview_routes.py b/backend/src/api/routes/translate/_preview_routes.py new file mode 100644 index 00000000..53c84522 --- /dev/null +++ b/backend/src/api/routes/translate/_preview_routes.py @@ -0,0 +1,194 @@ +# [DEF:TranslatePreviewRoutesModule:Module] +# @COMPLEXITY: 3 +# @PURPOSE: Translation Preview session management routes. +# @LAYER: API +# @SEMANTICS: api, routes, translate, preview + +from fastapi import APIRouter, Depends, HTTPException, status, Query +from typing import Any, Dict, List, Optional +from sqlalchemy.orm import Session + +from ....core.database import get_db +from ....core.logger import logger, belief_scope +from ....schemas.auth import User +from ....dependencies import get_current_user, has_permission, get_config_manager +from ....core.config_manager import ConfigManager +from ....plugins.translate.preview import TranslationPreview +from ....schemas.translate import ( + PreviewRequest, + PreviewRowUpdate, + PreviewAcceptResponse, + PreviewRow, +) + +from ._router import router + + +# ============================================================ +# Preview +# ============================================================ + +# [DEF:preview_translation:Function] +# @PURPOSE: Preview a translation before applying it. +# @PRE: User has translate.job.execute permission. +# @POST: Returns a preview session with records and cost estimation. +# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows. +@router.post("/jobs/{job_id}/preview", status_code=status.HTTP_201_CREATED) +async def preview_translation( + job_id: str, + payload: PreviewRequest = None, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "EXECUTE")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Preview a translation before applying it.""" + logger.info(f"[translate_routes][preview_translation] Job: {job_id}, User: {current_user.username}") + if payload is None: + payload = PreviewRequest() + try: + preview_service = TranslationPreview(db, config_manager, current_user.username) + result = preview_service.preview_rows( + job_id=job_id, + sample_size=payload.sample_size, + prompt_template=payload.prompt_template, + ) + return result + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + except Exception as e: + logger.error(f"[translate_routes][preview_translation] Error: {e}") + raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Preview failed: {e}") +# [/DEF:preview_translation:Function] + + +# [DEF:update_preview_row:Function] +# @PURPOSE: Approve, edit, or reject a preview row. +# @PRE: User has translate.job.execute permission. +# @POST: Preview row status is updated. +@router.put("/jobs/{job_id}/preview/rows/{row_key}") +async def update_preview_row( + job_id: str, + row_key: str, + payload: PreviewRowUpdate, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "EXECUTE")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Approve, edit, or reject a preview row.""" + logger.info(f"[translate_routes][update_preview_row] Job: {job_id}, Row: {row_key}, Action: {payload.action}") + try: + preview_service = TranslationPreview(db, config_manager, current_user.username) + result = preview_service.update_preview_row( + job_id=job_id, + row_id=row_key, + action=payload.action, + translation=payload.translation, + feedback=payload.feedback, + ) + return result + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# [/DEF:update_preview_row:Function] + + +# [DEF:accept_preview_session:Function] +# @PURPOSE: Accept a preview session, marking it as the quality gate for full execution. +# @PRE: User has translate.job.execute permission. Job has an ACTIVE preview session. +# @POST: Preview session is marked as APPLIED; full execution can proceed. +@router.post("/jobs/{job_id}/preview/accept") +async def accept_preview_session( + job_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "EXECUTE")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Accept a preview session, enabling full translation execution.""" + logger.info(f"[translate_routes][accept_preview_session] Job: {job_id}, User: {current_user.username}") + try: + preview_service = TranslationPreview(db, config_manager, current_user.username) + result = preview_service.accept_preview_session(job_id=job_id) + return result + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# [/DEF:accept_preview_session:Function] + + +# [DEF:apply_preview:Function] +# @PURPOSE: Apply a preview session (alias for accept when accepting at session level). +# @PRE: User has translate.job.execute permission. +# @POST: Preview is applied. +@router.post("/preview/{session_id}/apply") +async def apply_preview( + session_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "EXECUTE")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Apply a preview session by session ID.""" + logger.info(f"[translate_routes][apply_preview] Session: {session_id}, User: {current_user.username}") + # Find job_id from session + from ...models.translate import TranslationPreviewSession + session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first() + if not session: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Preview session '{session_id}' not found") + try: + preview_service = TranslationPreview(db, config_manager, current_user.username) + result = preview_service.accept_preview_session(job_id=session.job_id) + return result + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# [/DEF:apply_preview:Function] + + +# ============================================================ +# Preview Records +# ============================================================ + +# [DEF:get_preview_records:Function] +# @PURPOSE: Get records for a preview session. +# @PRE: User has translate.job.view permission. +# @POST: Returns list of preview records. +@router.get("/preview/{session_id}/records", response_model=List[PreviewRow]) +async def get_preview_records( + session_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "VIEW")), + db: Session = Depends(get_db), +): + """Get records for a preview session.""" + logger.info(f"[translate_routes][get_preview_records] Session: {session_id}, User: {current_user.username}") + try: + from ...models.translate import TranslationPreviewSession, TranslationPreviewRecord + session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first() + if not session: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Preview session '{session_id}' not found") + records = ( + db.query(TranslationPreviewRecord) + .filter(TranslationPreviewRecord.session_id == session_id) + .all() + ) + return [ + PreviewRow( + id=r.id, + source_sql=r.source_sql, + target_sql=r.target_sql, + source_object_type=r.source_object_type, + source_object_id=r.source_object_id, + source_object_name=r.source_object_name, + status=r.status, + feedback=r.feedback, + ) + for r in records + ] + except HTTPException: + raise + except Exception as e: + logger.error(f"[translate_routes][get_preview_records] Error: {e}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# [/DEF:get_preview_records:Function] + +# [/DEF:TranslatePreviewRoutesModule:Module] diff --git a/backend/src/api/routes/translate/_router.py b/backend/src/api/routes/translate/_router.py new file mode 100644 index 00000000..c4eca686 --- /dev/null +++ b/backend/src/api/routes/translate/_router.py @@ -0,0 +1,14 @@ +# [DEF:TranslateRouterModule:Module] +# @COMPLEXITY: 1 +# @PURPOSE: APIRouter instance for translate routes. +# @LAYER: API +# @SEMANTICS: api, routes, translate, router + +from fastapi import APIRouter + +# [DEF:translate_router:Global] +# @PURPOSE: APIRouter instance for all translate sub-routes. +router = APIRouter(prefix="/api/translate", tags=["translate"]) +# [/DEF:translate_router:Global] + +# [/DEF:TranslateRouterModule:Module] diff --git a/backend/src/api/routes/translate/_run_list_routes.py b/backend/src/api/routes/translate/_run_list_routes.py new file mode 100644 index 00000000..e7f63bff --- /dev/null +++ b/backend/src/api/routes/translate/_run_list_routes.py @@ -0,0 +1,206 @@ +# [DEF:TranslateRunListRoutesModule:Module] +# @COMPLEXITY: 3 +# @PURPOSE: Translation Run listing, detail, and CSV download routes (cross-job). +# @LAYER: API +# @SEMANTICS: api, routes, translate, runs, list, detail + +from fastapi import APIRouter, Depends, HTTPException, status, Query +from typing import Any, Dict, List, Optional +from sqlalchemy.orm import Session + +from ....core.database import get_db +from ....core.logger import logger, belief_scope +from ....schemas.auth import User +from ....dependencies import get_current_user, has_permission, get_config_manager +from ....core.config_manager import ConfigManager +from ....plugins.translate.orchestrator import TranslationOrchestrator +from ....plugins.translate.events import TranslationEventLog + +from ._router import router + + +# ============================================================ +# List Runs (cross-job) +# ============================================================ + +# [DEF:list_runs:Function] +# @PURPOSE: List all runs with cross-job filtering and pagination. +# @PRE: User has translate.history.view permission. +# @POST: Returns paginated list of runs. +@router.get("/runs") +async def list_runs( + job_id: Optional[str] = Query(None), + run_status: Optional[str] = Query(None), + trigger_type: Optional[str] = Query(None), + created_by: Optional[str] = Query(None), + date_from: Optional[str] = Query(None), + date_to: Optional[str] = Query(None), + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.history", "VIEW")), + db: Session = Depends(get_db), +): + """List all runs with cross-job filtering and pagination.""" + logger.info(f"[translate_routes][list_runs] User: {current_user.username}") + try: + from ...models.translate import TranslationRun + query = db.query(TranslationRun) + + if job_id: + query = query.filter(TranslationRun.job_id == job_id) + if run_status: + query = query.filter(TranslationRun.status == run_status) + if trigger_type: + query = query.filter(TranslationRun.trigger_type == trigger_type) + if created_by: + query = query.filter(TranslationRun.created_by == created_by) + if date_from: + try: + from datetime import datetime + dt_from = datetime.fromisoformat(date_from) + query = query.filter(TranslationRun.created_at >= dt_from) + except ValueError: + pass + if date_to: + try: + from datetime import datetime + dt_to = datetime.fromisoformat(date_to) + query = query.filter(TranslationRun.created_at <= dt_to) + except ValueError: + pass + + total = query.count() + runs = ( + query.order_by(TranslationRun.created_at.desc()) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) + + items = [] + for r in runs: + items.append({ + "id": r.id, + "job_id": r.job_id, + "status": r.status, + "trigger_type": r.trigger_type, + "started_at": r.started_at.isoformat() if r.started_at else None, + "completed_at": r.completed_at.isoformat() if r.completed_at else None, + "error_message": r.error_message, + "total_records": r.total_records or 0, + "successful_records": r.successful_records or 0, + "failed_records": r.failed_records or 0, + "skipped_records": r.skipped_records or 0, + "insert_status": r.insert_status, + "superset_execution_id": r.superset_execution_id, + "config_hash": r.config_hash, + "dict_snapshot_hash": r.dict_snapshot_hash, + "created_by": r.created_by, + "created_at": r.created_at.isoformat() if r.created_at else None, + }) + + return {"items": items, "total": total, "page": page, "page_size": page_size} + except Exception as e: + logger.error(f"[translate_routes][list_runs] Error: {e}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# [/DEF:list_runs:Function] + + +# ============================================================ +# Run Detail +# ============================================================ + +# [DEF:get_run_detail:Function] +# @PURPOSE: Get detailed run info with config_snapshot, records, events. +# @PRE: User has translate.history.view permission. +# @POST: Returns run detail with records and events. +@router.get("/runs/{run_id}/detail") +async def get_run_detail( + run_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.history", "VIEW")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Get detailed run info with config_snapshot, records, events.""" + logger.info(f"[translate_routes][get_run_detail] Run: {run_id}, User: {current_user.username}") + try: + orch = TranslationOrchestrator(db, config_manager, current_user.username) + status_info = orch.get_run_status(run_id) + records = orch.get_run_records(run_id, page=1, page_size=50) + events = TranslationEventLog(db).query_events(run_id=run_id) + event_summary = TranslationEventLog(db).get_run_event_summary(run_id) + + # Get config snapshot from run + from ...models.translate import TranslationRun + run = db.query(TranslationRun).filter(TranslationRun.id == run_id).first() + config_snapshot = run.config_snapshot if run else None + + return { + **status_info, + "config_snapshot": config_snapshot, + "config_hash": run.config_hash if run else None, + "records": records, + "events": events, + "event_invariants": event_summary, + } + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:get_run_detail:Function] + + +# ============================================================ +# Download Skipped CSV +# ============================================================ + +# [DEF:download_skipped_csv:Function] +# @PURPOSE: Download a CSV of skipped translation records from a run. +# @PRE: User has translate.job.view permission. +# @POST: Returns CSV file of skipped records. +@router.get("/runs/{run_id}/skipped.csv") +async def download_skipped_csv( + run_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "VIEW")), + db: Session = Depends(get_db), +): + """Download skipped translation records as CSV.""" + logger.info(f"[translate_routes][download_skipped_csv] Run: {run_id}, User: {current_user.username}") + try: + from ...models.translate import TranslationRecord + from fastapi.responses import StreamingResponse + import csv + import io + + records = ( + db.query(TranslationRecord) + .filter( + TranslationRecord.run_id == run_id, + TranslationRecord.status == "SKIPPED", + ) + .all() + ) + + output = io.StringIO() + writer = csv.writer(output) + writer.writerow(["id", "source_sql", "target_sql", "source_object_type", + "source_object_id", "source_object_name", "error_message"]) + for rec in records: + writer.writerow([ + rec.id, rec.source_sql, rec.target_sql, rec.source_object_type, + rec.source_object_id, rec.source_object_name, rec.error_message, + ]) + + output.seek(0) + return StreamingResponse( + iter([output.getvalue()]), + media_type="text/csv", + headers={"Content-Disposition": f"attachment; filename=skipped_{run_id}.csv"}, + ) + except Exception as e: + logger.error(f"[translate_routes][download_skipped_csv] Error: {e}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# [/DEF:download_skipped_csv:Function] + +# [/DEF:TranslateRunListRoutesModule:Module] diff --git a/backend/src/api/routes/translate/_run_routes.py b/backend/src/api/routes/translate/_run_routes.py new file mode 100644 index 00000000..a596d1c0 --- /dev/null +++ b/backend/src/api/routes/translate/_run_routes.py @@ -0,0 +1,252 @@ +# [DEF:TranslateRunRoutesModule:Module] +# @COMPLEXITY: 3 +# @PURPOSE: Translation Run execution, history, status, records and batches routes. +# @LAYER: API +# @SEMANTICS: api, routes, translate, runs + +from fastapi import APIRouter, Depends, HTTPException, status, Query +from typing import Any, Dict, List, Optional +from sqlalchemy.orm import Session + +from ....core.database import get_db +from ....core.logger import logger, belief_scope +from ....schemas.auth import User +from ....dependencies import get_current_user, has_permission, get_config_manager +from ....core.config_manager import ConfigManager +from ....plugins.translate.orchestrator import TranslationOrchestrator +from ....plugins.translate.events import TranslationEventLog + +from ._router import router +from ._helpers import _run_to_response + + +# ============================================================ +# Translation Run / Execute +# ============================================================ + +# [DEF:run_translation:Function] +# @PURPOSE: Execute a translation job (trigger a run). +# @PRE: User has translate.job.execute permission. +# @POST: Returns the created translation run. +@router.post("/jobs/{job_id}/run", status_code=status.HTTP_201_CREATED) +async def run_translation( + job_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "EXECUTE")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Execute a translation job (trigger a run).""" + logger.info(f"[translate_routes][run_translation] Job: {job_id}, User: {current_user.username}") + try: + orch = TranslationOrchestrator(db, config_manager, current_user.username) + run = orch.start_run(job_id=job_id, is_scheduled=False) + # Execute asynchronously in background + import threading + threading.Thread( + target=orch.execute_run, + args=(run,), + daemon=True, + ).start() + return _run_to_response(run) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + except Exception as e: + logger.error(f"[translate_routes][run_translation] Error: {e}") + raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Run failed: {e}") +# [/DEF:run_translation:Function] + + +# [DEF:retry_run:Function] +# @PURPOSE: Retry failed batches in a translation run. +# @PRE: User has translate.job.execute permission. +# @POST: Returns the updated translation run. +@router.post("/runs/{run_id}/retry") +async def retry_run( + run_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "EXECUTE")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Retry failed batches in a translation run.""" + logger.info(f"[translate_routes][retry_run] Run: {run_id}, User: {current_user.username}") + try: + orch = TranslationOrchestrator(db, config_manager, current_user.username) + run = orch.retry_failed_batches(run_id) + return _run_to_response(run) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + except Exception as e: + logger.error(f"[translate_routes][retry_run] Error: {e}") + raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry failed: {e}") +# [/DEF:retry_run:Function] + + +# [DEF:retry_insert:Function] +# @PURPOSE: Retry the SQL insert phase for a completed run. +# @PRE: User has translate.job.execute permission. +# @POST: Returns the updated run. +@router.post("/runs/{run_id}/retry-insert") +async def retry_insert( + run_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "EXECUTE")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Retry the SQL insert phase for a completed run.""" + logger.info(f"[translate_routes][retry_insert] Run: {run_id}, User: {current_user.username}") + try: + orch = TranslationOrchestrator(db, config_manager, current_user.username) + run = orch.retry_insert(run_id) + return _run_to_response(run) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + except Exception as e: + logger.error(f"[translate_routes][retry_insert] Error: {e}") + raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry insert failed: {e}") +# [/DEF:retry_insert:Function] + + +# [DEF:cancel_run:Function] +# @PURPOSE: Cancel a running translation. +# @PRE: User has translate.job.execute permission. +# @POST: Run is cancelled. +@router.post("/runs/{run_id}/cancel") +async def cancel_run( + run_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "EXECUTE")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Cancel a running translation.""" + logger.info(f"[translate_routes][cancel_run] Run: {run_id}, User: {current_user.username}") + try: + orch = TranslationOrchestrator(db, config_manager, current_user.username) + run = orch.cancel_run(run_id) + return _run_to_response(run) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# [/DEF:cancel_run:Function] + + +# [DEF:get_run_history:Function] +# @PURPOSE: Get run history for a translation job. +# @PRE: User has translate.history.view permission. +# @POST: Returns list of runs. +@router.get("/jobs/{job_id}/runs") +async def get_run_history( + job_id: str, + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.history", "VIEW")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Get run history for a translation job.""" + logger.info(f"[translate_routes][get_run_history] Job: {job_id}, User: {current_user.username}") + try: + orch = TranslationOrchestrator(db, config_manager, current_user.username) + total, runs = orch.get_run_history(job_id, page=page, page_size=page_size) + return {"items": runs, "total": total, "page": page, "page_size": page_size} + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:get_run_history:Function] + + +# [DEF:get_run_status:Function] +# @PURPOSE: Get status and statistics for a translation run. +# @PRE: User has translate.history.view permission. +# @POST: Returns run details with statistics. +@router.get("/runs/{run_id}") +async def get_run_status( + run_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.history", "VIEW")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Get status and statistics for a translation run.""" + logger.info(f"[translate_routes][get_run_status] Run: {run_id}, User: {current_user.username}") + try: + orch = TranslationOrchestrator(db, config_manager, current_user.username) + return orch.get_run_status(run_id) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:get_run_status:Function] + + +# [DEF:get_run_records:Function] +# @PURPOSE: Get paginated records for a translation run. +# @PRE: User has translate.history.view permission. +# @POST: Returns paginated records. +@router.get("/runs/{run_id}/records") +async def get_run_records( + run_id: str, + page: int = Query(1, ge=1), + page_size: int = Query(50, ge=1, le=500), + status: Optional[str] = Query(None), + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.history", "VIEW")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Get paginated records for a translation run.""" + logger.info(f"[translate_routes][get_run_records] Run: {run_id}, User: {current_user.username}") + try: + orch = TranslationOrchestrator(db, config_manager, current_user.username) + return orch.get_run_records(run_id, page=page, page_size=page_size, status_filter=status) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:get_run_records:Function] + + +# ============================================================ +# Batches +# ============================================================ + +# [DEF:get_batches:Function] +# @PURPOSE: Get batches for a translation run. +# @PRE: User has translate.job.view permission. +# @POST: Returns list of batches. +@router.get("/runs/{run_id}/batches") +async def get_batches( + run_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "VIEW")), + db: Session = Depends(get_db), +): + """Get batches for a translation run.""" + logger.info(f"[translate_routes][get_batches] Run: {run_id}, User: {current_user.username}") + try: + from ...models.translate import TranslationBatch + batches = ( + db.query(TranslationBatch) + .filter(TranslationBatch.run_id == run_id) + .order_by(TranslationBatch.batch_index.asc()) + .all() + ) + return [ + { + "id": b.id, + "run_id": b.run_id, + "batch_index": b.batch_index, + "status": b.status, + "total_records": b.total_records or 0, + "successful_records": b.successful_records or 0, + "failed_records": b.failed_records or 0, + "started_at": b.started_at.isoformat() if b.started_at else None, + "completed_at": b.completed_at.isoformat() if b.completed_at else None, + "created_at": b.created_at.isoformat() if b.created_at else None, + } + for b in batches + ] + except Exception as e: + logger.error(f"[translate_routes][get_batches] Error: {e}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# [/DEF:get_batches:Function] + +# [/DEF:TranslateRunRoutesModule:Module] diff --git a/backend/src/api/routes/translate/_schedule_routes.py b/backend/src/api/routes/translate/_schedule_routes.py new file mode 100644 index 00000000..ccdd9c74 --- /dev/null +++ b/backend/src/api/routes/translate/_schedule_routes.py @@ -0,0 +1,235 @@ +# [DEF:TranslateScheduleRoutesModule:Module] +# @COMPLEXITY: 3 +# @PURPOSE: Translation Schedule management routes. +# @LAYER: API +# @SEMANTICS: api, routes, translate, schedule + +from fastapi import APIRouter, Depends, HTTPException, status, Query +from typing import Any, Dict, List, Optional +from sqlalchemy.orm import Session + +from ....core.database import get_db +from ....core.logger import logger, belief_scope +from ....schemas.auth import User +from ....dependencies import get_current_user, has_permission, get_config_manager +from ....core.config_manager import ConfigManager +from ....plugins.translate.scheduler import TranslationScheduler +from ....schemas.translate import ScheduleConfig, ScheduleResponse + +from ._router import router + + +# ============================================================ +# Schedule +# ============================================================ + +# [DEF:get_schedule:Function] +# @PURPOSE: Get the schedule for a translation job. +# @PRE: User has translate.schedule.view permission. +# @POST: Returns the schedule configuration. +@router.get("/jobs/{job_id}/schedule") +async def get_schedule( + job_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.schedule", "VIEW")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Get schedule for a translation job.""" + logger.info(f"[translate_routes][get_schedule] Job: {job_id}, User: {current_user.username}") + try: + scheduler = TranslationScheduler(db, config_manager, current_user.username) + schedule = scheduler.get_schedule(job_id) + return { + "id": schedule.id, + "job_id": schedule.job_id, + "cron_expression": schedule.cron_expression, + "timezone": schedule.timezone, + "is_active": schedule.is_active, + "last_run_at": schedule.last_run_at.isoformat() if schedule.last_run_at else None, + "next_run_at": schedule.next_run_at.isoformat() if schedule.next_run_at else None, + "created_by": schedule.created_by, + "created_at": schedule.created_at.isoformat() if schedule.created_at else None, + "updated_at": schedule.updated_at.isoformat() if schedule.updated_at else None, + } + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:get_schedule:Function] + + +# [DEF:set_schedule:Function] +# @PURPOSE: Set or update the schedule for a translation job. +# @PRE: User has translate.schedule.manage permission. +# @POST: Schedule is created or updated. +@router.put("/jobs/{job_id}/schedule") +async def set_schedule( + job_id: str, + payload: ScheduleConfig, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.schedule", "MANAGE")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Set or update schedule for a translation job.""" + logger.info(f"[translate_routes][set_schedule] Job: {job_id}, User: {current_user.username}") + try: + scheduler = TranslationScheduler(db, config_manager, current_user.username) + # Check if schedule already exists + from ...models.translate import TranslationSchedule + existing = db.query(TranslationSchedule).filter( + TranslationSchedule.job_id == job_id + ).first() + if existing: + schedule = scheduler.update_schedule( + job_id, + cron_expression=payload.cron_expression, + timezone=payload.timezone, + is_active=payload.is_active, + ) + else: + schedule = scheduler.create_schedule( + job_id, + cron_expression=payload.cron_expression, + timezone=payload.timezone, + is_active=payload.is_active, + ) + # Register with APScheduler via SchedulerService + from ...dependencies import get_scheduler_service + sched_svc = get_scheduler_service() + sched_svc.add_translation_job( + schedule_id=schedule.id, + job_id=job_id, + cron_expression=schedule.cron_expression, + timezone=schedule.timezone, + ) + return { + "id": schedule.id, + "job_id": schedule.job_id, + "cron_expression": schedule.cron_expression, + "timezone": schedule.timezone, + "is_active": schedule.is_active, + "last_run_at": schedule.last_run_at.isoformat() if schedule.last_run_at else None, + "created_by": schedule.created_by, + "created_at": schedule.created_at.isoformat() if schedule.created_at else None, + "updated_at": schedule.updated_at.isoformat() if schedule.updated_at else None, + } + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# [/DEF:set_schedule:Function] + + +# [DEF:enable_schedule:Function] +# @PURPOSE: Enable a schedule for a translation job. +# @PRE: User has translate.schedule.manage permission. +# @POST: Schedule is enabled. +@router.post("/jobs/{job_id}/schedule/enable") +async def enable_schedule( + job_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.schedule", "MANAGE")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Enable a schedule for a translation job.""" + logger.info(f"[translate_routes][enable_schedule] Job: {job_id}, User: {current_user.username}") + try: + scheduler = TranslationScheduler(db, config_manager, current_user.username) + schedule = scheduler.set_schedule_active(job_id, is_active=True) + from ...dependencies import get_scheduler_service + sched_svc = get_scheduler_service() + sched_svc.add_translation_job( + schedule_id=schedule.id, + job_id=job_id, + cron_expression=schedule.cron_expression, + timezone=schedule.timezone, + ) + return {"status": "enabled", "job_id": job_id} + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:enable_schedule:Function] + + +# [DEF:disable_schedule:Function] +# @PURPOSE: Disable a schedule for a translation job. +# @PRE: User has translate.schedule.manage permission. +# @POST: Schedule is disabled. +@router.post("/jobs/{job_id}/schedule/disable") +async def disable_schedule( + job_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.schedule", "MANAGE")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Disable a schedule for a translation job.""" + logger.info(f"[translate_routes][disable_schedule] Job: {job_id}, User: {current_user.username}") + try: + scheduler = TranslationScheduler(db, config_manager, current_user.username) + scheduler.set_schedule_active(job_id, is_active=False) + from ...dependencies import get_scheduler_service + sched_svc = get_scheduler_service() + sched_svc.remove_translation_job(schedule_id=job_id) + return {"status": "disabled", "job_id": job_id} + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:disable_schedule:Function] + + +# [DEF:delete_schedule:Function] +# @PURPOSE: Delete the schedule for a translation job. +# @PRE: User has translate.schedule.manage permission. +# @POST: Schedule is removed. +@router.delete("/jobs/{job_id}/schedule", status_code=status.HTTP_204_NO_CONTENT) +async def delete_schedule( + job_id: str, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.schedule", "MANAGE")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Delete schedule for a translation job.""" + logger.info(f"[translate_routes][delete_schedule] Job: {job_id}, User: {current_user.username}") + try: + scheduler = TranslationScheduler(db, config_manager, current_user.username) + scheduler.delete_schedule(job_id) + from ...dependencies import get_scheduler_service + sched_svc = get_scheduler_service() + sched_svc.remove_translation_job(schedule_id=job_id) + return None + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:delete_schedule:Function] + + +# [DEF:get_next_executions:Function] +# @PURPOSE: Preview next N executions for a job's schedule. +# @PRE: User has translate.schedule.view permission. +# @POST: Returns next execution times. +@router.get("/jobs/{job_id}/schedule/next-executions") +async def get_next_executions( + job_id: str, + n: int = Query(3, ge=1, le=10), + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.schedule", "VIEW")), + db: Session = Depends(get_db), + config_manager: ConfigManager = Depends(get_config_manager), +): + """Preview next N executions for a job's schedule.""" + logger.info(f"[translate_routes][get_next_executions] Job: {job_id}, User: {current_user.username}") + try: + scheduler = TranslationScheduler(db, config_manager, current_user.username) + schedule = scheduler.get_schedule(job_id) + next_times = TranslationScheduler.get_next_executions( + schedule.cron_expression, schedule.timezone, n=n + ) + return { + "job_id": job_id, + "cron_expression": schedule.cron_expression, + "timezone": schedule.timezone, + "next_executions": next_times, + } + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:get_next_executions:Function] + +# [/DEF:TranslateScheduleRoutesModule:Module] diff --git a/backend/src/core/superset_client.py b/backend/src/core/superset_client.py deleted file mode 100644 index e41b37d2..00000000 --- a/backend/src/core/superset_client.py +++ /dev/null @@ -1,2145 +0,0 @@ -# [DEF:SupersetClientModule:Module] -# -# @COMPLEXITY: 3 -# @SEMANTICS: superset, api, client, rest, http, dashboard, dataset, import, export -# @PURPOSE: Предоставляет высокоуровневый клиент для взаимодействия с Superset REST API, инкапсулируя логику запросов, обработку ошибок и пагинацию. -# @LAYER: Core -# @RELATION: DEPENDS_ON -> [ConfigModels] -# @RELATION: DEPENDS_ON -> [APIClient] -# @RELATION: DEPENDS_ON -> [SupersetAPIError] -# @RELATION: DEPENDS_ON -> [get_filename_from_headers] -# -# @INVARIANT: All network operations must use the internal APIClient instance. -# @PUBLIC_API: SupersetClient - -# [SECTION: IMPORTS] -import json -import re -import zipfile -from copy import deepcopy -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union, cast -from requests import Response -from datetime import datetime -from .logger import logger as app_logger, belief_scope -from .utils.network import APIClient, SupersetAPIError -from .utils.fileio import get_filename_from_headers -from .config_models import Environment - -app_logger = cast(Any, app_logger) -# [/SECTION] - - -# [DEF:SupersetClient:Class] -# @COMPLEXITY: 3 -# @PURPOSE: Класс-обёртка над Superset REST API, предоставляющий методы для работы с дашбордами и датасетами. -# @RELATION: DEPENDS_ON -> [ConfigModels] -# @RELATION: DEPENDS_ON -> [APIClient] -# @RELATION: DEPENDS_ON -> [SupersetAPIError] -class SupersetClient: - # [DEF:SupersetClientInit:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент. - # @PRE: `env` должен быть валидным объектом Environment. - # @POST: Атрибуты `env` и `network` созданы и готовы к работе. - # @DATA_CONTRACT: Input[Environment] -> self.network[APIClient] - # @RELATION: DEPENDS_ON -> [Environment] - # @RELATION: DEPENDS_ON -> [APIClient] - def __init__(self, env: Environment): - with belief_scope("SupersetClientInit"): - app_logger.reason( - "Initializing Superset client for environment", - extra={"environment": getattr(env, "id", None), "env_name": env.name}, - ) - self.env = env - # Construct auth payload expected by Superset API - auth_payload = { - "username": env.username, - "password": env.password, - "provider": "db", - "refresh": "true", - } - self.network = APIClient( - config={"base_url": env.url, "auth": auth_payload}, - verify_ssl=env.verify_ssl, - timeout=env.timeout, - ) - self.delete_before_reimport: bool = False - app_logger.reflect( - "Superset client initialized", - extra={"environment": getattr(self.env, "id", None)}, - ) - - # [/DEF:SupersetClientInit:Function] - - # [DEF:SupersetClientAuthenticate:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Authenticates the client using the configured credentials. - # @PRE: self.network must be initialized with valid auth configuration. - # @POST: Client is authenticated and tokens are stored. - # @DATA_CONTRACT: None -> Output[Dict[str, str]] - # @RELATION: CALLS -> [APIClient] - def authenticate(self) -> Dict[str, str]: - with belief_scope("SupersetClientAuthenticate"): - app_logger.reason( - "Authenticating Superset client", - extra={"environment": getattr(self.env, "id", None)}, - ) - tokens = self.network.authenticate() - app_logger.reflect( - "Superset client authentication completed", - extra={ - "environment": getattr(self.env, "id", None), - "token_keys": sorted(tokens.keys()), - }, - ) - return tokens - # [/DEF:SupersetClientAuthenticate:Function] - - @property - # [DEF:SupersetClientHeaders:Function] - # @COMPLEXITY: 1 - # @PURPOSE: Возвращает базовые HTTP-заголовки, используемые сетевым клиентом. - # @PRE: APIClient is initialized and authenticated. - # @POST: Returns a dictionary of HTTP headers. - def headers(self) -> dict: - with belief_scope("headers"): - return self.network.headers - - # [/DEF:SupersetClientHeaders:Function] - - # [SECTION: DASHBOARD OPERATIONS] - - # [DEF:SupersetClientGetDashboards:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Получает полный список дашбордов, автоматически обрабатывая пагинацию. - # @PRE: Client is authenticated. - # @POST: Returns a tuple with total count and list of dashboards. - # @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]] - # @RELATION: CALLS -> [SupersetClientFetchAllPages] - def get_dashboards(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]: - with belief_scope("get_dashboards"): - app_logger.info("[get_dashboards][Enter] Fetching dashboards.") - validated_query = self._validate_query_params(query or {}) - if "columns" not in validated_query: - validated_query["columns"] = [ - "slug", - "id", - "url", - "changed_on_utc", - "dashboard_title", - "published", - "created_by", - "changed_by", - "changed_by_name", - "owners", - ] - - paginated_data = self._fetch_all_pages( - endpoint="/dashboard/", - pagination_options={ - "base_query": validated_query, - "results_field": "result", - }, - ) - total_count = len(paginated_data) - app_logger.info("[get_dashboards][Exit] Found %d dashboards.", total_count) - return total_count, paginated_data - - # [/DEF:SupersetClientGetDashboards:Function] - - # [DEF:SupersetClientGetDashboardsPage:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Fetches a single dashboards page from Superset without iterating all pages. - # @PRE: Client is authenticated. - # @POST: Returns total count and one page of dashboards. - # @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]] - # @RELATION: CALLS -> [APIClient] - def get_dashboards_page( - self, query: Optional[Dict] = None - ) -> Tuple[int, List[Dict]]: - with belief_scope("get_dashboards_page"): - validated_query = self._validate_query_params(query or {}) - if "columns" not in validated_query: - validated_query["columns"] = [ - "slug", - "id", - "url", - "changed_on_utc", - "dashboard_title", - "published", - "created_by", - "changed_by", - "changed_by_name", - "owners", - ] - - response_json = cast( - Dict[str, Any], - self.network.request( - method="GET", - endpoint="/dashboard/", - params={"q": json.dumps(validated_query)}, - ), - ) - result = response_json.get("result", []) - total_count = response_json.get("count", len(result)) - return total_count, result - - # [/DEF:SupersetClientGetDashboardsPage:Function] - - # [DEF:SupersetClientGetDashboardsSummary:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Fetches dashboard metadata optimized for the grid. - # @PRE: Client is authenticated. - # @POST: Returns a list of dashboard metadata summaries. - # @DATA_CONTRACT: None -> Output[List[Dict]] - # @RELATION: CALLS -> [SupersetClientGetDashboards] - def get_dashboards_summary(self, require_slug: bool = False) -> List[Dict]: - with belief_scope("SupersetClient.get_dashboards_summary"): - # Rely on list endpoint default projection to stay compatible - # across Superset versions and preserve owners in one request. - query: Dict[str, Any] = {} - if require_slug: - query["filters"] = [ - { - "col": "slug", - "opr": "neq", - "value": "", - } - ] - _, dashboards = self.get_dashboards(query=query) - - # Map fields to DashboardMetadata schema - result = [] - max_debug_samples = 12 - for index, dash in enumerate(dashboards): - raw_owners = dash.get("owners") - raw_created_by = dash.get("created_by") - raw_changed_by = dash.get("changed_by") - raw_changed_by_name = dash.get("changed_by_name") - - owners = self._extract_owner_labels(raw_owners) - # No per-dashboard detail requests here: keep list endpoint O(1). - if not owners: - owners = self._extract_owner_labels( - [raw_created_by, raw_changed_by], - ) - - projected_created_by = self._extract_user_display( - None, - raw_created_by, - ) - projected_modified_by = self._extract_user_display( - raw_changed_by_name, - raw_changed_by, - ) - - raw_owner_usernames: List[str] = [] - if isinstance(raw_owners, list): - for owner_payload in raw_owners: - if isinstance(owner_payload, dict): - owner_username = self._sanitize_user_text( - owner_payload.get("username") - ) - if owner_username: - raw_owner_usernames.append(owner_username) - - result.append( - { - "id": dash.get("id"), - "slug": dash.get("slug"), - "title": dash.get("dashboard_title"), - "url": dash.get("url"), - "last_modified": dash.get("changed_on_utc"), - "status": "published" if dash.get("published") else "draft", - "created_by": projected_created_by, - "modified_by": projected_modified_by, - "owners": owners, - } - ) - - if index < max_debug_samples: - app_logger.reflect( - "[REFLECT] Dashboard actor projection sample " - f"(env={getattr(self.env, 'id', None)}, dashboard_id={dash.get('id')}, " - f"raw_owners={raw_owners!r}, raw_owner_usernames={raw_owner_usernames!r}, " - f"raw_created_by={raw_created_by!r}, raw_changed_by={raw_changed_by!r}, " - f"raw_changed_by_name={raw_changed_by_name!r}, projected_owners={owners!r}, " - f"projected_created_by={projected_created_by!r}, projected_modified_by={projected_modified_by!r})" - ) - - app_logger.reflect( - "[REFLECT] Dashboard actor projection summary " - f"(env={getattr(self.env, 'id', None)}, dashboards={len(result)}, " - f"sampled={min(len(result), max_debug_samples)})" - ) - return result - - # [/DEF:SupersetClientGetDashboardsSummary:Function] - - # [DEF:SupersetClientGetDashboardsSummaryPage:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Fetches one page of dashboard metadata optimized for the grid. - # @PRE: page >= 1 and page_size > 0. - # @POST: Returns mapped summaries and total dashboard count. - # @DATA_CONTRACT: Input[page: int, page_size: int] -> Output[Tuple[int, List[Dict]]] - # @RELATION: CALLS -> [SupersetClientGetDashboardsPage] - def get_dashboards_summary_page( - self, - page: int, - page_size: int, - search: Optional[str] = None, - require_slug: bool = False, - ) -> Tuple[int, List[Dict]]: - with belief_scope("SupersetClient.get_dashboards_summary_page"): - query: Dict[str, Any] = { - "page": max(page - 1, 0), - "page_size": page_size, - } - filters: List[Dict[str, Any]] = [] - if require_slug: - filters.append( - { - "col": "slug", - "opr": "neq", - "value": "", - } - ) - normalized_search = (search or "").strip() - if normalized_search: - # Superset list API supports filter objects with `opr` operator. - # `ct` -> contains (ILIKE on most Superset backends). - filters.append( - { - "col": "dashboard_title", - "opr": "ct", - "value": normalized_search, - } - ) - if filters: - query["filters"] = filters - - total_count, dashboards = self.get_dashboards_page(query=query) - - result = [] - for dash in dashboards: - owners = self._extract_owner_labels(dash.get("owners")) - if not owners: - owners = self._extract_owner_labels( - [dash.get("created_by"), dash.get("changed_by")], - ) - - result.append( - { - "id": dash.get("id"), - "slug": dash.get("slug"), - "title": dash.get("dashboard_title"), - "url": dash.get("url"), - "last_modified": dash.get("changed_on_utc"), - "status": "published" if dash.get("published") else "draft", - "created_by": self._extract_user_display( - None, - dash.get("created_by"), - ), - "modified_by": self._extract_user_display( - dash.get("changed_by_name"), - dash.get("changed_by"), - ), - "owners": owners, - } - ) - - return total_count, result - - # [/DEF:SupersetClientGetDashboardsSummaryPage:Function] - - # [DEF:SupersetClientExtractOwnerLabels:Function] - # @COMPLEXITY: 1 - # @PURPOSE: Normalize dashboard owners payload to stable display labels. - # @PRE: owners payload can be scalar, object or list. - # @POST: Returns deduplicated non-empty owner labels preserving order. - # @DATA_CONTRACT: Input[Any] -> Output[List[str]] - def _extract_owner_labels(self, owners_payload: Any) -> List[str]: - if owners_payload is None: - return [] - - owners_list: List[Any] - if isinstance(owners_payload, list): - owners_list = owners_payload - else: - owners_list = [owners_payload] - - normalized: List[str] = [] - for owner in owners_list: - label: Optional[str] = None - if isinstance(owner, dict): - label = self._extract_user_display(None, owner) - else: - label = self._sanitize_user_text(owner) - if label and label not in normalized: - normalized.append(label) - return normalized - - # [/DEF:SupersetClientExtractOwnerLabels:Function] - - # [DEF:SupersetClientExtractUserDisplay:Function] - # @COMPLEXITY: 1 - # @PURPOSE: Normalize user payload to a stable display name. - # @PRE: user payload can be string, dict or None. - # @POST: Returns compact non-empty display value or None. - # @DATA_CONTRACT: Input[Optional[str], Optional[Dict]] -> Output[Optional[str]] - def _extract_user_display( - self, preferred_value: Optional[str], user_payload: Optional[Dict] - ) -> Optional[str]: - preferred = self._sanitize_user_text(preferred_value) - if preferred: - return preferred - - if isinstance(user_payload, dict): - full_name = self._sanitize_user_text(user_payload.get("full_name")) - if full_name: - return full_name - first_name = self._sanitize_user_text(user_payload.get("first_name")) or "" - last_name = self._sanitize_user_text(user_payload.get("last_name")) or "" - combined = " ".join( - part for part in [first_name, last_name] if part - ).strip() - if combined: - return combined - username = self._sanitize_user_text(user_payload.get("username")) - if username: - return username - email = self._sanitize_user_text(user_payload.get("email")) - if email: - return email - return None - - # [/DEF:SupersetClientExtractUserDisplay:Function] - - # [DEF:SupersetClientSanitizeUserText:Function] - # @COMPLEXITY: 1 - # @PURPOSE: Convert scalar value to non-empty user-facing text. - # @PRE: value can be any scalar type. - # @POST: Returns trimmed string or None. - def _sanitize_user_text(self, value: Optional[Union[str, int]]) -> Optional[str]: - if value is None: - return None - normalized = str(value).strip() - if not normalized: - return None - return normalized - - # [/DEF:SupersetClientSanitizeUserText:Function] - - # [DEF:SupersetClientGetDashboard:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Fetches a single dashboard by ID or slug. - # @PRE: Client is authenticated and dashboard_ref exists. - # @POST: Returns dashboard payload from Superset API. - # @DATA_CONTRACT: Input[dashboard_ref: Union[int, str]] -> Output[Dict] - # @RELATION: CALLS -> [APIClient] - def get_dashboard(self, dashboard_ref: Union[int, str]) -> Dict: - with belief_scope("SupersetClient.get_dashboard", f"ref={dashboard_ref}"): - response = self.network.request( - method="GET", endpoint=f"/dashboard/{dashboard_ref}" - ) - return cast(Dict, response) - - # [/DEF:SupersetClientGetDashboard:Function] - - # [DEF:SupersetClientGetDashboardPermalinkState:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Fetches stored dashboard permalink state by permalink key. - # @PRE: Client is authenticated and permalink key exists. - # @POST: Returns dashboard permalink state payload from Superset API. - # @DATA_CONTRACT: Input[permalink_key: str] -> Output[Dict] - # @RELATION: CALLS -> [APIClient] - def get_dashboard_permalink_state(self, permalink_key: str) -> Dict: - with belief_scope( - "SupersetClient.get_dashboard_permalink_state", f"key={permalink_key}" - ): - response = self.network.request( - method="GET", endpoint=f"/dashboard/permalink/{permalink_key}" - ) - return cast(Dict, response) - - # [/DEF:SupersetClientGetDashboardPermalinkState:Function] - - # [DEF:SupersetClientGetNativeFilterState:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Fetches stored native filter state by filter state key. - # @PRE: Client is authenticated and filter_state_key exists. - # @POST: Returns native filter state payload from Superset API. - # @DATA_CONTRACT: Input[dashboard_id: Union[int, str], filter_state_key: str] -> Output[Dict] - # @RELATION: CALLS -> [APIClient] - def get_native_filter_state( - self, dashboard_id: Union[int, str], filter_state_key: str - ) -> Dict: - with belief_scope( - "SupersetClient.get_native_filter_state", - f"dashboard={dashboard_id}, key={filter_state_key}", - ): - response = self.network.request( - method="GET", - endpoint=f"/dashboard/{dashboard_id}/filter_state/{filter_state_key}", - ) - return cast(Dict, response) - - # [/DEF:SupersetClientGetNativeFilterState:Function] - - # [DEF:SupersetClientExtractNativeFiltersFromPermalink:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Extract native filters dataMask from a permalink key. - # @PRE: Client is authenticated and permalink_key exists. - # @POST: Returns extracted dataMask with filter states. - # @DATA_CONTRACT: Input[permalink_key: str] -> Output[Dict] - # @RELATION: CALLS -> [SupersetClientGetDashboardPermalinkState] - def extract_native_filters_from_permalink(self, permalink_key: str) -> Dict: - with belief_scope( - "SupersetClient.extract_native_filters_from_permalink", - f"key={permalink_key}", - ): - permalink_response = self.get_dashboard_permalink_state(permalink_key) - - # Permalink response structure: { "result": { "state": { "dataMask": {...}, ... } } } - # or directly: { "state": { "dataMask": {...}, ... } } - result = permalink_response.get("result", permalink_response) - state = result.get("state", result) - data_mask = state.get("dataMask", {}) - - extracted_filters = {} - for filter_id, filter_data in data_mask.items(): - if not isinstance(filter_data, dict): - continue - extracted_filters[filter_id] = { - "extraFormData": filter_data.get("extraFormData", {}), - "filterState": filter_data.get("filterState", {}), - "ownState": filter_data.get("ownState", {}), - } - - return { - "dataMask": extracted_filters, - "activeTabs": state.get("activeTabs", []), - "anchor": state.get("anchor"), - "chartStates": state.get("chartStates", {}), - "permalink_key": permalink_key, - } - - # [/DEF:SupersetClientExtractNativeFiltersFromPermalink:Function] - - # [DEF:SupersetClientExtractNativeFiltersFromKey:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Extract native filters from a native_filters_key URL parameter. - # @PRE: Client is authenticated, dashboard_id and filter_state_key exist. - # @POST: Returns extracted filter state with extraFormData. - # @DATA_CONTRACT: Input[dashboard_id: Union[int, str], filter_state_key: str] -> Output[Dict] - # @RELATION: CALLS -> [SupersetClientGetNativeFilterState] - def extract_native_filters_from_key( - self, dashboard_id: Union[int, str], filter_state_key: str - ) -> Dict: - with belief_scope( - "SupersetClient.extract_native_filters_from_key", - f"dashboard={dashboard_id}, key={filter_state_key}", - ): - filter_response = self.get_native_filter_state( - dashboard_id, filter_state_key - ) - - # Filter state response structure: { "result": { "value": "{...json...}" } } - # or: { "value": "{...json...}" } - result = filter_response.get("result", filter_response) - value = result.get("value") - - if isinstance(value, str): - try: - parsed_value = json.loads(value) - except json.JSONDecodeError as e: - app_logger.warning( - "[extract_native_filters_from_key][Warning] Failed to parse filter state JSON: %s", - e, - ) - parsed_value = {} - elif isinstance(value, dict): - parsed_value = value - else: - parsed_value = {} - - # The parsed value contains filter state with structure: - # { "filter_id": { "id": "...", "extraFormData": {...}, "filterState": {...} } } - # or a single filter: { "id": "...", "extraFormData": {...}, "filterState": {...} } - extracted_filters = {} - - if "id" in parsed_value and "extraFormData" in parsed_value: - # Single filter format - filter_id = parsed_value.get("id", filter_state_key) - extracted_filters[filter_id] = { - "extraFormData": parsed_value.get("extraFormData", {}), - "filterState": parsed_value.get("filterState", {}), - "ownState": parsed_value.get("ownState", {}), - } - else: - # Multiple filters format - for filter_id, filter_data in parsed_value.items(): - if not isinstance(filter_data, dict): - continue - extracted_filters[filter_id] = { - "extraFormData": filter_data.get("extraFormData", {}), - "filterState": filter_data.get("filterState", {}), - "ownState": filter_data.get("ownState", {}), - } - - return { - "dataMask": extracted_filters, - "dashboard_id": dashboard_id, - "filter_state_key": filter_state_key, - } - - # [/DEF:SupersetClientExtractNativeFiltersFromKey:Function] - - # [DEF:SupersetClientParseDashboardUrlForFilters:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Parse a Superset dashboard URL and extract native filter state if present. - # @PRE: url must be a valid Superset dashboard URL with optional permalink or native_filters_key. - # @POST: Returns extracted filter state or empty dict if no filters found. - # @DATA_CONTRACT: Input[url: str] -> Output[Dict] - # @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromPermalink] - # @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromKey] - def parse_dashboard_url_for_filters(self, url: str) -> Dict: - with belief_scope( - "SupersetClient.parse_dashboard_url_for_filters", f"url={url}" - ): - import urllib.parse - - parsed_url = urllib.parse.urlparse(url) - query_params = urllib.parse.parse_qs(parsed_url.query) - path_parts = parsed_url.path.rstrip("/").split("/") - - result = { - "url": url, - "dashboard_id": None, - "filter_type": None, - "filters": {}, - } - - # Check for permalink URL: /dashboard/p/{key}/ or /superset/dashboard/p/{key}/ - if "p" in path_parts: - try: - p_index = path_parts.index("p") - if p_index + 1 < len(path_parts): - permalink_key = path_parts[p_index + 1] - filter_data = self.extract_native_filters_from_permalink( - permalink_key - ) - result["filter_type"] = "permalink" - result["filters"] = filter_data - return result - except ValueError: - pass - - # Check for native_filters_key in query params - native_filters_key = query_params.get("native_filters_key", [None])[0] - if native_filters_key: - # Extract dashboard ID or slug from URL path - dashboard_ref = None - if "dashboard" in path_parts: - try: - dash_index = path_parts.index("dashboard") - if dash_index + 1 < len(path_parts): - potential_id = path_parts[dash_index + 1] - # Skip if it's a reserved word - if potential_id not in ("p", "list", "new"): - dashboard_ref = potential_id - except ValueError: - pass - - if dashboard_ref: - # Resolve slug to numeric ID — the filter_state API requires a numeric ID - resolved_id = None - try: - resolved_id = int(dashboard_ref) - except (ValueError, TypeError): - try: - dash_resp = self.get_dashboard(dashboard_ref) - dash_data = ( - dash_resp.get("result", dash_resp) - if isinstance(dash_resp, dict) - else {} - ) - raw_id = dash_data.get("id") - if raw_id is not None: - resolved_id = int(raw_id) - except Exception as e: - app_logger.warning( - "[parse_dashboard_url_for_filters][Warning] Failed to resolve dashboard slug '%s' to ID: %s", - dashboard_ref, - e, - ) - - if resolved_id is not None: - filter_data = self.extract_native_filters_from_key( - resolved_id, native_filters_key - ) - result["filter_type"] = "native_filters_key" - result["dashboard_id"] = resolved_id - result["filters"] = filter_data - return result - else: - app_logger.warning( - "[parse_dashboard_url_for_filters][Warning] Could not resolve dashboard_id from URL for native_filters_key" - ) - - # Check for native_filters in query params (direct filter values) - native_filters = query_params.get("native_filters", [None])[0] - if native_filters: - try: - parsed_filters = json.loads(native_filters) - result["filter_type"] = "native_filters" - result["filters"] = {"dataMask": parsed_filters} - return result - except json.JSONDecodeError as e: - app_logger.warning( - "[parse_dashboard_url_for_filters][Warning] Failed to parse native_filters JSON: %s", - e, - ) - - return result - - # [/DEF:SupersetClientParseDashboardUrlForFilters:Function] - - # [DEF:SupersetClientGetChart:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Fetches a single chart by ID. - # @PRE: Client is authenticated and chart_id exists. - # @POST: Returns chart payload from Superset API. - # @DATA_CONTRACT: Input[chart_id: int] -> Output[Dict] - # @RELATION: CALLS -> [APIClient] - def get_chart(self, chart_id: int) -> Dict: - with belief_scope("SupersetClient.get_chart", f"id={chart_id}"): - response = self.network.request(method="GET", endpoint=f"/chart/{chart_id}") - return cast(Dict, response) - - # [/DEF:SupersetClientGetChart:Function] - - # [DEF:SupersetClientGetDashboardDetail:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Fetches detailed dashboard information including related charts and datasets. - # @PRE: Client is authenticated and dashboard reference exists. - # @POST: Returns dashboard metadata with charts and datasets lists. - # @DATA_CONTRACT: Input[dashboard_ref: Union[int, str]] -> Output[Dict] - # @RELATION: CALLS -> [SupersetClientGetDashboard] - # @RELATION: CALLS -> [SupersetClientGetChart] - def get_dashboard_detail(self, dashboard_ref: Union[int, str]) -> Dict: - with belief_scope( - "SupersetClient.get_dashboard_detail", f"ref={dashboard_ref}" - ): - dashboard_response = self.get_dashboard(dashboard_ref) - dashboard_data = dashboard_response.get("result", dashboard_response) - - charts: List[Dict] = [] - datasets: List[Dict] = [] - - # [DEF:extract_dataset_id_from_form_data:Function] - def extract_dataset_id_from_form_data( - form_data: Optional[Dict], - ) -> Optional[int]: - if not isinstance(form_data, dict): - return None - datasource = form_data.get("datasource") - if isinstance(datasource, str): - matched = re.match(r"^(\d+)__", datasource) - if matched: - try: - return int(matched.group(1)) - except ValueError: - return None - if isinstance(datasource, dict): - ds_id = datasource.get("id") - try: - return int(ds_id) if ds_id is not None else None - except (TypeError, ValueError): - return None - ds_id = form_data.get("datasource_id") - try: - return int(ds_id) if ds_id is not None else None - except (TypeError, ValueError): - return None - - # [/DEF:extract_dataset_id_from_form_data:Function] - - # Canonical endpoints from Superset OpenAPI: - # /dashboard/{id_or_slug}/charts and /dashboard/{id_or_slug}/datasets. - try: - charts_response = self.network.request( - method="GET", endpoint=f"/dashboard/{dashboard_ref}/charts" - ) - charts_payload = ( - charts_response.get("result", []) - if isinstance(charts_response, dict) - else [] - ) - for chart_obj in charts_payload: - if not isinstance(chart_obj, dict): - continue - chart_id = chart_obj.get("id") - if chart_id is None: - continue - form_data = chart_obj.get("form_data") - if isinstance(form_data, str): - try: - form_data = json.loads(form_data) - except Exception: - form_data = {} - dataset_id = extract_dataset_id_from_form_data( - form_data - ) or chart_obj.get("datasource_id") - charts.append( - { - "id": int(chart_id), - "title": chart_obj.get("slice_name") - or chart_obj.get("name") - or f"Chart {chart_id}", - "viz_type": ( - form_data.get("viz_type") - if isinstance(form_data, dict) - else None - ), - "dataset_id": int(dataset_id) - if dataset_id is not None - else None, - "last_modified": chart_obj.get("changed_on"), - "overview": chart_obj.get("description") - or ( - form_data.get("viz_type") - if isinstance(form_data, dict) - else None - ) - or "Chart", - } - ) - except Exception as e: - app_logger.warning( - "[get_dashboard_detail][Warning] Failed to fetch dashboard charts: %s", - e, - ) - - try: - datasets_response = self.network.request( - method="GET", endpoint=f"/dashboard/{dashboard_ref}/datasets" - ) - datasets_payload = ( - datasets_response.get("result", []) - if isinstance(datasets_response, dict) - else [] - ) - for dataset_obj in datasets_payload: - if not isinstance(dataset_obj, dict): - continue - dataset_id = dataset_obj.get("id") - if dataset_id is None: - continue - db_payload = dataset_obj.get("database") - db_name = ( - db_payload.get("database_name") - if isinstance(db_payload, dict) - else None - ) - table_name = ( - dataset_obj.get("table_name") - or dataset_obj.get("datasource_name") - or dataset_obj.get("name") - or f"Dataset {dataset_id}" - ) - schema = dataset_obj.get("schema") - fq_name = f"{schema}.{table_name}" if schema else table_name - datasets.append( - { - "id": int(dataset_id), - "table_name": table_name, - "schema": schema, - "database": db_name - or dataset_obj.get("database_name") - or "Unknown", - "last_modified": dataset_obj.get("changed_on"), - "overview": fq_name, - } - ) - except Exception as e: - app_logger.warning( - "[get_dashboard_detail][Warning] Failed to fetch dashboard datasets: %s", - e, - ) - - # Fallback: derive chart IDs from layout metadata if dashboard charts endpoint fails. - if not charts: - raw_position_json = dashboard_data.get("position_json") - chart_ids_from_position = set() - if isinstance(raw_position_json, str) and raw_position_json: - try: - parsed_position = json.loads(raw_position_json) - chart_ids_from_position.update( - self._extract_chart_ids_from_layout(parsed_position) - ) - except Exception: - pass - elif isinstance(raw_position_json, dict): - chart_ids_from_position.update( - self._extract_chart_ids_from_layout(raw_position_json) - ) - - raw_json_metadata = dashboard_data.get("json_metadata") - if isinstance(raw_json_metadata, str) and raw_json_metadata: - try: - parsed_metadata = json.loads(raw_json_metadata) - chart_ids_from_position.update( - self._extract_chart_ids_from_layout(parsed_metadata) - ) - except Exception: - pass - elif isinstance(raw_json_metadata, dict): - chart_ids_from_position.update( - self._extract_chart_ids_from_layout(raw_json_metadata) - ) - - app_logger.info( - "[get_dashboard_detail][State] Extracted %s fallback chart IDs from layout (dashboard_id=%s)", - len(chart_ids_from_position), - dashboard_ref, - ) - - for chart_id in sorted(chart_ids_from_position): - try: - chart_response = self.get_chart(int(chart_id)) - chart_data = chart_response.get("result", chart_response) - charts.append( - { - "id": int(chart_id), - "title": chart_data.get("slice_name") - or chart_data.get("name") - or f"Chart {chart_id}", - "viz_type": chart_data.get("viz_type"), - "dataset_id": chart_data.get("datasource_id"), - "last_modified": chart_data.get("changed_on"), - "overview": chart_data.get("description") - or chart_data.get("viz_type") - or "Chart", - } - ) - except Exception as e: - app_logger.warning( - "[get_dashboard_detail][Warning] Failed to resolve fallback chart %s: %s", - chart_id, - e, - ) - - # Backfill datasets from chart datasource IDs. - dataset_ids_from_charts = { - c.get("dataset_id") for c in charts if c.get("dataset_id") is not None - } - known_dataset_ids = { - d.get("id") for d in datasets if d.get("id") is not None - } - missing_dataset_ids: List[int] = [] - for raw_dataset_id in dataset_ids_from_charts: - if raw_dataset_id is None or raw_dataset_id in known_dataset_ids: - continue - try: - missing_dataset_ids.append(int(raw_dataset_id)) - except (TypeError, ValueError): - continue - - for dataset_id in missing_dataset_ids: - try: - dataset_response = self.get_dataset(int(dataset_id)) - dataset_data = dataset_response.get("result", dataset_response) - db_payload = dataset_data.get("database") - db_name = ( - db_payload.get("database_name") - if isinstance(db_payload, dict) - else None - ) - table_name = ( - dataset_data.get("table_name") or f"Dataset {dataset_id}" - ) - schema = dataset_data.get("schema") - fq_name = f"{schema}.{table_name}" if schema else table_name - datasets.append( - { - "id": int(dataset_id), - "table_name": table_name, - "schema": schema, - "database": db_name or "Unknown", - "last_modified": dataset_data.get("changed_on_utc") - or dataset_data.get("changed_on"), - "overview": fq_name, - } - ) - except Exception as e: - app_logger.warning( - "[get_dashboard_detail][Warning] Failed to resolve dataset %s: %s", - dataset_id, - e, - ) - - unique_charts = {} - for chart in charts: - unique_charts[chart["id"]] = chart - - unique_datasets = {} - for dataset in datasets: - unique_datasets[dataset["id"]] = dataset - - resolved_dashboard_id = dashboard_data.get("id", dashboard_ref) - return { - "id": resolved_dashboard_id, - "title": dashboard_data.get("dashboard_title") - or dashboard_data.get("title") - or f"Dashboard {resolved_dashboard_id}", - "slug": dashboard_data.get("slug"), - "url": dashboard_data.get("url"), - "description": dashboard_data.get("description") or "", - "last_modified": dashboard_data.get("changed_on_utc") - or dashboard_data.get("changed_on"), - "published": dashboard_data.get("published"), - "charts": list(unique_charts.values()), - "datasets": list(unique_datasets.values()), - "chart_count": len(unique_charts), - "dataset_count": len(unique_datasets), - } - - # [/DEF:SupersetClientGetDashboardDetail:Function] - - # [DEF:SupersetClientGetCharts:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Fetches all charts with pagination support. - # @PRE: Client is authenticated. - # @POST: Returns total count and charts list. - # @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]] - # @RELATION: CALLS -> [SupersetClientFetchAllPages] - def get_charts(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]: - with belief_scope("get_charts"): - validated_query = self._validate_query_params(query or {}) - if "columns" not in validated_query: - validated_query["columns"] = ["id", "uuid", "slice_name", "viz_type"] - - paginated_data = self._fetch_all_pages( - endpoint="/chart/", - pagination_options={ - "base_query": validated_query, - "results_field": "result", - }, - ) - return len(paginated_data), paginated_data - - # [/DEF:SupersetClientGetCharts:Function] - - # [DEF:SupersetClientExtractChartIdsFromLayout:Function] - # @COMPLEXITY: 1 - # @PURPOSE: Traverses dashboard layout metadata and extracts chart IDs from common keys. - # @PRE: payload can be dict/list/scalar. - # @POST: Returns a set of chart IDs found in nested structures. - def _extract_chart_ids_from_layout( - self, payload: Union[Dict, List, str, int, None] - ) -> set: - with belief_scope("_extract_chart_ids_from_layout"): - found = set() - - def walk(node): - if isinstance(node, dict): - for key, value in node.items(): - if key in ("chartId", "chart_id", "slice_id", "sliceId"): - try: - found.add(int(value)) - except (TypeError, ValueError): - pass - if key == "id" and isinstance(value, str): - match = re.match(r"^CHART-(\d+)$", value) - if match: - try: - found.add(int(match.group(1))) - except ValueError: - pass - walk(value) - elif isinstance(node, list): - for item in node: - walk(item) - - walk(payload) - return found - - # [/DEF:SupersetClientExtractChartIdsFromLayout:Function] - - # [DEF:SupersetClientExportDashboard:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Экспортирует дашборд в виде ZIP-архива. - # @PRE: dashboard_id must exist in Superset. - # @POST: Returns ZIP content and filename. - # @DATA_CONTRACT: Input[dashboard_id: int] -> Output[Tuple[bytes, str]] - # @SIDE_EFFECT: Performs network I/O to download archive. - # @RELATION: CALLS -> [APIClient] - def export_dashboard(self, dashboard_id: int) -> Tuple[bytes, str]: - with belief_scope("export_dashboard"): - app_logger.info( - "[export_dashboard][Enter] Exporting dashboard %s.", dashboard_id - ) - response = self.network.request( - method="GET", - endpoint="/dashboard/export/", - params={"q": json.dumps([dashboard_id])}, - stream=True, - raw_response=True, - ) - response = cast(Response, response) - self._validate_export_response(response, dashboard_id) - filename = self._resolve_export_filename(response, dashboard_id) - app_logger.info( - "[export_dashboard][Exit] Exported dashboard %s to %s.", - dashboard_id, - filename, - ) - return response.content, filename - - # [/DEF:SupersetClientExportDashboard:Function] - - # [DEF:SupersetClientImportDashboard:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Импортирует дашборд из ZIP-файла. - # @PRE: file_name must be a valid ZIP dashboard export. - # @POST: Dashboard is imported or re-imported after deletion. - # @DATA_CONTRACT: Input[file_name: Union[str, Path]] -> Output[Dict] - # @SIDE_EFFECT: Performs network I/O to upload archive. - # @RELATION: CALLS -> [SupersetClientDoImport] - # @RELATION: CALLS -> [APIClient] - def import_dashboard( - self, - file_name: Union[str, Path], - dash_id: Optional[int] = None, - dash_slug: Optional[str] = None, - ) -> Dict: - with belief_scope("import_dashboard"): - if file_name is None: - raise ValueError("file_name cannot be None") - file_path = str(file_name) - self._validate_import_file(file_path) - try: - return self._do_import(file_path) - except Exception as exc: - app_logger.error( - "[import_dashboard][Failure] First import attempt failed: %s", - exc, - exc_info=True, - ) - if not self.delete_before_reimport: - raise - - target_id = self._resolve_target_id_for_delete(dash_id, dash_slug) - if target_id is None: - app_logger.error( - "[import_dashboard][Failure] No ID available for delete-retry." - ) - raise - - self.delete_dashboard(target_id) - app_logger.info( - "[import_dashboard][State] Deleted dashboard ID %s, retrying import.", - target_id, - ) - return self._do_import(file_path) - - # [/DEF:SupersetClientImportDashboard:Function] - - # [DEF:SupersetClientDeleteDashboard:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Удаляет дашборд по его ID или slug. - # @PRE: dashboard_id must exist. - # @POST: Dashboard is removed from Superset. - # @SIDE_EFFECT: Deletes resource from upstream Superset environment. - # @RELATION: CALLS -> [APIClient] - def delete_dashboard(self, dashboard_id: Union[int, str]) -> None: - with belief_scope("delete_dashboard"): - app_logger.info( - "[delete_dashboard][Enter] Deleting dashboard %s.", dashboard_id - ) - response = self.network.request( - method="DELETE", endpoint=f"/dashboard/{dashboard_id}" - ) - response = cast(Dict, response) - if response.get("result", True) is not False: - app_logger.info( - "[delete_dashboard][Success] Dashboard %s deleted.", dashboard_id - ) - else: - app_logger.warning( - "[delete_dashboard][Warning] Unexpected response while deleting %s: %s", - dashboard_id, - response, - ) - - # [/DEF:SupersetClientDeleteDashboard:Function] - - # [DEF:SupersetClientGetDatasets:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Получает полный список датасетов, автоматически обрабатывая пагинацию. - # @PRE: Client is authenticated. - # @POST: Returns total count and list of datasets. - # @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]] - # @RELATION: CALLS -> [SupersetClientFetchAllPages] - def get_datasets(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]: - with belief_scope("get_datasets"): - app_logger.info("[get_datasets][Enter] Fetching datasets.") - validated_query = self._validate_query_params(query) - - paginated_data = self._fetch_all_pages( - endpoint="/dataset/", - pagination_options={ - "base_query": validated_query, - "results_field": "result", - }, - ) - total_count = len(paginated_data) - app_logger.info("[get_datasets][Exit] Found %d datasets.", total_count) - return total_count, paginated_data - - # [/DEF:SupersetClientGetDatasets:Function] - - # [DEF:SupersetClientGetDatasetsSummary:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Fetches dataset metadata optimized for the Dataset Hub grid. - # @PRE: Client is authenticated. - # @POST: Returns a list of dataset metadata summaries. - # @RETURN: List[Dict] - # @RELATION: CALLS -> [SupersetClientGetDatasets] - def get_datasets_summary(self) -> List[Dict]: - with belief_scope("SupersetClient.get_datasets_summary"): - query = {"columns": ["id", "table_name", "schema", "database"]} - _, datasets = self.get_datasets(query=query) - - # Map fields to match the contracts - result = [] - for ds in datasets: - result.append( - { - "id": ds.get("id"), - "table_name": ds.get("table_name"), - "schema": ds.get("schema"), - "database": ds.get("database", {}).get( - "database_name", "Unknown" - ), - } - ) - return result - - # [/DEF:SupersetClientGetDatasetsSummary:Function] - - # [DEF:SupersetClientGetDatasetDetail:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Fetches detailed dataset information including columns and linked dashboards - # @PRE: Client is authenticated and dataset_id exists. - # @POST: Returns detailed dataset info with columns and linked dashboards. - # @PARAM: dataset_id (int) - The dataset ID to fetch details for. - # @RETURN: Dict - Dataset details with columns and linked_dashboards. - # @RELATION: CALLS -> [SupersetClientGetDataset] - # @RELATION: CALLS -> [APIClient] - def get_dataset_detail(self, dataset_id: int) -> Dict: - with belief_scope("SupersetClient.get_dataset_detail", f"id={dataset_id}"): - - def as_bool(value, default=False): - if value is None: - return default - if isinstance(value, bool): - return value - if isinstance(value, str): - return value.strip().lower() in ("1", "true", "yes", "y", "on") - return bool(value) - - # Get base dataset info - response = self.get_dataset(dataset_id) - - # If the response is a dict and has a 'result' key, use that (standard Superset API) - if isinstance(response, dict) and "result" in response: - dataset = response["result"] - else: - dataset = response - - # Extract columns information - columns = dataset.get("columns", []) - column_info = [] - for col in columns: - col_id = col.get("id") - if col_id is None: - continue - column_info.append( - { - "id": int(col_id), - "name": col.get("column_name"), - "type": col.get("type"), - "is_dttm": as_bool(col.get("is_dttm"), default=False), - "is_active": as_bool(col.get("is_active"), default=True), - "description": col.get("description", ""), - } - ) - - # Get linked dashboards using related_objects endpoint - linked_dashboards = [] - try: - related_objects = self.network.request( - method="GET", endpoint=f"/dataset/{dataset_id}/related_objects" - ) - - # Handle different response formats - if isinstance(related_objects, dict): - if "dashboards" in related_objects: - dashboards_data = related_objects["dashboards"] - elif "result" in related_objects and isinstance( - related_objects["result"], dict - ): - dashboards_data = related_objects["result"].get( - "dashboards", [] - ) - else: - dashboards_data = [] - - for dash in dashboards_data: - if isinstance(dash, dict): - dash_id = dash.get("id") - if dash_id is None: - continue - linked_dashboards.append( - { - "id": int(dash_id), - "title": dash.get("dashboard_title") - or dash.get("title", f"Dashboard {dash_id}"), - "slug": dash.get("slug"), - } - ) - else: - try: - dash_id = int(dash) - except (TypeError, ValueError): - continue - linked_dashboards.append( - { - "id": dash_id, - "title": f"Dashboard {dash_id}", - "slug": None, - } - ) - except Exception as e: - app_logger.warning( - f"[get_dataset_detail][Warning] Failed to fetch related dashboards: {e}" - ) - linked_dashboards = [] - - # Extract SQL table information - sql = dataset.get("sql", "") - - result = { - "id": dataset.get("id"), - "table_name": dataset.get("table_name"), - "schema": dataset.get("schema"), - "database": ( - dataset.get("database", {}).get("database_name", "Unknown") - if isinstance(dataset.get("database"), dict) - else dataset.get("database_name") or "Unknown" - ), - "description": dataset.get("description", ""), - "columns": column_info, - "column_count": len(column_info), - "sql": sql, - "linked_dashboards": linked_dashboards, - "linked_dashboard_count": len(linked_dashboards), - "is_sqllab_view": as_bool(dataset.get("is_sqllab_view"), default=False), - "created_on": dataset.get("created_on"), - "changed_on": dataset.get("changed_on"), - } - - app_logger.info( - f"[get_dataset_detail][Exit] Got dataset {dataset_id} with {len(column_info)} columns and {len(linked_dashboards)} linked dashboards" - ) - return result - - # [/DEF:SupersetClientGetDatasetDetail:Function] - - # [DEF:SupersetClientGetDataset:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Получает информацию о конкретном датасете по его ID. - # @PRE: dataset_id must exist. - # @POST: Returns dataset details. - # @DATA_CONTRACT: Input[dataset_id: int] -> Output[Dict] - # @RELATION: CALLS -> [APIClient] - def get_dataset(self, dataset_id: int) -> Dict: - with belief_scope("SupersetClient.get_dataset", f"id={dataset_id}"): - app_logger.info("[get_dataset][Enter] Fetching dataset %s.", dataset_id) - response = self.network.request( - method="GET", endpoint=f"/dataset/{dataset_id}" - ) - response = cast(Dict, response) - app_logger.info("[get_dataset][Exit] Got dataset %s.", dataset_id) - return response - - # [/DEF:SupersetClientGetDataset:Function] - -from src.logger import belief_scope, logger - - # [DEF:SupersetClientCompileDatasetPreview:Function] - # @COMPLEXITY: 4 - # @PURPOSE: Compile dataset preview SQL through the strongest supported Superset preview endpoint family and return normalized SQL output. - # @PRE: dataset_id must be valid and template_params/effective_filters must represent the current preview session inputs. - # @POST: Returns normalized compiled SQL plus raw upstream response, preferring legacy form_data transport with explicit fallback to chart-data. - # @DATA_CONTRACT: Input[dataset_id:int, template_params:Dict, effective_filters:List[Dict]] -> Output[Dict[str, Any]] - # @RELATION: CALLS -> [SupersetClientGetDataset] - # @RELATION: CALLS -> [SupersetClientBuildDatasetPreviewQueryContext] - # @RELATION: CALLS -> [SupersetClientBuildDatasetPreviewLegacyFormData] - # @RELATION: CALLS -> [APIClient] - # @RELATION: CALLS -> [SupersetClientExtractCompiledSqlFromPreviewResponse] - # @SIDE_EFFECT: Performs upstream dataset lookup and preview network I/O against Superset. - def compile_dataset_preview(self, dataset_id: int, template_params: Optional[Dict[str, Any]]=None, effective_filters: Optional[List[Dict[str, Any]]]=None) -> Dict[str, Any]: - with belief_scope('SupersetClientCompileDatasetPreview'): - logger.reason('Belief protocol reasoning checkpoint for SupersetClientCompileDatasetPreview') - dataset_response = self.get_dataset(dataset_id) - dataset_record = dataset_response.get('result', dataset_response) if isinstance(dataset_response, dict) else {} - query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params or {}, effective_filters=effective_filters or []) - legacy_form_data = self.build_dataset_preview_legacy_form_data(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params or {}, effective_filters=effective_filters or []) - legacy_form_data_payload = json.dumps(legacy_form_data, sort_keys=True, default=str) - request_payload = json.dumps(query_context) - strategy_attempts: List[Dict[str, Any]] = [] - strategy_candidates: List[Dict[str, Any]] = [{'endpoint_kind': 'legacy_explore_form_data', 'endpoint': '/explore_json/form_data', 'request_transport': 'query_param_form_data', 'params': {'form_data': legacy_form_data_payload}}, {'endpoint_kind': 'legacy_data_form_data', 'endpoint': '/data', 'request_transport': 'query_param_form_data', 'params': {'form_data': legacy_form_data_payload}}, {'endpoint_kind': 'v1_chart_data', 'endpoint': '/chart/data', 'request_transport': 'json_body', 'data': request_payload, 'headers': {'Content-Type': 'application/json'}}] - for candidate in strategy_candidates: - endpoint_kind = candidate['endpoint_kind'] - endpoint_path = candidate['endpoint'] - request_transport = candidate['request_transport'] - request_params = deepcopy(candidate.get('params') or {}) - request_body = candidate.get('data') - request_headers = deepcopy(candidate.get('headers') or {}) - request_param_keys = sorted(request_params.keys()) - request_payload_keys: List[str] = [] - if isinstance(request_body, str): - try: - decoded_request_body = json.loads(request_body) - if isinstance(decoded_request_body, dict): - request_payload_keys = sorted(decoded_request_body.keys()) - except json.JSONDecodeError: - request_payload_keys = [] - elif isinstance(request_body, dict): - request_payload_keys = sorted(request_body.keys()) - strategy_diagnostics = {'endpoint': endpoint_path, 'endpoint_kind': endpoint_kind, 'request_transport': request_transport, 'contains_root_datasource': endpoint_kind == 'v1_chart_data' and 'datasource' in query_context, 'contains_form_datasource': endpoint_kind.startswith('legacy_') and 'datasource' in legacy_form_data, 'contains_query_object_datasource': bool(query_context.get('queries')) and isinstance(query_context['queries'][0], dict) and ('datasource' in query_context['queries'][0]), 'request_param_keys': request_param_keys, 'request_payload_keys': request_payload_keys} - app_logger.reason('Attempting Superset dataset preview compilation strategy', extra={'dataset_id': dataset_id, **strategy_diagnostics, 'request_params': request_params, 'request_payload': request_body, 'legacy_form_data': legacy_form_data if endpoint_kind.startswith('legacy_') else None, 'query_context': query_context if endpoint_kind == 'v1_chart_data' else None, 'template_param_count': len(template_params or {}), 'filter_count': len(effective_filters or [])}) - try: - response = self.network.request(method='POST', endpoint=endpoint_path, params=request_params or None, data=request_body, headers=request_headers or None) - normalized = self._extract_compiled_sql_from_preview_response(response) - normalized['query_context'] = query_context - normalized['legacy_form_data'] = legacy_form_data - normalized['endpoint'] = endpoint_path - normalized['endpoint_kind'] = endpoint_kind - normalized['dataset_id'] = dataset_id - normalized['strategy_attempts'] = strategy_attempts + [{**strategy_diagnostics, 'success': True}] - app_logger.reflect('Dataset preview compilation returned normalized SQL payload', extra={'dataset_id': dataset_id, **strategy_diagnostics, 'success': True, 'compiled_sql_length': len(str(normalized.get('compiled_sql') or '')), 'response_diagnostics': normalized.get('response_diagnostics')}) - logger.reflect('Belief protocol postcondition checkpoint for SupersetClientCompileDatasetPreview') - return normalized - except Exception as exc: - failure_diagnostics = {**strategy_diagnostics, 'success': False, 'error': str(exc)} - strategy_attempts.append(failure_diagnostics) - app_logger.explore('Superset dataset preview compilation strategy failed', extra={'dataset_id': dataset_id, **failure_diagnostics, 'request_params': request_params, 'request_payload': request_body}) - raise SupersetAPIError(f'Superset preview compilation failed for all known strategies (attempts={strategy_attempts!r})') - # [/DEF:SupersetClientCompileDatasetPreview:Function] - - # [DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function] - # @COMPLEXITY: 4 - # @PURPOSE: Build browser-style legacy form_data payload for Superset preview endpoints inferred from observed deployment traffic. - # @PRE: dataset_record should come from Superset dataset detail when possible. - # @POST: Returns one serialized-ready form_data structure preserving native filter clauses in legacy transport fields. - # @DATA_CONTRACT: Input[dataset_id:int,dataset_record:Dict,template_params:Dict,effective_filters:List[Dict]] -> Output[Dict[str, Any]] - # @RELATION: CALLS -> [SupersetClientBuildDatasetPreviewQueryContext] - # @SIDE_EFFECT: Emits reasoning diagnostics describing the inferred legacy payload shape. - def build_dataset_preview_legacy_form_data(self, dataset_id: int, dataset_record: Dict[str, Any], template_params: Dict[str, Any], effective_filters: List[Dict[str, Any]]) -> Dict[str, Any]: - with belief_scope('SupersetClientBuildDatasetPreviewLegacyFormData'): - logger.reason('Belief protocol reasoning checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData') - query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params, effective_filters=effective_filters) - query_object = deepcopy(query_context.get('queries', [{}])[0] if query_context.get('queries') else {}) - legacy_form_data = deepcopy(query_context.get('form_data', {})) - legacy_form_data.pop('datasource', None) - legacy_form_data['metrics'] = deepcopy(query_object.get('metrics', ['count'])) - legacy_form_data['columns'] = deepcopy(query_object.get('columns', [])) - legacy_form_data['orderby'] = deepcopy(query_object.get('orderby', [])) - legacy_form_data['annotation_layers'] = deepcopy(query_object.get('annotation_layers', [])) - legacy_form_data['row_limit'] = query_object.get('row_limit', 1000) - legacy_form_data['series_limit'] = query_object.get('series_limit', 0) - legacy_form_data['url_params'] = deepcopy(query_object.get('url_params', template_params)) - legacy_form_data['applied_time_extras'] = deepcopy(query_object.get('applied_time_extras', {})) - legacy_form_data['result_format'] = query_context.get('result_format', 'json') - legacy_form_data['result_type'] = query_context.get('result_type', 'query') - legacy_form_data['force'] = bool(query_context.get('force', True)) - extras = query_object.get('extras') - if isinstance(extras, dict): - legacy_form_data['extras'] = deepcopy(extras) - time_range = query_object.get('time_range') - if time_range: - legacy_form_data['time_range'] = time_range - app_logger.reflect('Built Superset legacy preview form_data payload from browser-observed request shape', extra={'dataset_id': dataset_id, 'legacy_endpoint_inference': 'POST /explore_json/form_data?form_data=... primary, POST /data?form_data=... fallback, based on observed browser traffic', 'contains_form_datasource': 'datasource' in legacy_form_data, 'legacy_form_data_keys': sorted(legacy_form_data.keys()), 'legacy_extra_filters': legacy_form_data.get('extra_filters', []), 'legacy_extra_form_data': legacy_form_data.get('extra_form_data', {})}) - logger.reflect('Belief protocol postcondition checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData') - return legacy_form_data - # [/DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function] - - # [DEF:SupersetClientBuildDatasetPreviewQueryContext:Function] - # @COMPLEXITY: 4 - # @PURPOSE: Build a reduced-scope chart-data query context for deterministic dataset preview compilation. - # @PRE: dataset_record should come from Superset dataset detail when possible. - # @POST: Returns an explicit chart-data payload based on current session inputs and dataset metadata. - # @DATA_CONTRACT: Input[dataset_id:int,dataset_record:Dict,template_params:Dict,effective_filters:List[Dict]] -> Output[Dict[str, Any]] - # @RELATION: CALLS -> [SupersetClientNormalizeEffectiveFiltersForQueryContext] - # @SIDE_EFFECT: Emits reasoning and reflection logs for deterministic preview payload construction. - def build_dataset_preview_query_context( - self, - dataset_id: int, - dataset_record: Dict[str, Any], - template_params: Dict[str, Any], - effective_filters: List[Dict[str, Any]], - ) -> Dict[str, Any]: - with belief_scope("SupersetClientBuildDatasetPreviewQueryContext"): - app_logger.reason( - "Building Superset dataset preview query context", - extra={"dataset_id": dataset_id, "filter_count": len(effective_filters or [])}, - ) - normalized_template_params = deepcopy(template_params or {}) - normalized_filter_payload = ( - self._normalize_effective_filters_for_query_context( - effective_filters or [] - ) - ) - normalized_filters = normalized_filter_payload["filters"] - normalized_extra_form_data = normalized_filter_payload["extra_form_data"] - - datasource_payload: Dict[str, Any] = { - "id": dataset_id, - "type": "table", - } - datasource = dataset_record.get("datasource") - if isinstance(datasource, dict): - datasource_id = datasource.get("id") - datasource_type = datasource.get("type") - if datasource_id is not None: - datasource_payload["id"] = datasource_id - if datasource_type: - datasource_payload["type"] = datasource_type - - serialized_dataset_template_params = dataset_record.get("template_params") - if ( - isinstance(serialized_dataset_template_params, str) - and serialized_dataset_template_params.strip() - ): - try: - parsed_dataset_template_params = json.loads( - serialized_dataset_template_params - ) - if isinstance(parsed_dataset_template_params, dict): - for key, value in parsed_dataset_template_params.items(): - normalized_template_params.setdefault(str(key), value) - except json.JSONDecodeError: - app_logger.explore( - "Dataset template_params could not be parsed while building preview query context", - extra={"dataset_id": dataset_id}, - ) - - extra_form_data: Dict[str, Any] = deepcopy(normalized_extra_form_data) - if normalized_filters: - extra_form_data["filters"] = deepcopy(normalized_filters) - - query_object: Dict[str, Any] = { - "filters": normalized_filters, - "extras": {"where": ""}, - "columns": [], - "metrics": ["count"], - "orderby": [], - "annotation_layers": [], - "row_limit": 1000, - "series_limit": 0, - "url_params": normalized_template_params, - "applied_time_extras": {}, - "result_type": "query", - } - - schema = dataset_record.get("schema") - if schema: - query_object["schema"] = schema - - time_range = extra_form_data.get("time_range") or dataset_record.get( - "default_time_range" - ) - if time_range: - query_object["time_range"] = time_range - extra_form_data["time_range"] = time_range - - result_format = dataset_record.get("result_format") or "json" - result_type = "query" - - form_data: Dict[str, Any] = { - "datasource": f"{datasource_payload['id']}__{datasource_payload['type']}", - "datasource_id": datasource_payload["id"], - "datasource_type": datasource_payload["type"], - "viz_type": "table", - "slice_id": None, - "query_mode": "raw", - "url_params": normalized_template_params, - "extra_filters": deepcopy(normalized_filters), - "adhoc_filters": [], - } - if extra_form_data: - form_data["extra_form_data"] = extra_form_data - - payload = { - "datasource": datasource_payload, - "queries": [query_object], - "form_data": form_data, - "result_format": result_format, - "result_type": result_type, - "force": True, - } - app_logger.reflect( - "Built Superset dataset preview query context", - extra={ - "dataset_id": dataset_id, - "datasource": datasource_payload, - "normalized_effective_filters": normalized_filters, - "normalized_filter_diagnostics": normalized_filter_payload[ - "diagnostics" - ], - "result_type": result_type, - "result_format": result_format, - }, - ) - return payload - - # [/DEF:SupersetClientBuildDatasetPreviewQueryContext:Function] - - # [DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Convert execution mappings into Superset chart-data filter objects. - # @PRE: effective_filters may contain mapping metadata and arbitrary scalar/list values. - # @POST: Returns only valid filter dictionaries suitable for the chart-data query payload. - # @RELATION: DEPENDS_ON -> [SupersetClient] - def _normalize_effective_filters_for_query_context( - self, - effective_filters: List[Dict[str, Any]], - ) -> Dict[str, Any]: - with belief_scope( - "SupersetClient._normalize_effective_filters_for_query_context" - ): - normalized_filters: List[Dict[str, Any]] = [] - merged_extra_form_data: Dict[str, Any] = {} - diagnostics: List[Dict[str, Any]] = [] - - for item in effective_filters: - if not isinstance(item, dict): - continue - - display_name = str( - item.get("display_name") - or item.get("filter_name") - or item.get("variable_name") - or "unresolved_filter" - ).strip() - value = item.get("effective_value") - normalized_payload = item.get("normalized_filter_payload") - preserved_clauses: List[Dict[str, Any]] = [] - preserved_extra_form_data: Dict[str, Any] = {} - used_preserved_clauses = False - - if isinstance(normalized_payload, dict): - raw_clauses = normalized_payload.get("filter_clauses") - if isinstance(raw_clauses, list): - preserved_clauses = [ - deepcopy(clause) - for clause in raw_clauses - if isinstance(clause, dict) - ] - raw_extra_form_data = normalized_payload.get("extra_form_data") - if isinstance(raw_extra_form_data, dict): - preserved_extra_form_data = deepcopy(raw_extra_form_data) - - if isinstance(preserved_extra_form_data, dict): - for key, extra_value in preserved_extra_form_data.items(): - if key == "filters": - continue - merged_extra_form_data[key] = deepcopy(extra_value) - - outgoing_clauses: List[Dict[str, Any]] = [] - if preserved_clauses: - for clause in preserved_clauses: - clause_copy = deepcopy(clause) - if "val" not in clause_copy and value is not None: - clause_copy["val"] = deepcopy(value) - outgoing_clauses.append(clause_copy) - used_preserved_clauses = True - elif preserved_extra_form_data: - outgoing_clauses = [] - else: - column = str( - item.get("variable_name") or item.get("filter_name") or "" - ).strip() - if column and value is not None: - operator = "IN" if isinstance(value, list) else "==" - outgoing_clauses.append( - { - "col": column, - "op": operator, - "val": value, - } - ) - - normalized_filters.extend(outgoing_clauses) - diagnostics.append( - { - "filter_name": display_name, - "value_origin": ( - normalized_payload.get("value_origin") - if isinstance(normalized_payload, dict) - else None - ), - "used_preserved_clauses": used_preserved_clauses, - "outgoing_clauses": deepcopy(outgoing_clauses), - } - ) - app_logger.reason( - "Normalized effective preview filter for Superset query context", - extra={ - "filter_name": display_name, - "used_preserved_clauses": used_preserved_clauses, - "outgoing_clauses": outgoing_clauses, - "value_origin": ( - normalized_payload.get("value_origin") - if isinstance(normalized_payload, dict) - else "heuristic_reconstruction" - ), - }, - ) - - return { - "filters": normalized_filters, - "extra_form_data": merged_extra_form_data, - "diagnostics": diagnostics, - } - - # [/DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function] - - # [DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Normalize compiled SQL from either chart-data or legacy form_data preview responses. - # @PRE: response must be the decoded preview response body from a supported Superset endpoint. - # @POST: Returns compiled SQL and raw response or raises SupersetAPIError when the endpoint does not expose query text. - # @RELATION: DEPENDS_ON -> [APIClient] - def _extract_compiled_sql_from_preview_response( - self, response: Any - ) -> Dict[str, Any]: - with belief_scope("SupersetClient._extract_compiled_sql_from_preview_response"): - if not isinstance(response, dict): - raise SupersetAPIError( - "Superset preview response was not a JSON object" - ) - - response_diagnostics: List[Dict[str, Any]] = [] - result_payload = response.get("result") - if isinstance(result_payload, list): - for index, item in enumerate(result_payload): - if not isinstance(item, dict): - continue - compiled_sql = str( - item.get("query") - or item.get("sql") - or item.get("compiled_sql") - or "" - ).strip() - response_diagnostics.append( - { - "index": index, - "status": item.get("status"), - "applied_filters": item.get("applied_filters"), - "rejected_filters": item.get("rejected_filters"), - "has_query": bool(compiled_sql), - "source": "result_list", - } - ) - if compiled_sql: - return { - "compiled_sql": compiled_sql, - "raw_response": response, - "response_diagnostics": response_diagnostics, - } - - top_level_candidates: List[Tuple[str, Any]] = [ - ("query", response.get("query")), - ("sql", response.get("sql")), - ("compiled_sql", response.get("compiled_sql")), - ] - if isinstance(result_payload, dict): - top_level_candidates.extend( - [ - ("result.query", result_payload.get("query")), - ("result.sql", result_payload.get("sql")), - ("result.compiled_sql", result_payload.get("compiled_sql")), - ] - ) - - for source, candidate in top_level_candidates: - compiled_sql = str(candidate or "").strip() - response_diagnostics.append( - { - "source": source, - "has_query": bool(compiled_sql), - } - ) - if compiled_sql: - return { - "compiled_sql": compiled_sql, - "raw_response": response, - "response_diagnostics": response_diagnostics, - } - - raise SupersetAPIError( - "Superset preview response did not expose compiled SQL " - f"(diagnostics={response_diagnostics!r})" - ) - - # [/DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function] - - # [DEF:SupersetClientUpdateDataset:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Обновляет данные датасета по его ID. - # @PRE: dataset_id must exist. - # @POST: Dataset is updated in Superset. - # @DATA_CONTRACT: Input[dataset_id: int, data: Dict] -> Output[Dict] - # @SIDE_EFFECT: Modifies resource in upstream Superset environment. - # @RELATION: CALLS -> [APIClient] - def update_dataset(self, dataset_id: int, data: Dict) -> Dict: - with belief_scope("SupersetClient.update_dataset", f"id={dataset_id}"): - app_logger.info("[update_dataset][Enter] Updating dataset %s.", dataset_id) - response = self.network.request( - method="PUT", - endpoint=f"/dataset/{dataset_id}", - data=json.dumps(data), - headers={"Content-Type": "application/json"}, - ) - response = cast(Dict, response) - app_logger.info("[update_dataset][Exit] Updated dataset %s.", dataset_id) - return response - - # [/DEF:SupersetClientUpdateDataset:Function] - - # [DEF:SupersetClientGetDatabases:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Получает полный список баз данных. - # @PRE: Client is authenticated. - # @POST: Returns total count and list of databases. - # @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]] - # @RELATION: CALLS -> [SupersetClientFetchAllPages] - def get_databases(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]: - with belief_scope("get_databases"): - app_logger.info("[get_databases][Enter] Fetching databases.") - validated_query = self._validate_query_params(query or {}) - if "columns" not in validated_query: - validated_query["columns"] = [] - - paginated_data = self._fetch_all_pages( - endpoint="/database/", - pagination_options={ - "base_query": validated_query, - "results_field": "result", - }, - ) - total_count = len(paginated_data) - app_logger.info("[get_databases][Exit] Found %d databases.", total_count) - return total_count, paginated_data - - # [/DEF:SupersetClientGetDatabases:Function] - - # [DEF:SupersetClientGetDatabase:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Получает информацию о конкретной базе данных по её ID. - # @PRE: database_id must exist. - # @POST: Returns database details. - # @DATA_CONTRACT: Input[database_id: int] -> Output[Dict] - # @RELATION: CALLS -> [APIClient] - def get_database(self, database_id: int) -> Dict: - with belief_scope("get_database"): - app_logger.info("[get_database][Enter] Fetching database %s.", database_id) - response = self.network.request( - method="GET", endpoint=f"/database/{database_id}" - ) - response = cast(Dict, response) - app_logger.info("[get_database][Exit] Got database %s.", database_id) - return response - - # [/DEF:SupersetClientGetDatabase:Function] - - # [DEF:SupersetClientGetDatabasesSummary:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Fetch a summary of databases including uuid, name, and engine. - # @PRE: Client is authenticated. - # @POST: Returns list of database summaries. - # @DATA_CONTRACT: None -> Output[List[Dict]] - # @RELATION: CALLS -> [SupersetClientGetDatabases] - def get_databases_summary(self) -> List[Dict]: - with belief_scope("SupersetClient.get_databases_summary"): - query = {"columns": ["uuid", "database_name", "backend"]} - _, databases = self.get_databases(query=query) - - # Map 'backend' to 'engine' for consistency with contracts - for db in databases: - db["engine"] = db.pop("backend", None) - - return databases - - # [/DEF:SupersetClientGetDatabasesSummary:Function] - - # [DEF:SupersetClientGetDatabaseByUuid:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Find a database by its UUID. - # @PRE: db_uuid must be a valid UUID string. - # @POST: Returns database info or None. - # @DATA_CONTRACT: Input[db_uuid: str] -> Output[Optional[Dict]] - # @RELATION: CALLS -> [SupersetClientGetDatabases] - def get_database_by_uuid(self, db_uuid: str) -> Optional[Dict]: - with belief_scope("SupersetClient.get_database_by_uuid", f"uuid={db_uuid}"): - query = {"filters": [{"col": "uuid", "op": "eq", "value": db_uuid}]} - _, databases = self.get_databases(query=query) - return databases[0] if databases else None - - # [/DEF:SupersetClientGetDatabaseByUuid:Function] - - # [DEF:SupersetClientResolveTargetIdForDelete:Function] - # @COMPLEXITY: 1 - # @PURPOSE: Resolves a dashboard ID from either an ID or a slug. - # @PRE: Either dash_id or dash_slug should be provided. - # @POST: Returns the resolved ID or None. - # @RELATION: CALLS -> [SupersetClientGetDashboards] - def _resolve_target_id_for_delete( - self, dash_id: Optional[int], dash_slug: Optional[str] - ) -> Optional[int]: - with belief_scope("_resolve_target_id_for_delete"): - if dash_id is not None: - return dash_id - if dash_slug is not None: - app_logger.debug( - "[_resolve_target_id_for_delete][State] Resolving ID by slug '%s'.", - dash_slug, - ) - try: - _, candidates = self.get_dashboards( - query={ - "filters": [{"col": "slug", "op": "eq", "value": dash_slug}] - } - ) - if candidates: - target_id = candidates[0]["id"] - app_logger.debug( - "[_resolve_target_id_for_delete][Success] Resolved slug to ID %s.", - target_id, - ) - return target_id - except Exception as e: - app_logger.warning( - "[_resolve_target_id_for_delete][Warning] Could not resolve slug '%s' to ID: %s", - dash_slug, - e, - ) - return None - - # [/DEF:SupersetClientResolveTargetIdForDelete:Function] - - # [DEF:SupersetClientDoImport:Function] - # @COMPLEXITY: 1 - # @PURPOSE: Performs the actual multipart upload for import. - # @PRE: file_name must be a path to an existing ZIP file. - # @POST: Returns the API response from the upload. - # @RELATION: CALLS -> [APIClient] - def _do_import(self, file_name: Union[str, Path]) -> Dict: - with belief_scope("_do_import"): - app_logger.debug(f"[_do_import][State] Uploading file: {file_name}") - file_path = Path(file_name) - if not file_path.exists(): - app_logger.error( - f"[_do_import][Failure] File does not exist: {file_name}" - ) - raise FileNotFoundError(f"File does not exist: {file_name}") - - return self.network.upload_file( - endpoint="/dashboard/import/", - file_info={ - "file_obj": file_path, - "file_name": file_path.name, - "form_field": "formData", - }, - extra_data={"overwrite": "true"}, - timeout=self.env.timeout * 2, - ) - - # [/DEF:SupersetClientDoImport:Function] - - # [DEF:SupersetClientValidateExportResponse:Function] - # @COMPLEXITY: 1 - # @PURPOSE: Validates that the export response is a non-empty ZIP archive. - # @PRE: response must be a valid requests.Response object. - # @POST: Raises SupersetAPIError if validation fails. - def _validate_export_response(self, response: Response, dashboard_id: int) -> None: - with belief_scope("_validate_export_response"): - content_type = response.headers.get("Content-Type", "") - if "application/zip" not in content_type: - raise SupersetAPIError( - f"Получен не ZIP-архив (Content-Type: {content_type})" - ) - if not response.content: - raise SupersetAPIError("Получены пустые данные при экспорте") - - # [/DEF:SupersetClientValidateExportResponse:Function] - - # [DEF:SupersetClientResolveExportFilename:Function] - # @COMPLEXITY: 1 - # @PURPOSE: Determines the filename for an exported dashboard. - # @PRE: response must contain Content-Disposition header or dashboard_id must be provided. - # @POST: Returns a sanitized filename string. - def _resolve_export_filename(self, response: Response, dashboard_id: int) -> str: - with belief_scope("_resolve_export_filename"): - filename = get_filename_from_headers(dict(response.headers)) - if not filename: - from datetime import datetime - - timestamp = datetime.now().strftime("%Y%m%dT%H%M%S") - filename = f"dashboard_export_{dashboard_id}_{timestamp}.zip" - app_logger.warning( - "[_resolve_export_filename][Warning] Generated filename: %s", - filename, - ) - return filename - - # [/DEF:SupersetClientResolveExportFilename:Function] - - # [DEF:SupersetClientValidateQueryParams:Function] - # @COMPLEXITY: 1 - # @PURPOSE: Ensures query parameters have default page and page_size. - # @PRE: query can be None or a dictionary. - # @POST: Returns a dictionary with at least page and page_size. - def _validate_query_params(self, query: Optional[Dict]) -> Dict: - with belief_scope("_validate_query_params"): - # Superset list endpoints commonly cap page_size at 100. - # Using 100 avoids partial fetches when larger values are silently truncated. - base_query = {"page": 0, "page_size": 100} - return {**base_query, **(query or {})} - - # [/DEF:SupersetClientValidateQueryParams:Function] - - # [DEF:SupersetClientFetchTotalObjectCount:Function] - # @COMPLEXITY: 1 - # @PURPOSE: Fetches the total number of items for a given endpoint. - # @PRE: endpoint must be a valid Superset API path. - # @POST: Returns the total count as an integer. - # @RELATION: CALLS -> [APIClient] - def _fetch_total_object_count(self, endpoint: str) -> int: - with belief_scope("_fetch_total_object_count"): - return self.network.fetch_paginated_count( - endpoint=endpoint, - query_params={"page": 0, "page_size": 1}, - count_field="count", - ) - - # [/DEF:SupersetClientFetchTotalObjectCount:Function] - - # [DEF:SupersetClientFetchAllPages:Function] - # @COMPLEXITY: 1 - # @PURPOSE: Iterates through all pages to collect all data items. - # @PRE: pagination_options must contain base_query, total_count, and results_field. - # @POST: Returns a combined list of all items. - # @RELATION: CALLS -> [APIClient] - def _fetch_all_pages(self, endpoint: str, pagination_options: Dict) -> List[Dict]: - with belief_scope("_fetch_all_pages"): - return self.network.fetch_paginated_data( - endpoint=endpoint, pagination_options=pagination_options - ) - - # [/DEF:SupersetClientFetchAllPages:Function] - - # [DEF:SupersetClientValidateImportFile:Function] - # @COMPLEXITY: 1 - # @PURPOSE: Validates that the file to be imported is a valid ZIP with metadata.yaml. - # @PRE: zip_path must be a path to a file. - # @POST: Raises error if file is missing, not a ZIP, or missing metadata. - def _validate_import_file(self, zip_path: Union[str, Path]) -> None: - with belief_scope("_validate_import_file"): - path = Path(zip_path) - if not path.exists(): - raise FileNotFoundError(f"Файл {zip_path} не существует") - if not zipfile.is_zipfile(path): - raise SupersetAPIError(f"Файл {zip_path} не является ZIP-архивом") - with zipfile.ZipFile(path, "r") as zf: - if not any(n.endswith("metadata.yaml") for n in zf.namelist()): - raise SupersetAPIError( - f"Архив {zip_path} не содержит 'metadata.yaml'" - ) - - # [/DEF:SupersetClientValidateImportFile:Function] - - # [DEF:SupersetClientGetAllResources:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Fetches all resources of a given type with id, uuid, and name columns. - # @PARAM: resource_type (str) - One of "chart", "dataset", "dashboard". - # @PRE: Client is authenticated. resource_type is valid. - # @POST: Returns a list of resource dicts with at minimum id, uuid, and name fields. - # @RETURN: List[Dict] - # @RELATION: CALLS -> [SupersetClientFetchAllPages] - def get_all_resources( - self, resource_type: str, since_dttm: Optional[datetime] = None - ) -> List[Dict]: - with belief_scope( - "SupersetClient.get_all_resources", - f"type={resource_type}, since={since_dttm}", - ): - column_map = { - "chart": { - "endpoint": "/chart/", - "columns": ["id", "uuid", "slice_name"], - }, - "dataset": { - "endpoint": "/dataset/", - "columns": ["id", "uuid", "table_name"], - }, - "dashboard": { - "endpoint": "/dashboard/", - "columns": ["id", "uuid", "slug", "dashboard_title"], - }, - } - config = column_map.get(resource_type) - if not config: - app_logger.warning( - "[get_all_resources][Warning] Unknown resource type: %s", - resource_type, - ) - return [] - - query = {"columns": config["columns"]} - - if since_dttm: - import math - - # Use int milliseconds to be safe - timestamp_ms = math.floor(since_dttm.timestamp() * 1000) - - query["filters"] = [ - {"col": "changed_on_dttm", "opr": "gt", "value": timestamp_ms} - ] - - validated = self._validate_query_params(query) - data = self._fetch_all_pages( - endpoint=config["endpoint"], - pagination_options={"base_query": validated, "results_field": "result"}, - ) - app_logger.info( - "[get_all_resources][Exit] Fetched %d %s resources.", - len(data), - resource_type, - ) - return data - - # [/DEF:SupersetClientGetAllResources:Function] - - -# [/DEF:SupersetClient:Class] - -# [/DEF:SupersetClientModule:Module] diff --git a/backend/src/core/superset_client/__init__.py b/backend/src/core/superset_client/__init__.py index 5dcb2882..d06e07ae 100644 --- a/backend/src/core/superset_client/__init__.py +++ b/backend/src/core/superset_client/__init__.py @@ -1,12 +1,15 @@ # [DEF:SupersetClientModule:Module] -# # @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: superset, api, client, rest, http, dashboard, dataset, import, export # @PURPOSE: Предоставляет высокоуровневый клиент для взаимодействия с Superset REST API, инкапсулируя логику запросов, обработку ошибок и пагинацию. # @RELATION: DEPENDS_ON -> [ConfigModels] # @RELATION: DEPENDS_ON -> [APIClient] # @RELATION: DEPENDS_ON -> [SupersetAPIError] # @RELATION: DEPENDS_ON -> [get_filename_from_headers] +# @RELATION: DEPENDS_ON -> [SupersetDatasetsPreviewFiltersMixin] # +# @INVARIANT: All network operations must use the internal APIClient instance. # @PUBLIC_API: SupersetClient # # @RATIONALE: Decomposed from monolithic superset_client.py (2145 lines) into @@ -23,6 +26,7 @@ from ._dashboards_crud import SupersetDashboardsCrudMixin from ._charts import SupersetChartsMixin from ._datasets import SupersetDatasetsMixin from ._datasets_preview import SupersetDatasetsPreviewMixin +from ._datasets_preview_filters import SupersetDatasetsPreviewFiltersMixin from ._databases import SupersetDatabasesMixin @@ -40,9 +44,11 @@ from ._databases import SupersetDatabasesMixin # @RELATION: INHERITS -> [SupersetChartsMixin] # @RELATION: INHERITS -> [SupersetDatasetsMixin] # @RELATION: INHERITS -> [SupersetDatasetsPreviewMixin] +# @RELATION: INHERITS -> [SupersetDatasetsPreviewFiltersMixin] # @RELATION: INHERITS -> [SupersetDatabasesMixin] class SupersetClient( SupersetDatabasesMixin, + SupersetDatasetsPreviewFiltersMixin, SupersetDatasetsPreviewMixin, SupersetDatasetsMixin, SupersetChartsMixin, diff --git a/backend/src/core/superset_client/_base.py b/backend/src/core/superset_client/_base.py index b16370c7..de2d2264 100644 --- a/backend/src/core/superset_client/_base.py +++ b/backend/src/core/superset_client/_base.py @@ -1,5 +1,7 @@ # [DEF:SupersetClientBase:Module] # @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: superset, api, client, base, pagination, auth, import, export # @PURPOSE: Base class for SupersetClient providing initialization, authentication, pagination, and import/export helpers. # @RELATION: DEPENDS_ON -> [ConfigModels] # @RELATION: DEPENDS_ON -> [APIClient] diff --git a/backend/src/core/superset_client/_charts.py b/backend/src/core/superset_client/_charts.py index e73626d2..6114adbd 100644 --- a/backend/src/core/superset_client/_charts.py +++ b/backend/src/core/superset_client/_charts.py @@ -1,5 +1,7 @@ # [DEF:SupersetChartsMixin:Module] # @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: superset, charts, list, get, layout # @PURPOSE: Chart domain mixin for SupersetClient — list, get, extract IDs from layout. # @RELATION: DEPENDS_ON -> [SupersetClientBase] diff --git a/backend/src/core/superset_client/_dashboards.py b/backend/src/core/superset_client/_dashboards.py deleted file mode 100644 index 96fae7a9..00000000 --- a/backend/src/core/superset_client/_dashboards.py +++ /dev/null @@ -1,2 +0,0 @@ -# This file has been decomposed into _dashboards_list.py, _dashboards_filters.py, _dashboards_crud.py -# See __init__.py for the composed SupersetClient class. diff --git a/backend/src/core/superset_client/_dashboards_crud.py b/backend/src/core/superset_client/_dashboards_crud.py index 43484ca5..84f25481 100644 --- a/backend/src/core/superset_client/_dashboards_crud.py +++ b/backend/src/core/superset_client/_dashboards_crud.py @@ -1,5 +1,7 @@ # [DEF:SupersetDashboardsCrudMixin:Module] # @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: superset, dashboards, crud, detail, export, import, delete # @PURPOSE: Dashboard CRUD mixin for SupersetClient — detail, export, import, delete. # @RELATION: DEPENDS_ON -> [SupersetClientBase] # @RELATION: DEPENDS_ON -> [SupersetDashboardsFiltersMixin] diff --git a/backend/src/core/superset_client/_dashboards_filters.py b/backend/src/core/superset_client/_dashboards_filters.py index 8bc96bf6..070d51f9 100644 --- a/backend/src/core/superset_client/_dashboards_filters.py +++ b/backend/src/core/superset_client/_dashboards_filters.py @@ -1,5 +1,7 @@ # [DEF:SupersetDashboardsFiltersMixin:Module] # @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: superset, dashboards, filters, permalink, native-filters # @PURPOSE: Dashboard native filter extraction mixin for SupersetClient. # @RELATION: DEPENDS_ON -> [SupersetClientBase] diff --git a/backend/src/core/superset_client/_dashboards_list.py b/backend/src/core/superset_client/_dashboards_list.py index acc7a25f..3f0c70c3 100644 --- a/backend/src/core/superset_client/_dashboards_list.py +++ b/backend/src/core/superset_client/_dashboards_list.py @@ -1,5 +1,7 @@ # [DEF:SupersetDashboardsListMixin:Module] # @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: superset, dashboards, list, pagination, summary # @PURPOSE: Dashboard listing mixin for SupersetClient — paginated list, summary projection. # @RELATION: DEPENDS_ON -> [SupersetClientBase] # @RELATION: DEPENDS_ON -> [SupersetUserProjectionMixin] diff --git a/backend/src/core/superset_client/_databases.py b/backend/src/core/superset_client/_databases.py index 25aacc38..e2bd9ed2 100644 --- a/backend/src/core/superset_client/_databases.py +++ b/backend/src/core/superset_client/_databases.py @@ -1,5 +1,7 @@ # [DEF:SupersetDatabasesMixin:Module] # @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: superset, databases, list, get, summary, uuid # @PURPOSE: Database domain mixin for SupersetClient — list, get, summary, by_uuid. # @RELATION: DEPENDS_ON -> [SupersetClientBase] diff --git a/backend/src/core/superset_client/_datasets.py b/backend/src/core/superset_client/_datasets.py index 9b628f00..11ac0bd3 100644 --- a/backend/src/core/superset_client/_datasets.py +++ b/backend/src/core/superset_client/_datasets.py @@ -1,5 +1,7 @@ # [DEF:SupersetDatasetsMixin:Module] # @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: superset, datasets, list, get, detail, update # @PURPOSE: Dataset domain mixin for SupersetClient — list, get, detail, update. # @RELATION: DEPENDS_ON -> [SupersetClientBase] @@ -147,17 +149,17 @@ class SupersetDatasetsMixin: ) linked_dashboards = [] + database_obj = dataset.get("database") + database_dict = database_obj if isinstance(database_obj, dict) else {} sql = dataset.get("sql", "") result = { "id": dataset.get("id"), "table_name": dataset.get("table_name"), "schema": dataset.get("schema"), - "database": ( - dataset.get("database", {}).get("database_name", "Unknown") - if isinstance(dataset.get("database"), dict) - else dataset.get("database_name") or "Unknown" - ), + "database": database_dict, + "database_id": dataset.get("database_id", database_dict.get("id")), + "database_backend": database_dict.get("backend") or database_dict.get("engine") or "", "description": dataset.get("description", ""), "columns": column_info, "column_count": len(column_info), diff --git a/backend/src/core/superset_client/_datasets_preview.py b/backend/src/core/superset_client/_datasets_preview.py index 61b553b6..b0cff164 100644 --- a/backend/src/core/superset_client/_datasets_preview.py +++ b/backend/src/core/superset_client/_datasets_preview.py @@ -1,13 +1,19 @@ # [DEF:SupersetDatasetsPreviewMixin:Module] +# @COMPLEXITY: 4 +# @LAYER: Infra +# @SEMANTICS: superset, datasets, preview, sql, compilation, filters # @PURPOSE: Dataset preview compilation mixin for SupersetClient — build query context, compile SQL. +# @RELATION: DEPENDS_ON -> [SupersetClientBase] +# @RELATION: DEPENDS_ON -> [SupersetDatasetsMixin] import json from copy import deepcopy -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional from ..logger import logger as app_logger, belief_scope from ..utils.network import SupersetAPIError # [DEF:SupersetDatasetsPreviewMixin:Class] +# @COMPLEXITY: 4 # @PURPOSE: Mixin providing dataset preview compilation and query context building. class SupersetDatasetsPreviewMixin: # [DEF:SupersetClientCompileDatasetPreview:Function] @@ -221,177 +227,5 @@ class SupersetDatasetsPreviewMixin: # [/DEF:SupersetClientBuildDatasetPreviewQueryContext:Function] - # [DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function] - def _normalize_effective_filters_for_query_context( - self, - effective_filters: List[Dict[str, Any]], - ) -> Dict[str, Any]: - with belief_scope( - "SupersetClient._normalize_effective_filters_for_query_context" - ): - normalized_filters: List[Dict[str, Any]] = [] - merged_extra_form_data: Dict[str, Any] = {} - diagnostics: List[Dict[str, Any]] = [] - - for item in effective_filters: - if not isinstance(item, dict): - continue - - display_name = str( - item.get("display_name") - or item.get("filter_name") - or item.get("variable_name") - or "unresolved_filter" - ).strip() - value = item.get("effective_value") - normalized_payload = item.get("normalized_filter_payload") - preserved_clauses: List[Dict[str, Any]] = [] - preserved_extra_form_data: Dict[str, Any] = {} - used_preserved_clauses = False - - if isinstance(normalized_payload, dict): - raw_clauses = normalized_payload.get("filter_clauses") - if isinstance(raw_clauses, list): - preserved_clauses = [ - deepcopy(clause) - for clause in raw_clauses - if isinstance(clause, dict) - ] - raw_extra_form_data = normalized_payload.get("extra_form_data") - if isinstance(raw_extra_form_data, dict): - preserved_extra_form_data = deepcopy(raw_extra_form_data) - - if isinstance(preserved_extra_form_data, dict): - for key, extra_value in preserved_extra_form_data.items(): - if key == "filters": - continue - merged_extra_form_data[key] = deepcopy(extra_value) - - outgoing_clauses: List[Dict[str, Any]] = [] - if preserved_clauses: - for clause in preserved_clauses: - clause_copy = deepcopy(clause) - if "val" not in clause_copy and value is not None: - clause_copy["val"] = deepcopy(value) - outgoing_clauses.append(clause_copy) - used_preserved_clauses = True - elif preserved_extra_form_data: - outgoing_clauses = [] - else: - column = str( - item.get("variable_name") or item.get("filter_name") or "" - ).strip() - if column and value is not None: - operator = "IN" if isinstance(value, list) else "==" - outgoing_clauses.append( - {"col": column, "op": operator, "val": value} - ) - - normalized_filters.extend(outgoing_clauses) - diagnostics.append( - { - "filter_name": display_name, - "value_origin": ( - normalized_payload.get("value_origin") - if isinstance(normalized_payload, dict) - else None - ), - "used_preserved_clauses": used_preserved_clauses, - "outgoing_clauses": deepcopy(outgoing_clauses), - } - ) - app_logger.reason( - "Normalized effective preview filter for Superset query context", - extra={ - "filter_name": display_name, - "used_preserved_clauses": used_preserved_clauses, - "outgoing_clauses": outgoing_clauses, - "value_origin": ( - normalized_payload.get("value_origin") - if isinstance(normalized_payload, dict) - else "heuristic_reconstruction" - ), - }, - ) - - return { - "filters": normalized_filters, - "extra_form_data": merged_extra_form_data, - "diagnostics": diagnostics, - } - - # [/DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function] - - # [DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function] - def _extract_compiled_sql_from_preview_response( - self, response: Any - ) -> Dict[str, Any]: - with belief_scope("SupersetClient._extract_compiled_sql_from_preview_response"): - if not isinstance(response, dict): - raise SupersetAPIError( - "Superset preview response was not a JSON object" - ) - - response_diagnostics: List[Dict[str, Any]] = [] - result_payload = response.get("result") - if isinstance(result_payload, list): - for index, item in enumerate(result_payload): - if not isinstance(item, dict): - continue - compiled_sql = str( - item.get("query") - or item.get("sql") - or item.get("compiled_sql") - or "" - ).strip() - response_diagnostics.append( - { - "index": index, - "status": item.get("status"), - "applied_filters": item.get("applied_filters"), - "rejected_filters": item.get("rejected_filters"), - "has_query": bool(compiled_sql), - "source": "result_list", - } - ) - if compiled_sql: - return { - "compiled_sql": compiled_sql, - "raw_response": response, - "response_diagnostics": response_diagnostics, - } - - top_level_candidates: List[Tuple[str, Any]] = [ - ("query", response.get("query")), - ("sql", response.get("sql")), - ("compiled_sql", response.get("compiled_sql")), - ] - if isinstance(result_payload, dict): - top_level_candidates.extend( - [ - ("result.query", result_payload.get("query")), - ("result.sql", result_payload.get("sql")), - ("result.compiled_sql", result_payload.get("compiled_sql")), - ] - ) - - for source, candidate in top_level_candidates: - compiled_sql = str(candidate or "").strip() - response_diagnostics.append( - {"source": source, "has_query": bool(compiled_sql)} - ) - if compiled_sql: - return { - "compiled_sql": compiled_sql, - "raw_response": response, - "response_diagnostics": response_diagnostics, - } - - raise SupersetAPIError( - "Superset preview response did not expose compiled SQL " - f"(diagnostics={response_diagnostics!r})" - ) - - # [/DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function] # [/DEF:SupersetDatasetsPreviewMixin:Class] # [/DEF:SupersetDatasetsPreviewMixin:Module] diff --git a/backend/src/core/superset_client/_datasets_preview_filters.py b/backend/src/core/superset_client/_datasets_preview_filters.py new file mode 100644 index 00000000..dbe22b0a --- /dev/null +++ b/backend/src/core/superset_client/_datasets_preview_filters.py @@ -0,0 +1,201 @@ +# [DEF:SupersetDatasetsPreviewFiltersMixin:Module] +# @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: superset, datasets, preview, filters, normalization, sql, extraction +# @PURPOSE: Filter normalization and SQL extraction helpers for dataset preview compilation. +# @RELATION: DEPENDS_ON -> [SupersetClientBase] +# @RELATION: DEPENDS_ON -> [SupersetDatasetsPreviewMixin] + +from copy import deepcopy +from typing import Any, Dict, List, Optional, Tuple + +from ..logger import logger as app_logger, belief_scope +from ..utils.network import SupersetAPIError + + +# [DEF:SupersetDatasetsPreviewFiltersMixin:Class] +# @COMPLEXITY: 3 +# @PURPOSE: Mixin providing filter normalization and compiled-SQL extraction for dataset preview operations. +# @RELATION: DEPENDS_ON -> [SupersetClientBase] +# @RELATION: DEPENDS_ON -> [SupersetDatasetsPreviewMixin] +class SupersetDatasetsPreviewFiltersMixin: + # [DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Convert execution mappings into Superset chart-data filter objects. + def _normalize_effective_filters_for_query_context( + self, + effective_filters: List[Dict[str, Any]], + ) -> Dict[str, Any]: + with belief_scope( + "SupersetClient._normalize_effective_filters_for_query_context" + ): + normalized_filters: List[Dict[str, Any]] = [] + merged_extra_form_data: Dict[str, Any] = {} + diagnostics: List[Dict[str, Any]] = [] + + for item in effective_filters: + if not isinstance(item, dict): + continue + + display_name = str( + item.get("display_name") + or item.get("filter_name") + or item.get("variable_name") + or "unresolved_filter" + ).strip() + value = item.get("effective_value") + normalized_payload = item.get("normalized_filter_payload") + preserved_clauses: List[Dict[str, Any]] = [] + preserved_extra_form_data: Dict[str, Any] = {} + used_preserved_clauses = False + + if isinstance(normalized_payload, dict): + raw_clauses = normalized_payload.get("filter_clauses") + if isinstance(raw_clauses, list): + preserved_clauses = [ + deepcopy(clause) + for clause in raw_clauses + if isinstance(clause, dict) + ] + raw_extra_form_data = normalized_payload.get("extra_form_data") + if isinstance(raw_extra_form_data, dict): + preserved_extra_form_data = deepcopy(raw_extra_form_data) + + if isinstance(preserved_extra_form_data, dict): + for key, extra_value in preserved_extra_form_data.items(): + if key == "filters": + continue + merged_extra_form_data[key] = deepcopy(extra_value) + + outgoing_clauses: List[Dict[str, Any]] = [] + if preserved_clauses: + for clause in preserved_clauses: + clause_copy = deepcopy(clause) + if "val" not in clause_copy and value is not None: + clause_copy["val"] = deepcopy(value) + outgoing_clauses.append(clause_copy) + used_preserved_clauses = True + elif preserved_extra_form_data: + outgoing_clauses = [] + else: + column = str( + item.get("variable_name") or item.get("filter_name") or "" + ).strip() + if column and value is not None: + operator = "IN" if isinstance(value, list) else "==" + outgoing_clauses.append( + {"col": column, "op": operator, "val": value} + ) + + normalized_filters.extend(outgoing_clauses) + diagnostics.append( + { + "filter_name": display_name, + "value_origin": ( + normalized_payload.get("value_origin") + if isinstance(normalized_payload, dict) + else None + ), + "used_preserved_clauses": used_preserved_clauses, + "outgoing_clauses": deepcopy(outgoing_clauses), + } + ) + app_logger.reason( + "Normalized effective preview filter for Superset query context", + extra={ + "filter_name": display_name, + "used_preserved_clauses": used_preserved_clauses, + "outgoing_clauses": outgoing_clauses, + "value_origin": ( + normalized_payload.get("value_origin") + if isinstance(normalized_payload, dict) + else "heuristic_reconstruction" + ), + }, + ) + + return { + "filters": normalized_filters, + "extra_form_data": merged_extra_form_data, + "diagnostics": diagnostics, + } + + # [/DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function] + + # [DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Normalize compiled SQL from either chart-data or legacy form_data preview responses. + def _extract_compiled_sql_from_preview_response( + self, response: Any + ) -> Dict[str, Any]: + with belief_scope("SupersetClient._extract_compiled_sql_from_preview_response"): + if not isinstance(response, dict): + raise SupersetAPIError( + "Superset preview response was not a JSON object" + ) + + response_diagnostics: List[Dict[str, Any]] = [] + result_payload = response.get("result") + if isinstance(result_payload, list): + for index, item in enumerate(result_payload): + if not isinstance(item, dict): + continue + compiled_sql = str( + item.get("query") + or item.get("sql") + or item.get("compiled_sql") + or "" + ).strip() + response_diagnostics.append( + { + "index": index, + "status": item.get("status"), + "applied_filters": item.get("applied_filters"), + "rejected_filters": item.get("rejected_filters"), + "has_query": bool(compiled_sql), + "source": "result_list", + } + ) + if compiled_sql: + return { + "compiled_sql": compiled_sql, + "raw_response": response, + "response_diagnostics": response_diagnostics, + } + + top_level_candidates: List[Tuple[str, Any]] = [ + ("query", response.get("query")), + ("sql", response.get("sql")), + ("compiled_sql", response.get("compiled_sql")), + ] + if isinstance(result_payload, dict): + top_level_candidates.extend( + [ + ("result.query", result_payload.get("query")), + ("result.sql", result_payload.get("sql")), + ("result.compiled_sql", result_payload.get("compiled_sql")), + ] + ) + + for source, candidate in top_level_candidates: + compiled_sql = str(candidate or "").strip() + response_diagnostics.append( + {"source": source, "has_query": bool(compiled_sql)} + ) + if compiled_sql: + return { + "compiled_sql": compiled_sql, + "raw_response": response, + "response_diagnostics": response_diagnostics, + } + + raise SupersetAPIError( + "Superset preview response did not expose compiled SQL " + f"(diagnostics={response_diagnostics!r})" + ) + + # [/DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function] + + +# [/DEF:SupersetDatasetsPreviewFiltersMixin:Class] +# [/DEF:SupersetDatasetsPreviewFiltersMixin:Module] diff --git a/backend/src/core/superset_client/_user_projection.py b/backend/src/core/superset_client/_user_projection.py index 5a6e693c..19b4d02d 100644 --- a/backend/src/core/superset_client/_user_projection.py +++ b/backend/src/core/superset_client/_user_projection.py @@ -1,5 +1,7 @@ # [DEF:SupersetUserProjection:Module] # @COMPLEXITY: 2 +# @LAYER: Infra +# @SEMANTICS: superset, users, owners, projection, normalization # @PURPOSE: User/owner payload normalization helpers for Superset client responses. # @RELATION: DEPENDS_ON -> [SupersetClientBase] diff --git a/backend/src/core/utils/superset_context_extractor.py b/backend/src/core/utils/superset_context_extractor.py deleted file mode 100644 index c3fb4922..00000000 --- a/backend/src/core/utils/superset_context_extractor.py +++ /dev/null @@ -1,1397 +0,0 @@ -# [DEF:SupersetContextExtractor:Module] -# @COMPLEXITY: 4 -# @SEMANTICS: dataset_review, superset, link_parsing, context_recovery, partial_recovery -# @PURPOSE: Recover dataset and dashboard context from Superset links while preserving explicit partial-recovery markers. -# @LAYER: Infra -# @RELATION: [DEPENDS_ON] ->[ImportedFilter] -# @RELATION: [DEPENDS_ON] ->[ImportedFilter] -# @RELATION: [DEPENDS_ON] ->[TemplateVariable] -# @PRE: Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource. -# @POST: Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary. -# @SIDE_EFFECT: Performs upstream Superset API reads. -# @INVARIANT: Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context. - -from __future__ import annotations - -# [DEF:SupersetContextExtractor.imports:Block] -import json -import re -from copy import deepcopy -from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Set, cast -from urllib.parse import parse_qs, unquote, urlparse - -from src.core.config_models import Environment -from src.core.logger import belief_scope, logger -from src.core.superset_client import SupersetClient -# [/DEF:SupersetContextExtractor.imports:Block] - -logger = cast(Any, logger) - -_EMAIL_PATTERN = re.compile( - r"(?P[A-Za-z0-9._%+-]+)@(?P[A-Za-z0-9.-]+\.[A-Za-z]{2,})" -) -_UUID_PATTERN = re.compile( - r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b" -) -_LONG_DIGIT_PATTERN = re.compile(r"\b\d{6,}\b") -_MIXED_IDENTIFIER_PATTERN = re.compile( - r"\b(?=[A-Za-z0-9_-]{10,}\b)(?=.*[A-Za-z])(?=.*\d)[A-Za-z0-9_-]+\b" -) - - -# [DEF:mask_pii_value:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Redact likely sensitive identifiers while preserving structural hints for assistant-facing dataset-review context. -def mask_pii_value(value: Any) -> tuple[Any, bool]: - if isinstance(value, str): - return _mask_sensitive_text(value) - if isinstance(value, list): - masked_any = False - masked_list: List[Any] = [] - for item in value: - masked_item, item_masked = mask_pii_value(item) - masked_list.append(masked_item) - masked_any = masked_any or item_masked - return masked_list, masked_any - if isinstance(value, dict): - masked_any = False - masked_dict: Dict[str, Any] = {} - for key, item in value.items(): - masked_item, item_masked = mask_pii_value(item) - masked_dict[key] = masked_item - masked_any = masked_any or item_masked - return masked_dict, masked_any - return value, False - - -# [/DEF:mask_pii_value:Function] - - -# [DEF:sanitize_imported_filter_for_assistant:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Build assistant-safe imported-filter payload with raw-value redaction metadata when sensitive tokens are detected. -def sanitize_imported_filter_for_assistant( - filter_payload: Dict[str, Any], -) -> Dict[str, Any]: - sanitized = deepcopy(filter_payload) - masked_raw_value, was_masked = mask_pii_value(filter_payload.get("raw_value")) - sanitized["raw_value"] = masked_raw_value - sanitized["raw_value_masked"] = bool( - filter_payload.get("raw_value_masked", False) or was_masked - ) - return sanitized - - -# [/DEF:sanitize_imported_filter_for_assistant:Function] - - -# [DEF:_mask_sensitive_text:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Redact likely PII substrings from one string while preserving non-sensitive delimiters and token shape. -def _mask_sensitive_text(value: str) -> tuple[str, bool]: - masked = value - masked_any = False - - def _replace_email(match: re.Match[str]) -> str: - nonlocal masked_any - masked_any = True - domain = match.group("domain") - return f"***@{domain}" - - def _replace_uuid(match: re.Match[str]) -> str: - nonlocal masked_any - masked_any = True - token = match.group(0) - return "********-****-****-****-" + token[-12:] - - def _replace_long_digits(match: re.Match[str]) -> str: - nonlocal masked_any - masked_any = True - token = match.group(0) - return "*" * max(len(token) - 2, 1) + token[-2:] - - def _replace_mixed_identifier(match: re.Match[str]) -> str: - nonlocal masked_any - token = match.group(0) - if token.isupper() and len(token) <= 5: - return token - masked_any = True - return token[:2] + "***" + token[-2:] - - masked = _EMAIL_PATTERN.sub(_replace_email, masked) - masked = _UUID_PATTERN.sub(_replace_uuid, masked) - masked = _LONG_DIGIT_PATTERN.sub(_replace_long_digits, masked) - masked = _MIXED_IDENTIFIER_PATTERN.sub(_replace_mixed_identifier, masked) - return masked, masked_any - - -# [/DEF:_mask_sensitive_text:Function] - - -# [DEF:SupersetParsedContext:Class] -# @COMPLEXITY: 2 -# @PURPOSE: Normalized output of Superset link parsing for session intake and recovery. -@dataclass -class SupersetParsedContext: - source_url: str - dataset_ref: str - dataset_id: Optional[int] = None - dashboard_id: Optional[int] = None - chart_id: Optional[int] = None - resource_type: str = "unknown" - query_state: Dict[str, Any] = field(default_factory=dict) - imported_filters: List[Dict[str, Any]] = field(default_factory=list) - unresolved_references: List[str] = field(default_factory=list) - partial_recovery: bool = False - dataset_payload: Optional[Dict[str, Any]] = None - - -# [/DEF:SupersetParsedContext:Class] - - -# [DEF:SupersetContextExtractor:Class] -# @COMPLEXITY: 4 -# @PURPOSE: Parse supported Superset URLs and recover canonical dataset/dashboard references for review-session intake. -# @RELATION: [DEPENDS_ON] ->[Environment] -# @PRE: constructor receives a configured environment with a usable Superset base URL. -# @POST: extractor instance is ready to parse links against one Superset environment. -# @SIDE_EFFECT: downstream parse operations may call Superset APIs through SupersetClient. -class SupersetContextExtractor: - # [DEF:SupersetContextExtractor.__init__:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Bind extractor to one Superset environment and client instance. - def __init__( - self, environment: Environment, client: Optional[SupersetClient] = None - ) -> None: - self.environment = environment - self.client = client or SupersetClient(environment) - - # [/DEF:SupersetContextExtractor.__init__:Function] - - # [DEF:SupersetContextExtractor.parse_superset_link:Function] - # @COMPLEXITY: 4 - # @PURPOSE: Extract candidate identifiers and query state from supported Superset URLs. - # @RELATION: [CALLS] ->[SupersetClient.get_dashboard_detail] - # @PRE: link is a non-empty Superset URL compatible with the configured environment. - # @POST: returns resolved dataset/dashboard context, preserving explicit partial-recovery state if some identifiers cannot be confirmed. - # @SIDE_EFFECT: may issue Superset API reads to resolve dataset references from dashboard or chart URLs. - # @DATA_CONTRACT: Input[link:str] -> Output[SupersetParsedContext] - def parse_superset_link(self, link: str) -> SupersetParsedContext: - with belief_scope("SupersetContextExtractor.parse_superset_link"): - normalized_link = str(link or "").strip() - if not normalized_link: - logger.explore("Rejected empty Superset link during intake") - raise ValueError("Superset link must be non-empty") - - parsed_url = urlparse(normalized_link) - if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc: - logger.explore( - "Superset link is not a parseable absolute URL", - extra={"link": normalized_link}, - ) - raise ValueError("Superset link must be an absolute http(s) URL") - - logger.reason( - "Parsing Superset link for dataset review intake", - extra={"path": parsed_url.path, "query": parsed_url.query}, - ) - - path_parts = [part for part in parsed_url.path.split("/") if part] - query_params = parse_qs(parsed_url.query, keep_blank_values=True) - query_state = self._decode_query_state(query_params) - - dataset_id = self._extract_numeric_identifier(path_parts, "dataset") - dashboard_id = self._extract_numeric_identifier(path_parts, "dashboard") - dashboard_ref = self._extract_dashboard_reference(path_parts) - dashboard_permalink_key = self._extract_dashboard_permalink_key(path_parts) - chart_id = self._extract_numeric_identifier(path_parts, "chart") - - resource_type = "unknown" - dataset_ref: Optional[str] = None - partial_recovery = False - unresolved_references: List[str] = [] - - if dataset_id is not None: - resource_type = "dataset" - dataset_ref = f"dataset:{dataset_id}" - logger.reason( - "Resolved direct dataset link", - extra={"dataset_id": dataset_id}, - ) - elif dashboard_permalink_key is not None: - resource_type = "dashboard" - partial_recovery = True - dataset_ref = f"dashboard_permalink:{dashboard_permalink_key}" - unresolved_references.append( - "dashboard_permalink_dataset_binding_unresolved" - ) - logger.reason( - "Resolving dashboard permalink state from Superset", - extra={"permalink_key": dashboard_permalink_key}, - ) - permalink_payload = self.client.get_dashboard_permalink_state( - dashboard_permalink_key - ) - permalink_state = ( - permalink_payload.get("state", permalink_payload) - if isinstance(permalink_payload, dict) - else {} - ) - if isinstance(permalink_state, dict): - for key, value in permalink_state.items(): - query_state.setdefault(key, value) - # Extract filters from permalink dataMask - data_mask = permalink_state.get("dataMask") - if isinstance(data_mask, dict) and data_mask: - query_state["dataMask"] = data_mask - logger.reason( - "Extracted native filters from permalink dataMask", - extra={"filter_count": len(data_mask)}, - ) - resolved_dashboard_id = self._extract_dashboard_id_from_state( - permalink_state - ) - resolved_chart_id = self._extract_chart_id_from_state( - permalink_state - ) - if resolved_dashboard_id is not None: - dashboard_id = resolved_dashboard_id - unresolved_references = [ - item - for item in unresolved_references - if item != "dashboard_permalink_dataset_binding_unresolved" - ] - dataset_id, unresolved_references = ( - self._recover_dataset_binding_from_dashboard( - dashboard_id=dashboard_id, - dataset_ref=dataset_ref, - unresolved_references=unresolved_references, - ) - ) - if dataset_id is not None: - dataset_ref = f"dataset:{dataset_id}" - elif resolved_chart_id is not None: - chart_id = resolved_chart_id - unresolved_references = [ - item - for item in unresolved_references - if item != "dashboard_permalink_dataset_binding_unresolved" - ] - try: - chart_payload = self.client.get_chart(chart_id) - chart_data = ( - chart_payload.get("result", chart_payload) - if isinstance(chart_payload, dict) - else {} - ) - datasource_id = chart_data.get("datasource_id") - if datasource_id is not None: - dataset_id = int(datasource_id) - dataset_ref = f"dataset:{dataset_id}" - logger.reason( - "Recovered dataset reference from permalink chart context", - extra={ - "chart_id": chart_id, - "dataset_id": dataset_id, - }, - ) - else: - unresolved_references.append( - "chart_dataset_binding_unresolved" - ) - except Exception as exc: - unresolved_references.append( - "chart_dataset_binding_unresolved" - ) - logger.explore( - "Chart lookup failed during permalink recovery", - extra={"chart_id": chart_id, "error": str(exc)}, - ) - else: - logger.explore( - "Dashboard permalink state was not a structured object", - extra={"permalink_key": dashboard_permalink_key}, - ) - elif dashboard_id is not None or dashboard_ref is not None: - resource_type = "dashboard" - resolved_dashboard_ref = ( - dashboard_id if dashboard_id is not None else dashboard_ref - ) - if resolved_dashboard_ref is None: - raise ValueError("Dashboard reference could not be resolved") - logger.reason( - "Resolving dashboard-bound dataset from Superset", - extra={"dashboard_ref": resolved_dashboard_ref}, - ) - - # Resolve dashboard detail first — handles both numeric ID and slug, - # ensuring dashboard_id is available for the native_filters_key fetch below. - dashboard_detail = self.client.get_dashboard_detail( - resolved_dashboard_ref - ) - resolved_dashboard_id = dashboard_detail.get("id") - if resolved_dashboard_id is not None: - dashboard_id = int(resolved_dashboard_id) - - # Check for native_filters_key in query params and fetch filter state. - # This must run AFTER dashboard_id is resolved from slug above. - native_filters_key = query_params.get("native_filters_key", [None])[0] - if native_filters_key and dashboard_id is not None: - try: - logger.reason( - "Fetching native filter state from Superset", - extra={ - "dashboard_id": dashboard_id, - "filter_key": native_filters_key, - }, - ) - extracted = self.client.extract_native_filters_from_key( - dashboard_id, native_filters_key - ) - data_mask = extracted.get("dataMask") - if isinstance(data_mask, dict) and data_mask: - query_state["native_filter_state"] = data_mask - logger.reason( - "Extracted native filter state from Superset via native_filters_key", - extra={"filter_count": len(data_mask)}, - ) - else: - logger.explore( - "Native filter state returned empty dataMask", - extra={ - "dashboard_id": dashboard_id, - "filter_key": native_filters_key, - }, - ) - except Exception as exc: - logger.explore( - "Failed to fetch native filter state from Superset", - extra={ - "dashboard_id": dashboard_id, - "filter_key": native_filters_key, - "error": str(exc), - }, - ) - - datasets = dashboard_detail.get("datasets") or [] - if datasets: - first_dataset = datasets[0] - resolved_dataset_id = first_dataset.get("id") - if resolved_dataset_id is not None: - dataset_id = int(resolved_dataset_id) - dataset_ref = f"dataset:{dataset_id}" - logger.reason( - "Recovered dataset reference from dashboard context", - extra={ - "dashboard_id": dashboard_id, - "dataset_id": dataset_id, - "dataset_count": len(datasets), - }, - ) - if len(datasets) > 1: - partial_recovery = True - unresolved_references.append("multiple_dashboard_datasets") - else: - partial_recovery = True - unresolved_references.append("dashboard_dataset_id_missing") - else: - partial_recovery = True - unresolved_references.append("dashboard_dataset_binding_missing") - elif chart_id is not None: - resource_type = "chart" - partial_recovery = True - unresolved_references.append("chart_dataset_binding_unresolved") - dataset_ref = f"chart:{chart_id}" - logger.reason( - "Accepted chart link with explicit partial recovery", - extra={"chart_id": chart_id}, - ) - else: - logger.explore( - "Unsupported Superset link shape encountered", - extra={"path": parsed_url.path}, - ) - raise ValueError("Unsupported Superset link shape") - - dataset_payload: Optional[Dict[str, Any]] = None - if dataset_id is not None: - try: - dataset_payload = self.client.get_dataset_detail(dataset_id) - table_name = str(dataset_payload.get("table_name") or "").strip() - schema_name = str(dataset_payload.get("schema") or "").strip() - if table_name: - dataset_ref = ( - f"{schema_name}.{table_name}" if schema_name else table_name - ) - logger.reason( - "Canonicalized dataset reference from dataset detail", - extra={ - "dataset_ref": dataset_ref, - "dataset_id": dataset_id, - }, - ) - except Exception as exc: - partial_recovery = True - unresolved_references.append("dataset_detail_lookup_failed") - logger.explore( - "Dataset detail lookup failed during link parsing; keeping session usable", - extra={"dataset_id": dataset_id, "error": str(exc)}, - ) - - imported_filters = self._extract_imported_filters(query_state) - result = SupersetParsedContext( - source_url=normalized_link, - dataset_ref=dataset_ref or "unresolved", - dataset_id=dataset_id, - dashboard_id=dashboard_id, - chart_id=chart_id, - resource_type=resource_type, - query_state=query_state, - imported_filters=imported_filters, - unresolved_references=unresolved_references, - partial_recovery=partial_recovery, - dataset_payload=dataset_payload, - ) - logger.reflect( - "Superset link parsing completed", - extra={ - "dataset_ref": result.dataset_ref, - "dataset_id": result.dataset_id, - "dashboard_id": result.dashboard_id, - "chart_id": result.chart_id, - "partial_recovery": result.partial_recovery, - "unresolved_references": result.unresolved_references, - "imported_filters": len(result.imported_filters), - }, - ) - return result - - # [/DEF:SupersetContextExtractor.parse_superset_link:Function] - - # [DEF:SupersetContextExtractor.recover_imported_filters:Function] - # @COMPLEXITY: 4 - # @PURPOSE: Build imported filter entries from URL state and Superset-side saved context. - # @RELATION: [CALLS] ->[SupersetClient.get_dashboard] - # @PRE: parsed_context comes from a successful Superset link parse for one environment. - # @POST: returns explicit recovered and partial filter entries with preserved provenance and confirmation requirements. - # @SIDE_EFFECT: may issue Superset reads for dashboard metadata enrichment. - # @DATA_CONTRACT: Input[SupersetParsedContext] -> Output[List[Dict[str,Any]]] - def recover_imported_filters( - self, parsed_context: SupersetParsedContext - ) -> List[Dict[str, Any]]: - with belief_scope("SupersetContextExtractor.recover_imported_filters"): - recovered_filters: List[Dict[str, Any]] = [] - seen_filter_keys: Set[str] = set() - metadata_filters: List[Dict[str, Any]] = [] - metadata_filters_by_id: Dict[str, Dict[str, Any]] = {} - - def merge_recovered_filter(candidate: Dict[str, Any]) -> None: - filter_key = candidate["filter_name"].strip().lower() - existing_index = next( - ( - index - for index, existing in enumerate(recovered_filters) - if existing["filter_name"].strip().lower() == filter_key - ), - None, - ) - if existing_index is None: - seen_filter_keys.add(filter_key) - recovered_filters.append(candidate) - return - - existing = recovered_filters[existing_index] - if existing.get("display_name") in { - None, - "", - existing.get("filter_name"), - } and candidate.get("display_name"): - existing["display_name"] = candidate["display_name"] - if ( - existing.get("raw_value") is None - and candidate.get("raw_value") is not None - ): - existing["raw_value"] = candidate["raw_value"] - existing["confidence_state"] = candidate.get( - "confidence_state", "imported" - ) - existing["requires_confirmation"] = candidate.get( - "requires_confirmation", False - ) - existing["recovery_status"] = candidate.get( - "recovery_status", "recovered" - ) - existing["source"] = candidate.get("source", existing.get("source")) - if ( - existing.get("normalized_value") is None - and candidate.get("normalized_value") is not None - ): - existing["normalized_value"] = deepcopy( - candidate["normalized_value"] - ) - if ( - existing.get("notes") - and candidate.get("notes") - and candidate["notes"] not in existing["notes"] - ): - existing["notes"] = f"{existing['notes']}; {candidate['notes']}" - - if parsed_context.dashboard_id is not None: - try: - dashboard_payload = self.client.get_dashboard( - parsed_context.dashboard_id - ) - dashboard_record = ( - dashboard_payload.get("result", dashboard_payload) - if isinstance(dashboard_payload, dict) - else {} - ) - json_metadata = dashboard_record.get("json_metadata") - if isinstance(json_metadata, str) and json_metadata.strip(): - json_metadata = json.loads(json_metadata) - if not isinstance(json_metadata, dict): - json_metadata = {} - - native_filter_configuration = ( - json_metadata.get("native_filter_configuration") or [] - ) - default_filters = json_metadata.get("default_filters") or {} - if isinstance(default_filters, str) and default_filters.strip(): - try: - default_filters = json.loads(default_filters) - except Exception: - logger.explore( - "Superset default_filters payload was not valid JSON", - extra={"dashboard_id": parsed_context.dashboard_id}, - ) - default_filters = {} - - for item in native_filter_configuration: - if not isinstance(item, dict): - continue - filter_name = str( - item.get("name") - or item.get("filter_name") - or item.get("column") - or "" - ).strip() - if not filter_name: - continue - - display_name = ( - item.get("label") or item.get("name") or filter_name - ) - filter_id = str(item.get("id") or "").strip() - - default_value = None - if isinstance(default_filters, dict): - default_value = default_filters.get(filter_name) - - metadata_filter = self._normalize_imported_filter_payload( - { - "filter_name": filter_name, - "display_name": display_name, - "raw_value": default_value, - "source": "superset_native", - "recovery_status": "recovered" - if default_value is not None - else "partial", - "requires_confirmation": default_value is None, - "notes": "Recovered from Superset dashboard native filter configuration", - }, - default_source="superset_native", - default_note="Recovered from Superset dashboard native filter configuration", - ) - metadata_filters.append(metadata_filter) - - if filter_id: - metadata_filters_by_id[filter_id.lower()] = { - "filter_name": filter_name, - "display_name": display_name, - } - except Exception as exc: - logger.explore( - "Dashboard native filter enrichment failed; preserving partial imported filters", - extra={ - "dashboard_id": parsed_context.dashboard_id, - "error": str(exc), - "filter_count": len(recovered_filters), - }, - ) - metadata_filters = [] - metadata_filters_by_id = {} - - for item in parsed_context.imported_filters: - normalized = self._normalize_imported_filter_payload( - item, - default_source="superset_url", - default_note="Recovered from Superset URL state", - ) - metadata_match = metadata_filters_by_id.get( - normalized["filter_name"].strip().lower() - ) - if metadata_match is not None: - normalized["filter_name"] = metadata_match["filter_name"] - normalized["display_name"] = metadata_match["display_name"] - normalized["notes"] = ( - "Recovered from Superset URL state and reconciled against dashboard native filter metadata" - ) - - merge_recovered_filter(normalized) - logger.reflect( - "Recovered filter from URL state", - extra={ - "filter_name": normalized["filter_name"], - "source": normalized["source"], - "has_value": normalized["raw_value"] is not None, - "canonicalized": metadata_match is not None, - }, - ) - - if parsed_context.dashboard_id is None: - logger.reflect( - "Imported filter recovery completed without dashboard enrichment", - extra={ - "dashboard_id": None, - "filter_count": len(recovered_filters), - "partial_recovery": parsed_context.partial_recovery, - }, - ) - return recovered_filters - - for saved_filter in metadata_filters: - merge_recovered_filter(saved_filter) - logger.reflect( - "Recovered filter from dashboard metadata", - extra={ - "filter_name": saved_filter["filter_name"], - "has_value": saved_filter["raw_value"] is not None, - }, - ) - - if not recovered_filters: - recovered_filters.append( - self._normalize_imported_filter_payload( - { - "filter_name": f"dashboard_{parsed_context.dashboard_id}_filters", - "display_name": "Dashboard native filters", - "raw_value": None, - "source": "superset_native", - "recovery_status": "partial", - "requires_confirmation": True, - "notes": "Superset dashboard filter configuration could not be recovered fully", - }, - default_source="superset_native", - default_note="Superset dashboard filter configuration could not be recovered fully", - ) - ) - - logger.reflect( - "Imported filter recovery completed with dashboard enrichment", - extra={ - "dashboard_id": parsed_context.dashboard_id, - "filter_count": len(recovered_filters), - "partial_entries": len( - [ - item - for item in recovered_filters - if item["recovery_status"] == "partial" - ] - ), - }, - ) - return recovered_filters - - # [/DEF:SupersetContextExtractor.recover_imported_filters:Function] - - # [DEF:SupersetContextExtractor.discover_template_variables:Function] - # @COMPLEXITY: 4 - # @PURPOSE: Detect runtime variables and Jinja references from dataset query-bearing fields. - # @RELATION: [DEPENDS_ON] ->[TemplateVariable] - # @PRE: dataset_payload is a Superset dataset-detail style payload with query-bearing fields when available. - # @POST: returns deduplicated explicit variable records without executing Jinja or fabricating runtime values. - # @SIDE_EFFECT: none. - # @DATA_CONTRACT: Input[dataset_payload:Dict[str,Any]] -> Output[List[Dict[str,Any]]] - def discover_template_variables( - self, dataset_payload: Dict[str, Any] - ) -> List[Dict[str, Any]]: - with belief_scope("SupersetContextExtractor.discover_template_variables"): - discovered: List[Dict[str, Any]] = [] - seen_variable_names: Set[str] = set() - - for expression_source in self._collect_query_bearing_expressions( - dataset_payload - ): - for filter_match in re.finditer( - r"filter_values\(\s*['\"]([^'\"]+)['\"]\s*\)", - expression_source, - flags=re.IGNORECASE, - ): - variable_name = str(filter_match.group(1) or "").strip() - if not variable_name: - continue - self._append_template_variable( - discovered=discovered, - seen_variable_names=seen_variable_names, - variable_name=variable_name, - expression_source=expression_source, - variable_kind="native_filter", - is_required=True, - default_value=None, - ) - - for url_param_match in re.finditer( - r"url_param\(\s*['\"]([^'\"]+)['\"]\s*(?:,\s*([^)]+))?\)", - expression_source, - flags=re.IGNORECASE, - ): - variable_name = str(url_param_match.group(1) or "").strip() - if not variable_name: - continue - default_literal = url_param_match.group(2) - self._append_template_variable( - discovered=discovered, - seen_variable_names=seen_variable_names, - variable_name=variable_name, - expression_source=expression_source, - variable_kind="parameter", - is_required=default_literal is None, - default_value=self._normalize_default_literal(default_literal), - ) - - for jinja_match in re.finditer( - r"\{\{\s*(.*?)\s*\}\}", expression_source, flags=re.DOTALL - ): - expression = str(jinja_match.group(1) or "").strip() - if not expression: - continue - if any( - token in expression - for token in ("filter_values(", "url_param(", "get_filters(") - ): - continue - variable_name = self._extract_primary_jinja_identifier(expression) - if not variable_name: - continue - self._append_template_variable( - discovered=discovered, - seen_variable_names=seen_variable_names, - variable_name=variable_name, - expression_source=expression_source, - variable_kind="derived" - if "." in expression or "|" in expression - else "parameter", - is_required=True, - default_value=None, - ) - - logger.reflect( - "Template variable discovery completed deterministically", - extra={ - "dataset_id": dataset_payload.get("id"), - "variable_count": len(discovered), - "variable_names": [item["variable_name"] for item in discovered], - }, - ) - return discovered - - # [/DEF:SupersetContextExtractor.discover_template_variables:Function] - - # [DEF:SupersetContextExtractor.build_recovery_summary:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Summarize recovered, partial, and unresolved context for session state and UX. - def build_recovery_summary( - self, parsed_context: SupersetParsedContext - ) -> Dict[str, Any]: - return { - "dataset_ref": parsed_context.dataset_ref, - "dataset_id": parsed_context.dataset_id, - "dashboard_id": parsed_context.dashboard_id, - "chart_id": parsed_context.chart_id, - "partial_recovery": parsed_context.partial_recovery, - "unresolved_references": list(parsed_context.unresolved_references), - "imported_filter_count": len(parsed_context.imported_filters), - } - - # [/DEF:SupersetContextExtractor.build_recovery_summary:Function] - - # [DEF:SupersetContextExtractor._extract_numeric_identifier:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Extract a numeric identifier from a REST-like Superset URL path. - def _extract_numeric_identifier( - self, path_parts: List[str], resource_name: str - ) -> Optional[int]: - if resource_name not in path_parts: - return None - try: - resource_index = path_parts.index(resource_name) - except ValueError: - return None - - if resource_index + 1 >= len(path_parts): - return None - - candidate = str(path_parts[resource_index + 1]).strip() - if not candidate.isdigit(): - return None - return int(candidate) - - # [/DEF:SupersetContextExtractor._extract_numeric_identifier:Function] - - # [DEF:SupersetContextExtractor._extract_dashboard_reference:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Extract a dashboard id-or-slug reference from a Superset URL path. - def _extract_dashboard_reference(self, path_parts: List[str]) -> Optional[str]: - if "dashboard" not in path_parts: - return None - try: - resource_index = path_parts.index("dashboard") - except ValueError: - return None - - if resource_index + 1 >= len(path_parts): - return None - - candidate = str(path_parts[resource_index + 1]).strip() - if not candidate or candidate == "p": - return None - return candidate - - # [/DEF:SupersetContextExtractor._extract_dashboard_reference:Function] - - # [DEF:SupersetContextExtractor._extract_dashboard_permalink_key:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Extract a dashboard permalink key from a Superset URL path. - def _extract_dashboard_permalink_key(self, path_parts: List[str]) -> Optional[str]: - if "dashboard" not in path_parts: - return None - try: - resource_index = path_parts.index("dashboard") - except ValueError: - return None - - if resource_index + 2 >= len(path_parts): - return None - - permalink_marker = str(path_parts[resource_index + 1]).strip() - permalink_key = str(path_parts[resource_index + 2]).strip() - if permalink_marker != "p" or not permalink_key: - return None - return permalink_key - - # [/DEF:SupersetContextExtractor._extract_dashboard_permalink_key:Function] - - # [DEF:SupersetContextExtractor._extract_dashboard_id_from_state:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Extract a dashboard identifier from returned permalink state when present. - def _extract_dashboard_id_from_state(self, state: Dict[str, Any]) -> Optional[int]: - return self._search_nested_numeric_key( - payload=state, - candidate_keys={"dashboardId", "dashboard_id", "dashboard_id_value"}, - ) - - # [/DEF:SupersetContextExtractor._extract_dashboard_id_from_state:Function] - - # [DEF:SupersetContextExtractor._extract_chart_id_from_state:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Extract a chart identifier from returned permalink state when dashboard id is absent. - def _extract_chart_id_from_state(self, state: Dict[str, Any]) -> Optional[int]: - return self._search_nested_numeric_key( - payload=state, - candidate_keys={"slice_id", "sliceId", "chartId", "chart_id"}, - ) - - # [/DEF:SupersetContextExtractor._extract_chart_id_from_state:Function] - - # [DEF:SupersetContextExtractor._search_nested_numeric_key:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Recursively search nested dict/list payloads for the first numeric value under a candidate key set. - # @RELATION: [DEPENDS_ON] ->[SupersetContextExtractor.parse_superset_link] - def _search_nested_numeric_key( - self, payload: Any, candidate_keys: Set[str] - ) -> Optional[int]: - if isinstance(payload, dict): - for key, value in payload.items(): - if key in candidate_keys: - try: - if value is not None: - return int(value) - except (TypeError, ValueError): - pass - found = self._search_nested_numeric_key(value, candidate_keys) - if found is not None: - return found - elif isinstance(payload, list): - for item in payload: - found = self._search_nested_numeric_key(item, candidate_keys) - if found is not None: - return found - return None - - # [/DEF:SupersetContextExtractor._search_nested_numeric_key:Function] - - # [DEF:SupersetContextExtractor._recover_dataset_binding_from_dashboard:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Recover a dataset binding from resolved dashboard context while preserving explicit unresolved markers. - # @RELATION: [CALLS] ->[SupersetClient.get_dashboard_detail] - def _recover_dataset_binding_from_dashboard( - self, - dashboard_id: int, - dataset_ref: Optional[str], - unresolved_references: List[str], - ) -> tuple[Optional[int], List[str]]: - dashboard_detail = self.client.get_dashboard_detail(dashboard_id) - datasets = dashboard_detail.get("datasets") or [] - if datasets: - first_dataset = datasets[0] - resolved_dataset_id = first_dataset.get("id") - if resolved_dataset_id is not None: - resolved_dataset = int(resolved_dataset_id) - logger.reason( - "Recovered dataset reference from dashboard permalink context", - extra={ - "dashboard_id": dashboard_id, - "dataset_id": resolved_dataset, - "dataset_count": len(datasets), - "dataset_ref": dataset_ref, - }, - ) - if ( - len(datasets) > 1 - and "multiple_dashboard_datasets" not in unresolved_references - ): - unresolved_references.append("multiple_dashboard_datasets") - return resolved_dataset, unresolved_references - if "dashboard_dataset_id_missing" not in unresolved_references: - unresolved_references.append("dashboard_dataset_id_missing") - return None, unresolved_references - - if "dashboard_dataset_binding_missing" not in unresolved_references: - unresolved_references.append("dashboard_dataset_binding_missing") - return None, unresolved_references - - # [/DEF:SupersetContextExtractor._recover_dataset_binding_from_dashboard:Function] - - # [DEF:SupersetContextExtractor._decode_query_state:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Decode query-string structures used by Superset URL state transport. - def _decode_query_state(self, query_params: Dict[str, List[str]]) -> Dict[str, Any]: - query_state: Dict[str, Any] = {} - for key, values in query_params.items(): - if not values: - continue - raw_value = values[-1] - decoded_value = unquote(raw_value) - if key in {"native_filters", "form_data", "q"}: - try: - query_state[key] = json.loads(decoded_value) - continue - except Exception: - logger.explore( - "Failed to decode structured Superset query state; preserving raw value", - extra={"key": key}, - ) - query_state[key] = decoded_value - return query_state - - # [/DEF:SupersetContextExtractor._decode_query_state:Function] - - # [DEF:SupersetContextExtractor._extract_imported_filters:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Normalize imported filters from decoded query state without fabricating missing values. - def _extract_imported_filters( - self, query_state: Dict[str, Any] - ) -> List[Dict[str, Any]]: - imported_filters: List[Dict[str, Any]] = [] - - native_filters_payload = query_state.get("native_filters") - if isinstance(native_filters_payload, list): - for index, item in enumerate(native_filters_payload): - if not isinstance(item, dict): - continue - filter_name = ( - item.get("filter_name") - or item.get("column") - or item.get("name") - or f"native_filter_{index}" - ) - direct_clause = None - if item.get("column") and ("value" in item or "val" in item): - direct_clause = { - "col": item.get("column"), - "op": item.get("op") - or ("IN" if isinstance(item.get("value"), list) else "=="), - "val": item.get("val", item.get("value")), - } - imported_filters.append( - { - "filter_name": str(filter_name), - "raw_value": item.get("value"), - "display_name": item.get("label") or item.get("name"), - "normalized_value": { - "filter_clauses": [direct_clause] - if isinstance(direct_clause, dict) - else [], - "extra_form_data": {}, - "value_origin": "native_filters", - }, - "source": "superset_url", - "recovery_status": "recovered" - if item.get("value") is not None - else "partial", - "requires_confirmation": item.get("value") is None, - "notes": "Recovered from Superset native filter URL state", - } - ) - - # Extract filters from permalink dataMask - dashboard_data_mask = query_state.get("dataMask") - if isinstance(dashboard_data_mask, dict): - for filter_key, item in dashboard_data_mask.items(): - if not isinstance(item, dict): - continue - filter_state = item.get("filterState") - extra_form_data = item.get("extraFormData") - display_name = None - raw_value = None - normalized_value = { - "filter_clauses": [], - "extra_form_data": deepcopy(extra_form_data) - if isinstance(extra_form_data, dict) - else {}, - "value_origin": "unresolved", - } - - # Try to get value from filterState - if isinstance(filter_state, dict): - display_name = filter_state.get("label") - # Superset filterState uses 'value' for single values, 'values' for multi-select - raw_value = filter_state.get("value") or filter_state.get("values") - if raw_value is not None: - normalized_value["value_origin"] = "filter_state" - - # Preserve exact Superset clauses from extraFormData.filters - if isinstance(extra_form_data, dict): - extra_filters = extra_form_data.get("filters") - if isinstance(extra_filters, list): - normalized_value["filter_clauses"] = [ - deepcopy(extra_filter) - for extra_filter in extra_filters - if isinstance(extra_filter, dict) - ] - - # If no value found, try extraFormData.filters - if raw_value is None and normalized_value["filter_clauses"]: - first_filter = normalized_value["filter_clauses"][0] - raw_value = first_filter.get("val") - if raw_value is None: - raw_value = first_filter.get("value") - if raw_value is not None: - normalized_value["value_origin"] = "extra_form_data.filters" - - # If still no value, try extraFormData directly for time_range, time_grain, etc. - if raw_value is None and isinstance(extra_form_data, dict): - # Common Superset filter fields - for field in [ - "time_range", - "time_grain_sqla", - "time_column", - "granularity", - ]: - if field in extra_form_data: - raw_value = extra_form_data[field] - normalized_value["value_origin"] = ( - f"extra_form_data.{field}" - ) - break - - imported_filters.append( - { - "filter_name": str(item.get("id") or filter_key), - "raw_value": raw_value, - "display_name": display_name, - "normalized_value": normalized_value, - "source": "superset_permalink", - "recovery_status": "recovered" - if raw_value is not None - else "partial", - "requires_confirmation": raw_value is None, - "notes": "Recovered from Superset dashboard permalink state", - } - ) - - # Extract filters from native_filter_state (fetched from Superset via native_filters_key) - native_filter_state = query_state.get("native_filter_state") - if isinstance(native_filter_state, dict): - for filter_key, item in native_filter_state.items(): - if not isinstance(item, dict): - continue - # Handle both single filter format and multi-filter format - filter_id = item.get("id") or filter_key - filter_state = item.get("filterState") - extra_form_data = item.get("extraFormData") - display_name = None - raw_value = None - normalized_value = { - "filter_clauses": [], - "extra_form_data": deepcopy(extra_form_data) - if isinstance(extra_form_data, dict) - else {}, - "value_origin": "unresolved", - } - - # Try to get value from filterState - if isinstance(filter_state, dict): - display_name = filter_state.get("label") - # Superset filterState uses 'value' for single values, 'values' for multi-select - raw_value = filter_state.get("value") or filter_state.get("values") - if raw_value is not None: - normalized_value["value_origin"] = "filter_state" - - # Preserve exact Superset clauses from extraFormData.filters - if isinstance(extra_form_data, dict): - extra_filters = extra_form_data.get("filters") - if isinstance(extra_filters, list): - normalized_value["filter_clauses"] = [ - deepcopy(extra_filter) - for extra_filter in extra_filters - if isinstance(extra_filter, dict) - ] - - # If no value found, try extraFormData.filters - if raw_value is None and normalized_value["filter_clauses"]: - first_filter = normalized_value["filter_clauses"][0] - raw_value = first_filter.get("val") - if raw_value is None: - raw_value = first_filter.get("value") - if raw_value is not None: - normalized_value["value_origin"] = "extra_form_data.filters" - - # If still no value, try extraFormData directly for time_range, time_grain, etc. - if raw_value is None and isinstance(extra_form_data, dict): - # Common Superset filter fields - for field in [ - "time_range", - "time_grain_sqla", - "time_column", - "granularity", - ]: - if field in extra_form_data: - raw_value = extra_form_data[field] - normalized_value["value_origin"] = ( - f"extra_form_data.{field}" - ) - break - - imported_filters.append( - { - "filter_name": str(filter_id), - "raw_value": raw_value, - "display_name": display_name, - "normalized_value": normalized_value, - "source": "superset_native_filters_key", - "recovery_status": "recovered" - if raw_value is not None - else "partial", - "requires_confirmation": raw_value is None, - "notes": "Recovered from Superset native_filters_key state", - } - ) - - form_data_payload = query_state.get("form_data") - if isinstance(form_data_payload, dict): - extra_filters = form_data_payload.get("extra_filters") or [] - for index, item in enumerate(extra_filters): - if not isinstance(item, dict): - continue - filter_name = ( - item.get("col") or item.get("column") or f"extra_filter_{index}" - ) - imported_filters.append( - { - "filter_name": str(filter_name), - "raw_value": item.get("val"), - "display_name": item.get("label"), - "normalized_value": { - "filter_clauses": [deepcopy(item)], - "extra_form_data": {}, - "value_origin": "form_data.extra_filters", - }, - "source": "superset_url", - "recovery_status": "recovered" - if item.get("val") is not None - else "partial", - "requires_confirmation": item.get("val") is None, - "notes": "Recovered from Superset form_data extra_filters", - } - ) - - return imported_filters - - # [/DEF:SupersetContextExtractor._extract_imported_filters:Function] - - # [DEF:SupersetContextExtractor._normalize_imported_filter_payload:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Normalize one imported-filter payload with explicit provenance and confirmation state. - def _normalize_imported_filter_payload( - self, - payload: Dict[str, Any], - default_source: str, - default_note: str, - ) -> Dict[str, Any]: - raw_value = payload.get("raw_value") - if "raw_value" not in payload and "value" in payload: - raw_value = payload.get("value") - - recovery_status = ( - str( - payload.get("recovery_status") - or ("recovered" if raw_value is not None else "partial") - ) - .strip() - .lower() - ) - requires_confirmation = bool( - payload.get( - "requires_confirmation", - raw_value is None or recovery_status != "recovered", - ) - ) - return { - "filter_name": str( - payload.get("filter_name") or "unresolved_filter" - ).strip(), - "display_name": payload.get("display_name"), - "raw_value": raw_value, - "normalized_value": payload.get("normalized_value"), - "source": str(payload.get("source") or default_source), - "confidence_state": "imported" if raw_value is not None else "unresolved", - "requires_confirmation": requires_confirmation, - "recovery_status": recovery_status, - "notes": str(payload.get("notes") or default_note), - } - - # [/DEF:SupersetContextExtractor._normalize_imported_filter_payload:Function] - - # [DEF:SupersetContextExtractor._collect_query_bearing_expressions:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Collect SQL and expression-bearing dataset fields for deterministic template-variable discovery. - # @RELATION: [DEPENDS_ON] ->[SupersetContextExtractor.discover_template_variables] - def _collect_query_bearing_expressions( - self, dataset_payload: Dict[str, Any] - ) -> List[str]: - expressions: List[str] = [] - - def append_expression(candidate: Any) -> None: - if not isinstance(candidate, str): - return - normalized = candidate.strip() - if normalized: - expressions.append(normalized) - - append_expression(dataset_payload.get("sql")) - append_expression(dataset_payload.get("query")) - append_expression(dataset_payload.get("template_sql")) - - metrics_payload = dataset_payload.get("metrics") or [] - if isinstance(metrics_payload, list): - for metric in metrics_payload: - if isinstance(metric, str): - append_expression(metric) - continue - if not isinstance(metric, dict): - continue - append_expression(metric.get("expression")) - append_expression(metric.get("sqlExpression")) - append_expression(metric.get("metric_name")) - - columns_payload = dataset_payload.get("columns") or [] - if isinstance(columns_payload, list): - for column in columns_payload: - if not isinstance(column, dict): - continue - append_expression(column.get("sqlExpression")) - append_expression(column.get("expression")) - - return expressions - - # [/DEF:SupersetContextExtractor._collect_query_bearing_expressions:Function] - - # [DEF:SupersetContextExtractor._append_template_variable:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Append one deduplicated template-variable descriptor. - def _append_template_variable( - self, - discovered: List[Dict[str, Any]], - seen_variable_names: Set[str], - variable_name: str, - expression_source: str, - variable_kind: str, - is_required: bool, - default_value: Any, - ) -> None: - normalized_name = str(variable_name or "").strip() - if not normalized_name: - return - seen_key = normalized_name.lower() - if seen_key in seen_variable_names: - return - seen_variable_names.add(seen_key) - discovered.append( - { - "variable_name": normalized_name, - "expression_source": expression_source, - "variable_kind": variable_kind, - "is_required": is_required, - "default_value": default_value, - "mapping_status": "unmapped", - } - ) - - # [/DEF:SupersetContextExtractor._append_template_variable:Function] - - # [DEF:SupersetContextExtractor._extract_primary_jinja_identifier:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Extract a deterministic primary identifier from a Jinja expression without executing it. - def _extract_primary_jinja_identifier(self, expression: str) -> Optional[str]: - matched = re.match(r"([A-Za-z_][A-Za-z0-9_]*)", expression.strip()) - if matched is None: - return None - candidate = matched.group(1) - if candidate in {"if", "else", "for", "set", "True", "False", "none", "None"}: - return None - return candidate - - # [/DEF:SupersetContextExtractor._extract_primary_jinja_identifier:Function] - - # [DEF:SupersetContextExtractor._normalize_default_literal:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Normalize literal default fragments from template helper calls into JSON-safe values. - def _normalize_default_literal(self, literal: Optional[str]) -> Any: - normalized_literal = str(literal or "").strip() - if not normalized_literal: - return None - if ( - normalized_literal.startswith("'") and normalized_literal.endswith("'") - ) or (normalized_literal.startswith('"') and normalized_literal.endswith('"')): - return normalized_literal[1:-1] - lowered = normalized_literal.lower() - if lowered in {"true", "false"}: - return lowered == "true" - if lowered in {"null", "none"}: - return None - try: - return int(normalized_literal) - except ValueError: - try: - return float(normalized_literal) - except ValueError: - return normalized_literal - - # [/DEF:SupersetContextExtractor._normalize_default_literal:Function] - - -# [/DEF:SupersetContextExtractor:Class] - -# [/DEF:SupersetContextExtractor:Module] diff --git a/backend/src/core/utils/superset_context_extractor/__init__.py b/backend/src/core/utils/superset_context_extractor/__init__.py new file mode 100644 index 00000000..8a49a811 --- /dev/null +++ b/backend/src/core/utils/superset_context_extractor/__init__.py @@ -0,0 +1,66 @@ +# [DEF:SupersetContextExtractorPackage:Module] +# @COMPLEXITY: 4 +# @LAYER: Infra +# @SEMANTICS: dataset_review, superset, link_parsing, context_recovery, partial_recovery +# @PURPOSE: Recover dataset and dashboard context from Superset links while preserving explicit partial-recovery markers. +# @RELATION: DEPENDS_ON -> [ImportedFilter] +# @RELATION: DEPENDS_ON -> [TemplateVariable] +# @RELATION: DEPENDS_ON -> [SupersetClient] +# @PRE: Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource. +# @POST: Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary. +# @SIDE_EFFECT: Performs upstream Superset API reads. +# @INVARIANT: Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context. +# +# @RATIONALE: Decomposed from monolithic superset_context_extractor.py (1397 lines) into +# domain-scoped mixins to satisfy INV_7 (module < 400 lines). The composed class +# preserves the original public API surface — all consumers continue to import +# from `src.core.utils.superset_context_extractor` without changes. +# @REJECTED: Keeping a single 1397-line file — violates fractal limit INV_7. +# [/DEF:SupersetContextExtractorPackage:Module] + +from ._base import SupersetContextExtractorBase, SupersetParsedContext +from ._parsing import SupersetContextParsingMixin +from ._recovery import SupersetContextRecoveryMixin +from ._templates import SupersetContextTemplatesMixin +from ._filters import SupersetContextFiltersExtractMixin +from ._pii import mask_pii_value, sanitize_imported_filter_for_assistant, _mask_sensitive_text + + +# [DEF:SupersetContextExtractor:Class] +# @COMPLEXITY: 4 +# @PURPOSE: Parse supported Superset URLs and recover canonical dataset/dashboard references for review-session intake. +# @RELATION: DEPENDS_ON -> [Environment] +# @RELATION: INHERITS -> [SupersetContextExtractorBase] +# @RELATION: INHERITS -> [SupersetContextParsingMixin] +# @RELATION: INHERITS -> [SupersetContextRecoveryMixin] +# @RELATION: INHERITS -> [SupersetContextTemplatesMixin] +# @RELATION: INHERITS -> [SupersetContextFiltersExtractMixin] +# @PRE: constructor receives a configured environment with a usable Superset base URL. +# @POST: extractor instance is ready to parse links against one Superset environment. +# @SIDE_EFFECT: downstream parse operations may call Superset APIs through SupersetClient. +class SupersetContextExtractor( + SupersetContextFiltersExtractMixin, + SupersetContextRecoveryMixin, + SupersetContextTemplatesMixin, + SupersetContextParsingMixin, + SupersetContextExtractorBase, +): + """Composed Superset context extractor. + + MRO: FiltersExtractMixin -> RecoveryMixin -> TemplatesMixin -> ParsingMixin -> Base. + All consumers continue to use: + from src.core.utils.superset_context_extractor import SupersetContextExtractor + """ + pass + + +# [/DEF:SupersetContextExtractor:Class] + + +__all__ = [ + "SupersetContextExtractor", + "SupersetParsedContext", + "mask_pii_value", + "sanitize_imported_filter_for_assistant", + "_mask_sensitive_text", +] diff --git a/backend/src/core/utils/superset_context_extractor/_base.py b/backend/src/core/utils/superset_context_extractor/_base.py new file mode 100644 index 00000000..6664e009 --- /dev/null +++ b/backend/src/core/utils/superset_context_extractor/_base.py @@ -0,0 +1,231 @@ +# [DEF:SupersetContextExtractorBase:Module] +# @COMPLEXITY: 4 +# @LAYER: Infra +# @SEMANTICS: dataset_review, superset, link_parsing, context_recovery, partial_recovery +# @PURPOSE: Base class and helpers for recovering dataset/dashboard context from Superset links with explicit partial-recovery markers. +# @RELATION: DEPENDS_ON -> [ImportedFilter] +# @RELATION: DEPENDS_ON -> [TemplateVariable] +# @RELATION: CALLS -> [SupersetClient] +# @PRE: Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource. +# @POST: Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary. +# @SIDE_EFFECT: Performs upstream Superset API reads. +# @INVARIANT: Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context. +# [/DEF:SupersetContextExtractorBase:Module] + +# [DEF:_base_imports:Block] +from __future__ import annotations + +import json +import re +from copy import deepcopy +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Set, cast +from urllib.parse import parse_qs, unquote, urlparse + +from ...config_models import Environment +from ...logger import belief_scope, logger +from ...superset_client import SupersetClient +# [/DEF:_base_imports:Block] + +logger = cast(Any, logger) + + +# [DEF:SupersetParsedContext:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Normalized output of Superset link parsing for session intake and recovery. +@dataclass +class SupersetParsedContext: + source_url: str + dataset_ref: str + dataset_id: Optional[int] = None + dashboard_id: Optional[int] = None + chart_id: Optional[int] = None + resource_type: str = "unknown" + query_state: Dict[str, Any] = field(default_factory=dict) + imported_filters: List[Dict[str, Any]] = field(default_factory=list) + unresolved_references: List[str] = field(default_factory=list) + partial_recovery: bool = False + dataset_payload: Optional[Dict[str, Any]] = None + + +# [/DEF:SupersetParsedContext:Class] + + +# [DEF:SupersetContextExtractorBase:Class] +# @COMPLEXITY: 4 +# @PURPOSE: Bind extractor to one Superset environment and client instance; provide shared URL-parsing helpers. +# @RELATION: DEPENDS_ON -> [Environment] +# @PRE: constructor receives a configured environment with a usable Superset base URL. +# @POST: extractor instance is ready to parse links against one Superset environment. +# @SIDE_EFFECT: downstream parse operations may call Superset APIs through SupersetClient. +class SupersetContextExtractorBase: + # [DEF:SupersetContextExtractorBase.__init__:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Bind extractor to one Superset environment and client instance. + def __init__( + self, environment: Environment, client: Optional[SupersetClient] = None + ) -> None: + self.environment = environment + self.client = client or SupersetClient(environment) + + # [/DEF:SupersetContextExtractorBase.__init__:Function] + + # [DEF:SupersetContextExtractorBase.build_recovery_summary:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Summarize recovered, partial, and unresolved context for session state and UX. + def build_recovery_summary( + self, parsed_context: SupersetParsedContext + ) -> Dict[str, Any]: + return { + "dataset_ref": parsed_context.dataset_ref, + "dataset_id": parsed_context.dataset_id, + "dashboard_id": parsed_context.dashboard_id, + "chart_id": parsed_context.chart_id, + "partial_recovery": parsed_context.partial_recovery, + "unresolved_references": list(parsed_context.unresolved_references), + "imported_filter_count": len(parsed_context.imported_filters), + } + + # [/DEF:SupersetContextExtractorBase.build_recovery_summary:Function] + + # [DEF:SupersetContextExtractorBase._extract_numeric_identifier:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Extract a numeric identifier from a REST-like Superset URL path. + def _extract_numeric_identifier( + self, path_parts: List[str], resource_name: str + ) -> Optional[int]: + if resource_name not in path_parts: + return None + try: + resource_index = path_parts.index(resource_name) + except ValueError: + return None + + if resource_index + 1 >= len(path_parts): + return None + + candidate = str(path_parts[resource_index + 1]).strip() + if not candidate.isdigit(): + return None + return int(candidate) + + # [/DEF:SupersetContextExtractorBase._extract_numeric_identifier:Function] + + # [DEF:SupersetContextExtractorBase._extract_dashboard_reference:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Extract a dashboard id-or-slug reference from a Superset URL path. + def _extract_dashboard_reference(self, path_parts: List[str]) -> Optional[str]: + if "dashboard" not in path_parts: + return None + try: + resource_index = path_parts.index("dashboard") + except ValueError: + return None + + if resource_index + 1 >= len(path_parts): + return None + + candidate = str(path_parts[resource_index + 1]).strip() + if not candidate or candidate == "p": + return None + return candidate + + # [/DEF:SupersetContextExtractorBase._extract_dashboard_reference:Function] + + # [DEF:SupersetContextExtractorBase._extract_dashboard_permalink_key:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Extract a dashboard permalink key from a Superset URL path. + def _extract_dashboard_permalink_key(self, path_parts: List[str]) -> Optional[str]: + if "dashboard" not in path_parts: + return None + try: + resource_index = path_parts.index("dashboard") + except ValueError: + return None + + if resource_index + 2 >= len(path_parts): + return None + + permalink_marker = str(path_parts[resource_index + 1]).strip() + permalink_key = str(path_parts[resource_index + 2]).strip() + if permalink_marker != "p" or not permalink_key: + return None + return permalink_key + + # [/DEF:SupersetContextExtractorBase._extract_dashboard_permalink_key:Function] + + # [DEF:SupersetContextExtractorBase._extract_dashboard_id_from_state:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Extract a dashboard identifier from returned permalink state when present. + def _extract_dashboard_id_from_state(self, state: Dict[str, Any]) -> Optional[int]: + return self._search_nested_numeric_key( + payload=state, + candidate_keys={"dashboardId", "dashboard_id", "dashboard_id_value"}, + ) + + # [/DEF:SupersetContextExtractorBase._extract_dashboard_id_from_state:Function] + + # [DEF:SupersetContextExtractorBase._extract_chart_id_from_state:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Extract a chart identifier from returned permalink state when dashboard id is absent. + def _extract_chart_id_from_state(self, state: Dict[str, Any]) -> Optional[int]: + return self._search_nested_numeric_key( + payload=state, + candidate_keys={"slice_id", "sliceId", "chartId", "chart_id"}, + ) + + # [/DEF:SupersetContextExtractorBase._extract_chart_id_from_state:Function] + + # [DEF:SupersetContextExtractorBase._search_nested_numeric_key:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Recursively search nested dict/list payloads for the first numeric value under a candidate key set. + # @RELATION: DEPENDS_ON -> [SupersetContextExtractorBase] + def _search_nested_numeric_key( + self, payload: Any, candidate_keys: Set[str] + ) -> Optional[int]: + if isinstance(payload, dict): + for key, value in payload.items(): + if key in candidate_keys: + try: + if value is not None: + return int(value) + except (TypeError, ValueError): + pass + found = self._search_nested_numeric_key(value, candidate_keys) + if found is not None: + return found + elif isinstance(payload, list): + for item in payload: + found = self._search_nested_numeric_key(item, candidate_keys) + if found is not None: + return found + return None + + # [/DEF:SupersetContextExtractorBase._search_nested_numeric_key:Function] + + # [DEF:SupersetContextExtractorBase._decode_query_state:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Decode query-string structures used by Superset URL state transport. + def _decode_query_state(self, query_params: Dict[str, List[str]]) -> Dict[str, Any]: + query_state: Dict[str, Any] = {} + for key, values in query_params.items(): + if not values: + continue + raw_value = values[-1] + decoded_value = unquote(raw_value) + if key in {"native_filters", "form_data", "q"}: + try: + query_state[key] = json.loads(decoded_value) + continue + except Exception: + logger.explore( + "Failed to decode structured Superset query state; preserving raw value", + extra={"key": key}, + ) + query_state[key] = decoded_value + return query_state + + # [/DEF:SupersetContextExtractorBase._decode_query_state:Function] + + +# [/DEF:SupersetContextExtractorBase:Class] diff --git a/backend/src/core/utils/superset_context_extractor/_filters.py b/backend/src/core/utils/superset_context_extractor/_filters.py new file mode 100644 index 00000000..06070815 --- /dev/null +++ b/backend/src/core/utils/superset_context_extractor/_filters.py @@ -0,0 +1,261 @@ +# [DEF:SupersetContextFiltersExtractMixin:Module] +# @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: superset, filters, extraction, query_state, dataMask, native_filters +# @PURPOSE: Extract normalized imported filters from decoded Superset query state (native_filters, dataMask, native_filter_state, form_data). +# @RELATION: DEPENDS_ON -> [SupersetContextExtractorBase] +# [/DEF:SupersetContextFiltersExtractMixin:Module] + +# [DEF:_filters_imports:Block] +from __future__ import annotations + +from copy import deepcopy +from typing import Any, Dict, List, cast + +from ...logger import logger as app_logger + +app_logger = cast(Any, app_logger) +# [/DEF:_filters_imports:Block] + + +# [DEF:SupersetContextFiltersExtractMixin:Class] +# @COMPLEXITY: 3 +# @PURPOSE: Mixin providing query-state filter extraction for the composed SupersetContextExtractor. +class SupersetContextFiltersExtractMixin: + # [DEF:SupersetContextFiltersExtractMixin._extract_imported_filters:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Normalize imported filters from decoded query state without fabricating missing values. + def _extract_imported_filters( + self, query_state: Dict[str, Any] + ) -> List[Dict[str, Any]]: + imported_filters: List[Dict[str, Any]] = [] + + native_filters_payload = query_state.get("native_filters") + if isinstance(native_filters_payload, list): + for index, item in enumerate(native_filters_payload): + if not isinstance(item, dict): + continue + filter_name = ( + item.get("filter_name") + or item.get("column") + or item.get("name") + or f"native_filter_{index}" + ) + direct_clause = None + if item.get("column") and ("value" in item or "val" in item): + direct_clause = { + "col": item.get("column"), + "op": item.get("op") + or ( + "IN" + if isinstance(item.get("value"), list) + else "==" + ), + "val": item.get("val", item.get("value")), + } + imported_filters.append( + { + "filter_name": str(filter_name), + "raw_value": item.get("value"), + "display_name": item.get("label") or item.get("name"), + "normalized_value": { + "filter_clauses": [direct_clause] + if isinstance(direct_clause, dict) + else [], + "extra_form_data": {}, + "value_origin": "native_filters", + }, + "source": "superset_url", + "recovery_status": "recovered" + if item.get("value") is not None + else "partial", + "requires_confirmation": item.get("value") is None, + "notes": "Recovered from Superset native filter URL state", + } + ) + + # Extract filters from permalink dataMask + dashboard_data_mask = query_state.get("dataMask") + if isinstance(dashboard_data_mask, dict): + for filter_key, item in dashboard_data_mask.items(): + if not isinstance(item, dict): + continue + filter_state = item.get("filterState") + extra_form_data = item.get("extraFormData") + display_name = None + raw_value = None + normalized_value = { + "filter_clauses": [], + "extra_form_data": deepcopy(extra_form_data) + if isinstance(extra_form_data, dict) + else {}, + "value_origin": "unresolved", + } + + if isinstance(filter_state, dict): + display_name = filter_state.get("label") + raw_value = filter_state.get("value") or filter_state.get( + "values" + ) + if raw_value is not None: + normalized_value["value_origin"] = "filter_state" + + if isinstance(extra_form_data, dict): + extra_filters = extra_form_data.get("filters") + if isinstance(extra_filters, list): + normalized_value["filter_clauses"] = [ + deepcopy(extra_filter) + for extra_filter in extra_filters + if isinstance(extra_filter, dict) + ] + + if raw_value is None and normalized_value["filter_clauses"]: + first_filter = normalized_value["filter_clauses"][0] + raw_value = first_filter.get("val") + if raw_value is None: + raw_value = first_filter.get("value") + if raw_value is not None: + normalized_value["value_origin"] = ( + "extra_form_data.filters" + ) + + if raw_value is None and isinstance(extra_form_data, dict): + for field in [ + "time_range", + "time_grain_sqla", + "time_column", + "granularity", + ]: + if field in extra_form_data: + raw_value = extra_form_data[field] + normalized_value["value_origin"] = ( + f"extra_form_data.{field}" + ) + break + + imported_filters.append( + { + "filter_name": str(item.get("id") or filter_key), + "raw_value": raw_value, + "display_name": display_name, + "normalized_value": normalized_value, + "source": "superset_permalink", + "recovery_status": "recovered" + if raw_value is not None + else "partial", + "requires_confirmation": raw_value is None, + "notes": "Recovered from Superset dashboard permalink state", + } + ) + + # Extract filters from native_filter_state (fetched via native_filters_key) + native_filter_state = query_state.get("native_filter_state") + if isinstance(native_filter_state, dict): + for filter_key, item in native_filter_state.items(): + if not isinstance(item, dict): + continue + filter_id = item.get("id") or filter_key + filter_state = item.get("filterState") + extra_form_data = item.get("extraFormData") + display_name = None + raw_value = None + normalized_value = { + "filter_clauses": [], + "extra_form_data": deepcopy(extra_form_data) + if isinstance(extra_form_data, dict) + else {}, + "value_origin": "unresolved", + } + + if isinstance(filter_state, dict): + display_name = filter_state.get("label") + raw_value = filter_state.get("value") or filter_state.get( + "values" + ) + if raw_value is not None: + normalized_value["value_origin"] = "filter_state" + + if isinstance(extra_form_data, dict): + extra_filters = extra_form_data.get("filters") + if isinstance(extra_filters, list): + normalized_value["filter_clauses"] = [ + deepcopy(extra_filter) + for extra_filter in extra_filters + if isinstance(extra_filter, dict) + ] + + if raw_value is None and normalized_value["filter_clauses"]: + first_filter = normalized_value["filter_clauses"][0] + raw_value = first_filter.get("val") + if raw_value is None: + raw_value = first_filter.get("value") + if raw_value is not None: + normalized_value["value_origin"] = ( + "extra_form_data.filters" + ) + + if raw_value is None and isinstance(extra_form_data, dict): + for field in [ + "time_range", + "time_grain_sqla", + "time_column", + "granularity", + ]: + if field in extra_form_data: + raw_value = extra_form_data[field] + normalized_value["value_origin"] = ( + f"extra_form_data.{field}" + ) + break + + imported_filters.append( + { + "filter_name": str(filter_id), + "raw_value": raw_value, + "display_name": display_name, + "normalized_value": normalized_value, + "source": "superset_native_filters_key", + "recovery_status": "recovered" + if raw_value is not None + else "partial", + "requires_confirmation": raw_value is None, + "notes": "Recovered from Superset native_filters_key state", + } + ) + + form_data_payload = query_state.get("form_data") + if isinstance(form_data_payload, dict): + extra_filters = form_data_payload.get("extra_filters") or [] + for index, item in enumerate(extra_filters): + if not isinstance(item, dict): + continue + filter_name = ( + item.get("col") + or item.get("column") + or f"extra_filter_{index}" + ) + imported_filters.append( + { + "filter_name": str(filter_name), + "raw_value": item.get("val"), + "display_name": item.get("label"), + "normalized_value": { + "filter_clauses": [deepcopy(item)], + "extra_form_data": {}, + "value_origin": "form_data.extra_filters", + }, + "source": "superset_url", + "recovery_status": "recovered" + if item.get("val") is not None + else "partial", + "requires_confirmation": item.get("val") is None, + "notes": "Recovered from Superset form_data extra_filters", + } + ) + + return imported_filters + + # [/DEF:SupersetContextFiltersExtractMixin._extract_imported_filters:Function] + + +# [/DEF:SupersetContextFiltersExtractMixin:Class] diff --git a/backend/src/core/utils/superset_context_extractor/_parsing.py b/backend/src/core/utils/superset_context_extractor/_parsing.py new file mode 100644 index 00000000..28070035 --- /dev/null +++ b/backend/src/core/utils/superset_context_extractor/_parsing.py @@ -0,0 +1,380 @@ +# [DEF:SupersetContextParsingMixin:Module] +# @COMPLEXITY: 4 +# @LAYER: Infra +# @SEMANTICS: superset, link_parsing, url, permalink, recovery +# @PURPOSE: Parse supported Superset URLs and recover canonical dataset/dashboard references for review-session intake. +# @RELATION: CALLS -> [SupersetClient] +# @RELATION: CALLS -> [SupersetContextExtractorBase] +# [/DEF:SupersetContextParsingMixin:Module] + +# [DEF:_parsing_imports:Block] +from __future__ import annotations + +from typing import Any, Dict, List, Optional, cast +from urllib.parse import parse_qs, urlparse + +from ...logger import belief_scope, logger +from ._base import SupersetParsedContext + +logger = cast(Any, logger) +# [/DEF:_parsing_imports:Block] + + +# [DEF:SupersetContextParsingMixin:Class] +# @COMPLEXITY: 4 +# @PURPOSE: Mixin providing Superset URL parsing logic for the composed SupersetContextExtractor. +class SupersetContextParsingMixin: + # [DEF:SupersetContextParsingMixin.parse_superset_link:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Extract candidate identifiers and query state from supported Superset URLs. + # @RELATION: CALLS -> [SupersetClient.get_dashboard_detail] + # @PRE: link is a non-empty Superset URL compatible with the configured environment. + # @POST: returns resolved dataset/dashboard context, preserving explicit partial-recovery state if some identifiers cannot be confirmed. + # @SIDE_EFFECT: may issue Superset API reads to resolve dataset references from dashboard or chart URLs. + # @DATA_CONTRACT: Input[link:str] -> Output[SupersetParsedContext] + def parse_superset_link(self, link: str) -> SupersetParsedContext: + with belief_scope("SupersetContextExtractor.parse_superset_link"): + normalized_link = str(link or "").strip() + if not normalized_link: + logger.explore("Rejected empty Superset link during intake") + raise ValueError("Superset link must be non-empty") + + parsed_url = urlparse(normalized_link) + if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc: + logger.explore( + "Superset link is not a parseable absolute URL", + extra={"link": normalized_link}, + ) + raise ValueError("Superset link must be an absolute http(s) URL") + + logger.reason( + "Parsing Superset link for dataset review intake", + extra={"path": parsed_url.path, "query": parsed_url.query}, + ) + + path_parts = [part for part in parsed_url.path.split("/") if part] + query_params = parse_qs(parsed_url.query, keep_blank_values=True) + query_state = self._decode_query_state(query_params) + + dataset_id = self._extract_numeric_identifier(path_parts, "dataset") + dashboard_id = self._extract_numeric_identifier(path_parts, "dashboard") + dashboard_ref = self._extract_dashboard_reference(path_parts) + dashboard_permalink_key = self._extract_dashboard_permalink_key(path_parts) + chart_id = self._extract_numeric_identifier(path_parts, "chart") + + resource_type = "unknown" + dataset_ref: Optional[str] = None + partial_recovery = False + unresolved_references: List[str] = [] + + if dataset_id is not None: + resource_type = "dataset" + dataset_ref = f"dataset:{dataset_id}" + logger.reason( + "Resolved direct dataset link", + extra={"dataset_id": dataset_id}, + ) + elif dashboard_permalink_key is not None: + resource_type = "dashboard" + partial_recovery = True + dataset_ref = f"dashboard_permalink:{dashboard_permalink_key}" + unresolved_references.append( + "dashboard_permalink_dataset_binding_unresolved" + ) + logger.reason( + "Resolving dashboard permalink state from Superset", + extra={"permalink_key": dashboard_permalink_key}, + ) + permalink_payload = self.client.get_dashboard_permalink_state( + dashboard_permalink_key + ) + permalink_state = ( + permalink_payload.get("state", permalink_payload) + if isinstance(permalink_payload, dict) + else {} + ) + if isinstance(permalink_state, dict): + for key, value in permalink_state.items(): + query_state.setdefault(key, value) + # Extract filters from permalink dataMask + data_mask = permalink_state.get("dataMask") + if isinstance(data_mask, dict) and data_mask: + query_state["dataMask"] = data_mask + logger.reason( + "Extracted native filters from permalink dataMask", + extra={"filter_count": len(data_mask)}, + ) + resolved_dashboard_id = self._extract_dashboard_id_from_state( + permalink_state + ) + resolved_chart_id = self._extract_chart_id_from_state( + permalink_state + ) + if resolved_dashboard_id is not None: + dashboard_id = resolved_dashboard_id + unresolved_references = [ + item + for item in unresolved_references + if item + != "dashboard_permalink_dataset_binding_unresolved" + ] + dataset_id, unresolved_references = ( + self._recover_dataset_binding_from_dashboard( + dashboard_id=dashboard_id, + dataset_ref=dataset_ref, + unresolved_references=unresolved_references, + ) + ) + if dataset_id is not None: + dataset_ref = f"dataset:{dataset_id}" + elif resolved_chart_id is not None: + chart_id = resolved_chart_id + unresolved_references = [ + item + for item in unresolved_references + if item + != "dashboard_permalink_dataset_binding_unresolved" + ] + try: + chart_payload = self.client.get_chart(chart_id) + chart_data = ( + chart_payload.get("result", chart_payload) + if isinstance(chart_payload, dict) + else {} + ) + datasource_id = chart_data.get("datasource_id") + if datasource_id is not None: + dataset_id = int(datasource_id) + dataset_ref = f"dataset:{dataset_id}" + logger.reason( + "Recovered dataset reference from permalink chart context", + extra={ + "chart_id": chart_id, + "dataset_id": dataset_id, + }, + ) + else: + unresolved_references.append( + "chart_dataset_binding_unresolved" + ) + except Exception as exc: + unresolved_references.append( + "chart_dataset_binding_unresolved" + ) + logger.explore( + "Chart lookup failed during permalink recovery", + extra={"chart_id": chart_id, "error": str(exc)}, + ) + else: + logger.explore( + "Dashboard permalink state was not a structured object", + extra={"permalink_key": dashboard_permalink_key}, + ) + elif dashboard_id is not None or dashboard_ref is not None: + resource_type = "dashboard" + resolved_dashboard_ref = ( + dashboard_id if dashboard_id is not None else dashboard_ref + ) + if resolved_dashboard_ref is None: + raise ValueError("Dashboard reference could not be resolved") + logger.reason( + "Resolving dashboard-bound dataset from Superset", + extra={"dashboard_ref": resolved_dashboard_ref}, + ) + + # Resolve dashboard detail first — handles both numeric ID and slug, + # ensuring dashboard_id is available for the native_filters_key fetch below. + dashboard_detail = self.client.get_dashboard_detail( + resolved_dashboard_ref + ) + resolved_dashboard_id = dashboard_detail.get("id") + if resolved_dashboard_id is not None: + dashboard_id = int(resolved_dashboard_id) + + # Check for native_filters_key in query params and fetch filter state. + # This must run AFTER dashboard_id is resolved from slug above. + native_filters_key = query_params.get("native_filters_key", [None])[0] + if native_filters_key and dashboard_id is not None: + try: + logger.reason( + "Fetching native filter state from Superset", + extra={ + "dashboard_id": dashboard_id, + "filter_key": native_filters_key, + }, + ) + extracted = self.client.extract_native_filters_from_key( + dashboard_id, native_filters_key + ) + data_mask = extracted.get("dataMask") + if isinstance(data_mask, dict) and data_mask: + query_state["native_filter_state"] = data_mask + logger.reason( + "Extracted native filter state from Superset via native_filters_key", + extra={"filter_count": len(data_mask)}, + ) + else: + logger.explore( + "Native filter state returned empty dataMask", + extra={ + "dashboard_id": dashboard_id, + "filter_key": native_filters_key, + }, + ) + except Exception as exc: + logger.explore( + "Failed to fetch native filter state from Superset", + extra={ + "dashboard_id": dashboard_id, + "filter_key": native_filters_key, + "error": str(exc), + }, + ) + + datasets = dashboard_detail.get("datasets") or [] + if datasets: + first_dataset = datasets[0] + resolved_dataset_id = first_dataset.get("id") + if resolved_dataset_id is not None: + dataset_id = int(resolved_dataset_id) + dataset_ref = f"dataset:{dataset_id}" + logger.reason( + "Recovered dataset reference from dashboard context", + extra={ + "dashboard_id": dashboard_id, + "dataset_id": dataset_id, + "dataset_count": len(datasets), + }, + ) + if len(datasets) > 1: + partial_recovery = True + unresolved_references.append( + "multiple_dashboard_datasets" + ) + else: + partial_recovery = True + unresolved_references.append("dashboard_dataset_id_missing") + else: + partial_recovery = True + unresolved_references.append("dashboard_dataset_binding_missing") + elif chart_id is not None: + resource_type = "chart" + partial_recovery = True + unresolved_references.append("chart_dataset_binding_unresolved") + dataset_ref = f"chart:{chart_id}" + logger.reason( + "Accepted chart link with explicit partial recovery", + extra={"chart_id": chart_id}, + ) + else: + logger.explore( + "Unsupported Superset link shape encountered", + extra={"path": parsed_url.path}, + ) + raise ValueError("Unsupported Superset link shape") + + dataset_payload: Optional[Dict[str, Any]] = None + if dataset_id is not None: + try: + dataset_payload = self.client.get_dataset_detail(dataset_id) + table_name = str( + dataset_payload.get("table_name") or "" + ).strip() + schema_name = str( + dataset_payload.get("schema") or "" + ).strip() + if table_name: + dataset_ref = ( + f"{schema_name}.{table_name}" + if schema_name + else table_name + ) + logger.reason( + "Canonicalized dataset reference from dataset detail", + extra={ + "dataset_ref": dataset_ref, + "dataset_id": dataset_id, + }, + ) + except Exception as exc: + partial_recovery = True + unresolved_references.append("dataset_detail_lookup_failed") + logger.explore( + "Dataset detail lookup failed during link parsing; keeping session usable", + extra={"dataset_id": dataset_id, "error": str(exc)}, + ) + + imported_filters = self._extract_imported_filters(query_state) + result = SupersetParsedContext( + source_url=normalized_link, + dataset_ref=dataset_ref or "unresolved", + dataset_id=dataset_id, + dashboard_id=dashboard_id, + chart_id=chart_id, + resource_type=resource_type, + query_state=query_state, + imported_filters=imported_filters, + unresolved_references=unresolved_references, + partial_recovery=partial_recovery, + dataset_payload=dataset_payload, + ) + logger.reflect( + "Superset link parsing completed", + extra={ + "dataset_ref": result.dataset_ref, + "dataset_id": result.dataset_id, + "dashboard_id": result.dashboard_id, + "chart_id": result.chart_id, + "partial_recovery": result.partial_recovery, + "unresolved_references": result.unresolved_references, + "imported_filters": len(result.imported_filters), + }, + ) + return result + + # [/DEF:SupersetContextParsingMixin.parse_superset_link:Function] + + # [DEF:SupersetContextParsingMixin._recover_dataset_binding_from_dashboard:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Recover a dataset binding from resolved dashboard context while preserving explicit unresolved markers. + # @RELATION: CALLS -> [SupersetClient.get_dashboard_detail] + def _recover_dataset_binding_from_dashboard( + self, + dashboard_id: int, + dataset_ref: Optional[str], + unresolved_references: List[str], + ) -> tuple[Optional[int], List[str]]: + dashboard_detail = self.client.get_dashboard_detail(dashboard_id) + datasets = dashboard_detail.get("datasets") or [] + if datasets: + first_dataset = datasets[0] + resolved_dataset_id = first_dataset.get("id") + if resolved_dataset_id is not None: + resolved_dataset = int(resolved_dataset_id) + logger.reason( + "Recovered dataset reference from dashboard permalink context", + extra={ + "dashboard_id": dashboard_id, + "dataset_id": resolved_dataset, + "dataset_count": len(datasets), + "dataset_ref": dataset_ref, + }, + ) + if ( + len(datasets) > 1 + and "multiple_dashboard_datasets" not in unresolved_references + ): + unresolved_references.append("multiple_dashboard_datasets") + return resolved_dataset, unresolved_references + if "dashboard_dataset_id_missing" not in unresolved_references: + unresolved_references.append("dashboard_dataset_id_missing") + return None, unresolved_references + + if "dashboard_dataset_binding_missing" not in unresolved_references: + unresolved_references.append("dashboard_dataset_binding_missing") + return None, unresolved_references + + # [/DEF:SupersetContextParsingMixin._recover_dataset_binding_from_dashboard:Function] + + +# [/DEF:SupersetContextParsingMixin:Class] diff --git a/backend/src/core/utils/superset_context_extractor/_pii.py b/backend/src/core/utils/superset_context_extractor/_pii.py new file mode 100644 index 00000000..dea57138 --- /dev/null +++ b/backend/src/core/utils/superset_context_extractor/_pii.py @@ -0,0 +1,112 @@ +# [DEF:SupersetContextExtractorPII:Module] +# @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: superset, pii, masking, sanitization, privacy +# @PURPOSE: PII redaction helpers for assistant-facing dataset-review context — mask emails, UUIDs, long digit strings, and mixed identifiers. +# [/DEF:SupersetContextExtractorPII:Module] + +# [DEF:_pii_imports:Block] +import re +from copy import deepcopy +from typing import Any, Dict, List, Tuple +# [/DEF:_pii_imports:Block] + +_EMAIL_PATTERN = re.compile( + r"(?P[A-Za-z0-9._%+-]+)@(?P[A-Za-z0-9.-]+\.[A-Za-z]{2,})" +) +_UUID_PATTERN = re.compile( + r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b" +) +_LONG_DIGIT_PATTERN = re.compile(r"\b\d{6,}\b") +_MIXED_IDENTIFIER_PATTERN = re.compile( + r"\b(?=[A-Za-z0-9_-]{10,}\b)(?=.*[A-Za-z])(?=.*\d)[A-Za-z0-9_-]+\b" +) + + +# [DEF:mask_pii_value:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Redact likely sensitive identifiers while preserving structural hints for assistant-facing dataset-review context. +def mask_pii_value(value: Any) -> Tuple[Any, bool]: + if isinstance(value, str): + return _mask_sensitive_text(value) + if isinstance(value, list): + masked_any = False + masked_list: List[Any] = [] + for item in value: + masked_item, item_masked = mask_pii_value(item) + masked_list.append(masked_item) + masked_any = masked_any or item_masked + return masked_list, masked_any + if isinstance(value, dict): + masked_any = False + masked_dict: Dict[str, Any] = {} + for key, item in value.items(): + masked_item, item_masked = mask_pii_value(item) + masked_dict[key] = masked_item + masked_any = masked_any or item_masked + return masked_dict, masked_any + return value, False + + +# [/DEF:mask_pii_value:Function] + + +# [DEF:sanitize_imported_filter_for_assistant:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Build assistant-safe imported-filter payload with raw-value redaction metadata when sensitive tokens are detected. +def sanitize_imported_filter_for_assistant( + filter_payload: Dict[str, Any], +) -> Dict[str, Any]: + sanitized = deepcopy(filter_payload) + masked_raw_value, was_masked = mask_pii_value(filter_payload.get("raw_value")) + sanitized["raw_value"] = masked_raw_value + sanitized["raw_value_masked"] = bool( + filter_payload.get("raw_value_masked", False) or was_masked + ) + return sanitized + + +# [/DEF:sanitize_imported_filter_for_assistant:Function] + + +# [DEF:_mask_sensitive_text:Function] +# @COMPLEXITY: 2 +# @PURPOSE: Redact likely PII substrings from one string while preserving non-sensitive delimiters and token shape. +def _mask_sensitive_text(value: str) -> Tuple[str, bool]: + masked = value + masked_any = False + + def _replace_email(match: re.Match[str]) -> str: + nonlocal masked_any + masked_any = True + domain = match.group("domain") + return f"***@{domain}" + + def _replace_uuid(match: re.Match[str]) -> str: + nonlocal masked_any + masked_any = True + token = match.group(0) + return "********-****-****-****-" + token[-12:] + + def _replace_long_digits(match: re.Match[str]) -> str: + nonlocal masked_any + masked_any = True + token = match.group(0) + return "*" * max(len(token) - 2, 1) + token[-2:] + + def _replace_mixed_identifier(match: re.Match[str]) -> str: + nonlocal masked_any + token = match.group(0) + if token.isupper() and len(token) <= 5: + return token + masked_any = True + return token[:2] + "***" + token[-2:] + + masked = _EMAIL_PATTERN.sub(_replace_email, masked) + masked = _UUID_PATTERN.sub(_replace_uuid, masked) + masked = _LONG_DIGIT_PATTERN.sub(_replace_long_digits, masked) + masked = _MIXED_IDENTIFIER_PATTERN.sub(_replace_mixed_identifier, masked) + return masked, masked_any + + +# [/DEF:_mask_sensitive_text:Function] diff --git a/backend/src/core/utils/superset_context_extractor/_recovery.py b/backend/src/core/utils/superset_context_extractor/_recovery.py new file mode 100644 index 00000000..1ca11888 --- /dev/null +++ b/backend/src/core/utils/superset_context_extractor/_recovery.py @@ -0,0 +1,314 @@ +# [DEF:SupersetContextRecoveryMixin:Module] +# @COMPLEXITY: 4 +# @LAYER: Infra +# @SEMANTICS: superset, filters, recovery, imported_filters +# @PURPOSE: Recover imported filters from Superset parsed context and dashboard metadata. +# @RELATION: CALLS -> [SupersetClient] +# @RELATION: DEPENDS_ON -> [SupersetContextExtractorBase] +# [/DEF:SupersetContextRecoveryMixin:Module] + +# [DEF:_recovery_imports:Block] +from __future__ import annotations + +import json +from copy import deepcopy +from typing import Any, Dict, List, Set, cast + +from ...logger import belief_scope, logger +from ._base import SupersetParsedContext + +logger = cast(Any, logger) +# [/DEF:_recovery_imports:Block] + + +# [DEF:SupersetContextRecoveryMixin:Class] +# @COMPLEXITY: 4 +# @PURPOSE: Mixin providing filter recovery for the composed SupersetContextExtractor. +class SupersetContextRecoveryMixin: + # [DEF:SupersetContextRecoveryMixin.recover_imported_filters:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Build imported filter entries from URL state and Superset-side saved context. + # @RELATION: CALLS -> [SupersetClient.get_dashboard] + # @PRE: parsed_context comes from a successful Superset link parse for one environment. + # @POST: returns explicit recovered and partial filter entries with preserved provenance and confirmation requirements. + # @SIDE_EFFECT: may issue Superset reads for dashboard metadata enrichment. + # @DATA_CONTRACT: Input[SupersetParsedContext] -> Output[List[Dict[str,Any]]] + def recover_imported_filters( + self, parsed_context: SupersetParsedContext + ) -> List[Dict[str, Any]]: + with belief_scope("SupersetContextExtractor.recover_imported_filters"): + recovered_filters: List[Dict[str, Any]] = [] + seen_filter_keys: Set[str] = set() + metadata_filters: List[Dict[str, Any]] = [] + metadata_filters_by_id: Dict[str, Dict[str, Any]] = {} + + def merge_recovered_filter(candidate: Dict[str, Any]) -> None: + filter_key = candidate["filter_name"].strip().lower() + existing_index = next( + ( + index + for index, existing in enumerate(recovered_filters) + if existing["filter_name"].strip().lower() == filter_key + ), + None, + ) + if existing_index is None: + seen_filter_keys.add(filter_key) + recovered_filters.append(candidate) + return + + existing = recovered_filters[existing_index] + if existing.get("display_name") in { + None, + "", + existing.get("filter_name"), + } and candidate.get("display_name"): + existing["display_name"] = candidate["display_name"] + if ( + existing.get("raw_value") is None + and candidate.get("raw_value") is not None + ): + existing["raw_value"] = candidate["raw_value"] + existing["confidence_state"] = candidate.get( + "confidence_state", "imported" + ) + existing["requires_confirmation"] = candidate.get( + "requires_confirmation", False + ) + existing["recovery_status"] = candidate.get( + "recovery_status", "recovered" + ) + existing["source"] = candidate.get( + "source", existing.get("source") + ) + if ( + existing.get("normalized_value") is None + and candidate.get("normalized_value") is not None + ): + existing["normalized_value"] = deepcopy( + candidate["normalized_value"] + ) + if ( + existing.get("notes") + and candidate.get("notes") + and candidate["notes"] not in existing["notes"] + ): + existing["notes"] = f"{existing['notes']}; {candidate['notes']}" + + if parsed_context.dashboard_id is not None: + try: + dashboard_payload = self.client.get_dashboard( + parsed_context.dashboard_id + ) + dashboard_record = ( + dashboard_payload.get("result", dashboard_payload) + if isinstance(dashboard_payload, dict) + else {} + ) + json_metadata = dashboard_record.get("json_metadata") + if isinstance(json_metadata, str) and json_metadata.strip(): + json_metadata = json.loads(json_metadata) + if not isinstance(json_metadata, dict): + json_metadata = {} + + native_filter_configuration = ( + json_metadata.get("native_filter_configuration") or [] + ) + default_filters = json_metadata.get("default_filters") or {} + if isinstance(default_filters, str) and default_filters.strip(): + try: + default_filters = json.loads(default_filters) + except Exception: + logger.explore( + "Superset default_filters payload was not valid JSON", + extra={ + "dashboard_id": parsed_context.dashboard_id + }, + ) + default_filters = {} + + for item in native_filter_configuration: + if not isinstance(item, dict): + continue + filter_name = str( + item.get("name") + or item.get("filter_name") + or item.get("column") + or "" + ).strip() + if not filter_name: + continue + + display_name = ( + item.get("label") or item.get("name") or filter_name + ) + filter_id = str(item.get("id") or "").strip() + + default_value = None + if isinstance(default_filters, dict): + default_value = default_filters.get(filter_name) + + metadata_filter = self._normalize_imported_filter_payload( + { + "filter_name": filter_name, + "display_name": display_name, + "raw_value": default_value, + "source": "superset_native", + "recovery_status": "recovered" + if default_value is not None + else "partial", + "requires_confirmation": default_value is None, + "notes": "Recovered from Superset dashboard native filter configuration", + }, + default_source="superset_native", + default_note="Recovered from Superset dashboard native filter configuration", + ) + metadata_filters.append(metadata_filter) + + if filter_id: + metadata_filters_by_id[filter_id.lower()] = { + "filter_name": filter_name, + "display_name": display_name, + } + except Exception as exc: + logger.explore( + "Dashboard native filter enrichment failed; preserving partial imported filters", + extra={ + "dashboard_id": parsed_context.dashboard_id, + "error": str(exc), + "filter_count": len(recovered_filters), + }, + ) + metadata_filters = [] + metadata_filters_by_id = {} + + for item in parsed_context.imported_filters: + normalized = self._normalize_imported_filter_payload( + item, + default_source="superset_url", + default_note="Recovered from Superset URL state", + ) + metadata_match = metadata_filters_by_id.get( + normalized["filter_name"].strip().lower() + ) + if metadata_match is not None: + normalized["filter_name"] = metadata_match["filter_name"] + normalized["display_name"] = metadata_match["display_name"] + normalized["notes"] = ( + "Recovered from Superset URL state and reconciled against dashboard native filter metadata" + ) + + merge_recovered_filter(normalized) + logger.reflect( + "Recovered filter from URL state", + extra={ + "filter_name": normalized["filter_name"], + "source": normalized["source"], + "has_value": normalized["raw_value"] is not None, + "canonicalized": metadata_match is not None, + }, + ) + + if parsed_context.dashboard_id is None: + logger.reflect( + "Imported filter recovery completed without dashboard enrichment", + extra={ + "dashboard_id": None, + "filter_count": len(recovered_filters), + "partial_recovery": parsed_context.partial_recovery, + }, + ) + return recovered_filters + + for saved_filter in metadata_filters: + merge_recovered_filter(saved_filter) + logger.reflect( + "Recovered filter from dashboard metadata", + extra={ + "filter_name": saved_filter["filter_name"], + "has_value": saved_filter["raw_value"] is not None, + }, + ) + + if not recovered_filters: + recovered_filters.append( + self._normalize_imported_filter_payload( + { + "filter_name": f"dashboard_{parsed_context.dashboard_id}_filters", + "display_name": "Dashboard native filters", + "raw_value": None, + "source": "superset_native", + "recovery_status": "partial", + "requires_confirmation": True, + "notes": "Superset dashboard filter configuration could not be recovered fully", + }, + default_source="superset_native", + default_note="Superset dashboard filter configuration could not be recovered fully", + ) + ) + + logger.reflect( + "Imported filter recovery completed with dashboard enrichment", + extra={ + "dashboard_id": parsed_context.dashboard_id, + "filter_count": len(recovered_filters), + "partial_entries": len( + [ + item + for item in recovered_filters + if item["recovery_status"] == "partial" + ] + ), + }, + ) + return recovered_filters + + # [/DEF:SupersetContextRecoveryMixin.recover_imported_filters:Function] + + # [DEF:SupersetContextRecoveryMixin._normalize_imported_filter_payload:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Normalize one imported-filter payload with explicit provenance and confirmation state. + def _normalize_imported_filter_payload( + self, + payload: Dict[str, Any], + default_source: str, + default_note: str, + ) -> Dict[str, Any]: + raw_value = payload.get("raw_value") + if "raw_value" not in payload and "value" in payload: + raw_value = payload.get("value") + + recovery_status = ( + str( + payload.get("recovery_status") + or ("recovered" if raw_value is not None else "partial") + ) + .strip() + .lower() + ) + requires_confirmation = bool( + payload.get( + "requires_confirmation", + raw_value is None or recovery_status != "recovered", + ) + ) + return { + "filter_name": str( + payload.get("filter_name") or "unresolved_filter" + ).strip(), + "display_name": payload.get("display_name"), + "raw_value": raw_value, + "normalized_value": payload.get("normalized_value"), + "source": str(payload.get("source") or default_source), + "confidence_state": "imported" + if raw_value is not None + else "unresolved", + "requires_confirmation": requires_confirmation, + "recovery_status": recovery_status, + "notes": str(payload.get("notes") or default_note), + } + + # [/DEF:SupersetContextRecoveryMixin._normalize_imported_filter_payload:Function] + + +# [/DEF:SupersetContextRecoveryMixin:Class] diff --git a/backend/src/core/utils/superset_context_extractor/_templates.py b/backend/src/core/utils/superset_context_extractor/_templates.py new file mode 100644 index 00000000..7c076b0c --- /dev/null +++ b/backend/src/core/utils/superset_context_extractor/_templates.py @@ -0,0 +1,255 @@ +# [DEF:SupersetContextTemplatesMixin:Module] +# @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: superset, template_variables, jinja, discovery, deterministic +# @PURPOSE: Deterministically detect runtime variables and Jinja references from dataset query-bearing fields without execution. +# @RELATION: DEPENDS_ON -> [TemplateVariable] +# @RELATION: DEPENDS_ON -> [SupersetContextExtractorBase] +# [/DEF:SupersetContextTemplatesMixin:Module] + +# [DEF:_templates_imports:Block] +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional, Set, cast + +from ...logger import belief_scope, logger + +logger = cast(Any, logger) +# [/DEF:_templates_imports:Block] + + +# [DEF:SupersetContextTemplatesMixin:Class] +# @COMPLEXITY: 3 +# @PURPOSE: Mixin providing template variable discovery for the composed SupersetContextExtractor. +class SupersetContextTemplatesMixin: + # [DEF:SupersetContextTemplatesMixin.discover_template_variables:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Detect runtime variables and Jinja references from dataset query-bearing fields. + # @PRE: dataset_payload is a Superset dataset-detail style payload with query-bearing fields when available. + # @POST: returns deduplicated explicit variable records without executing Jinja or fabricating runtime values. + # @SIDE_EFFECT: none. + # @DATA_CONTRACT: Input[dataset_payload:Dict[str,Any]] -> Output[List[Dict[str,Any]]] + def discover_template_variables( + self, dataset_payload: Dict[str, Any] + ) -> List[Dict[str, Any]]: + with belief_scope("SupersetContextExtractor.discover_template_variables"): + discovered: List[Dict[str, Any]] = [] + seen_variable_names: Set[str] = set() + + for expression_source in self._collect_query_bearing_expressions( + dataset_payload + ): + for filter_match in re.finditer( + r"filter_values\(\s*['\"]([^'\"]+)['\"]\s*\)", + expression_source, + flags=re.IGNORECASE, + ): + variable_name = str(filter_match.group(1) or "").strip() + if not variable_name: + continue + self._append_template_variable( + discovered=discovered, + seen_variable_names=seen_variable_names, + variable_name=variable_name, + expression_source=expression_source, + variable_kind="native_filter", + is_required=True, + default_value=None, + ) + + for url_param_match in re.finditer( + r"url_param\(\s*['\"]([^'\"]+)['\"]\s*(?:,\s*([^)]+))?\)", + expression_source, + flags=re.IGNORECASE, + ): + variable_name = str(url_param_match.group(1) or "").strip() + if not variable_name: + continue + default_literal = url_param_match.group(2) + self._append_template_variable( + discovered=discovered, + seen_variable_names=seen_variable_names, + variable_name=variable_name, + expression_source=expression_source, + variable_kind="parameter", + is_required=default_literal is None, + default_value=self._normalize_default_literal( + default_literal + ), + ) + + for jinja_match in re.finditer( + r"\{\{\s*(.*?)\s*\}\}", expression_source, flags=re.DOTALL + ): + expression = str(jinja_match.group(1) or "").strip() + if not expression: + continue + if any( + token in expression + for token in ( + "filter_values(", + "url_param(", + "get_filters(", + ) + ): + continue + variable_name = self._extract_primary_jinja_identifier(expression) + if not variable_name: + continue + self._append_template_variable( + discovered=discovered, + seen_variable_names=seen_variable_names, + variable_name=variable_name, + expression_source=expression_source, + variable_kind="derived" + if "." in expression or "|" in expression + else "parameter", + is_required=True, + default_value=None, + ) + + logger.reflect( + "Template variable discovery completed deterministically", + extra={ + "dataset_id": dataset_payload.get("id"), + "variable_count": len(discovered), + "variable_names": [ + item["variable_name"] for item in discovered + ], + }, + ) + return discovered + + # [/DEF:SupersetContextTemplatesMixin.discover_template_variables:Function] + + # [DEF:SupersetContextTemplatesMixin._collect_query_bearing_expressions:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Collect SQL and expression-bearing dataset fields for deterministic template-variable discovery. + # @RELATION: DEPENDS_ON -> [SupersetContextTemplatesMixin.discover_template_variables] + def _collect_query_bearing_expressions( + self, dataset_payload: Dict[str, Any] + ) -> List[str]: + expressions: List[str] = [] + + def append_expression(candidate: Any) -> None: + if not isinstance(candidate, str): + return + normalized = candidate.strip() + if normalized: + expressions.append(normalized) + + append_expression(dataset_payload.get("sql")) + append_expression(dataset_payload.get("query")) + append_expression(dataset_payload.get("template_sql")) + + metrics_payload = dataset_payload.get("metrics") or [] + if isinstance(metrics_payload, list): + for metric in metrics_payload: + if isinstance(metric, str): + append_expression(metric) + continue + if not isinstance(metric, dict): + continue + append_expression(metric.get("expression")) + append_expression(metric.get("sqlExpression")) + append_expression(metric.get("metric_name")) + + columns_payload = dataset_payload.get("columns") or [] + if isinstance(columns_payload, list): + for column in columns_payload: + if not isinstance(column, dict): + continue + append_expression(column.get("sqlExpression")) + append_expression(column.get("expression")) + + return expressions + + # [/DEF:SupersetContextTemplatesMixin._collect_query_bearing_expressions:Function] + + # [DEF:SupersetContextTemplatesMixin._append_template_variable:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Append one deduplicated template-variable descriptor. + def _append_template_variable( + self, + discovered: List[Dict[str, Any]], + seen_variable_names: Set[str], + variable_name: str, + expression_source: str, + variable_kind: str, + is_required: bool, + default_value: Any, + ) -> None: + normalized_name = str(variable_name or "").strip() + if not normalized_name: + return + seen_key = normalized_name.lower() + if seen_key in seen_variable_names: + return + seen_variable_names.add(seen_key) + discovered.append( + { + "variable_name": normalized_name, + "expression_source": expression_source, + "variable_kind": variable_kind, + "is_required": is_required, + "default_value": default_value, + "mapping_status": "unmapped", + } + ) + + # [/DEF:SupersetContextTemplatesMixin._append_template_variable:Function] + + # [DEF:SupersetContextTemplatesMixin._extract_primary_jinja_identifier:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Extract a deterministic primary identifier from a Jinja expression without executing it. + def _extract_primary_jinja_identifier(self, expression: str) -> Optional[str]: + matched = re.match(r"([A-Za-z_][A-Za-z0-9_]*)", expression.strip()) + if matched is None: + return None + candidate = matched.group(1) + if candidate in { + "if", + "else", + "for", + "set", + "True", + "False", + "none", + "None", + }: + return None + return candidate + + # [/DEF:SupersetContextTemplatesMixin._extract_primary_jinja_identifier:Function] + + # [DEF:SupersetContextTemplatesMixin._normalize_default_literal:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Normalize literal default fragments from template helper calls into JSON-safe values. + def _normalize_default_literal(self, literal: Optional[str]) -> Any: + normalized_literal = str(literal or "").strip() + if not normalized_literal: + return None + if ( + normalized_literal.startswith("'") and normalized_literal.endswith("'") + ) or ( + normalized_literal.startswith('"') and normalized_literal.endswith('"') + ): + return normalized_literal[1:-1] + lowered = normalized_literal.lower() + if lowered in {"true", "false"}: + return lowered == "true" + if lowered in {"null", "none"}: + return None + try: + return int(normalized_literal) + except ValueError: + try: + return float(normalized_literal) + except ValueError: + return normalized_literal + + # [/DEF:SupersetContextTemplatesMixin._normalize_default_literal:Function] + + +# [/DEF:SupersetContextTemplatesMixin:Class] diff --git a/backend/src/services/git/__init__.py b/backend/src/services/git/__init__.py new file mode 100644 index 00000000..02781332 --- /dev/null +++ b/backend/src/services/git/__init__.py @@ -0,0 +1,66 @@ +# [DEF:GitServiceModule:Module] +# @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: git, service, decomposition, mixin, composition +# @PURPOSE: Composed GitService via multiple inheritance from domain-specific mixins. +# @RELATION: DEPENDS_ON -> [GitServiceBase] +# @RELATION: DEPENDS_ON -> [GitServiceBranchMixin] +# @RELATION: DEPENDS_ON -> [GitServiceSyncMixin] +# @RELATION: DEPENDS_ON -> [GitServiceStatusMixin] +# @RELATION: DEPENDS_ON -> [GitServiceMergeMixin] +# @RELATION: DEPENDS_ON -> [GitServiceUrlMixin] +# @RELATION: DEPENDS_ON -> [GitServiceGiteaMixin] +# @RELATION: DEPENDS_ON -> [GitServiceGithubMixin] +# @RELATION: DEPENDS_ON -> [GitServiceGitlabMixin] +# +# @RATIONALE: Decomposed from monolithic git_service.py (2101 lines) into +# domain-scoped mixins to satisfy INV_7 (module < 400 lines). The composed class +# preserves the original public API surface — all consumers continue to import +# `from src.services.git_service import GitService` without changes. +# @REJECTED: Keeping a single 2101-line file — violates fractal limit INV_7. + +from ._base import GitServiceBase +from ._branch import GitServiceBranchMixin +from ._sync import GitServiceSyncMixin +from ._status import GitServiceStatusMixin +from ._merge import GitServiceMergeMixin +from ._url import GitServiceUrlMixin +from ._gitea import GitServiceGiteaMixin +from ._remote_providers import GitServiceGithubMixin, GitServiceGitlabMixin + +__all__ = ["GitService"] + + +# [DEF:GitService:Class] +# @COMPLEXITY: 3 +# @PURPOSE: Composed GitService class providing all Git operations — init, clone, branches, commits, push, pull, +# merge, status, diff, history, and provider API operations (Gitea, GitHub, GitLab). +# @RELATION: INHERITS -> [GitServiceBase] +# @RELATION: INHERITS -> [GitServiceBranchMixin] +# @RELATION: INHERITS -> [GitServiceSyncMixin] +# @RELATION: INHERITS -> [GitServiceStatusMixin] +# @RELATION: INHERITS -> [GitServiceMergeMixin] +# @RELATION: INHERITS -> [GitServiceUrlMixin] +# @RELATION: INHERITS -> [GitServiceGiteaMixin] +# @RELATION: INHERITS -> [GitServiceGithubMixin] +# @RELATION: INHERITS -> [GitServiceGitlabMixin] +class GitService( + GitServiceGiteaMixin, + GitServiceGithubMixin, + GitServiceGitlabMixin, + GitServiceMergeMixin, + GitServiceSyncMixin, + GitServiceStatusMixin, + GitServiceBranchMixin, + GitServiceUrlMixin, + GitServiceBase, +): + """ + Composed GitService — all Git operations via domain-specific mixins. + + MRO ensures provider mixins and domain mixins resolve before the base class. + All consumers continue to use: from src.services.git_service import GitService + """ + pass +# [/DEF:GitService:Class] +# [/DEF:GitServiceModule:Module] diff --git a/backend/src/services/git/_base.py b/backend/src/services/git/_base.py new file mode 100644 index 00000000..a414725d --- /dev/null +++ b/backend/src/services/git/_base.py @@ -0,0 +1,319 @@ +# [DEF:GitServiceBase:Module] +# @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: git, service, repository, version_control, initialization +# @PURPOSE: Core GitService base class — initialization, path resolution, repo lifecycle (init/delete/get), and identity configuration. +# @RELATION: INHERITED_BY -> [GitService] +# @RELATION: DEPENDS_ON -> [SessionLocal] +# @RELATION: DEPENDS_ON -> [AppConfigRecord] +# @RELATION: DEPENDS_ON -> [GitRepository] + +import os +import re +import shutil +from pathlib import Path +from typing import Any, Dict, List, Optional +from git import Repo +from git.exc import InvalidGitRepositoryError, NoSuchPathError +from fastapi import HTTPException +from src.core.logger import logger, belief_scope +from src.models.git import GitRepository +from src.models.config import AppConfigRecord +from src.core.database import SessionLocal + + +# [DEF:GitServiceBase:Class] +# @COMPLEXITY: 3 +# @PURPOSE: Base class for GitService providing initialization, path resolution, repository lifecycle, and identity. +class GitServiceBase: + # [DEF:GitService_init:Function] + # @PURPOSE: Initializes the GitService with a base path for repositories. + # @PARAM: base_path (str) - Root directory for all Git clones. + # @PRE: base_path is a valid string path. + # @POST: GitService is initialized; base_path directory exists. + def __init__(self, base_path: str = "git_repos"): + with belief_scope("GitService.__init__"): + backend_root = Path(__file__).parents[3] + self.legacy_base_path = str((backend_root / "git_repos").resolve()) + self._uses_default_base_path = base_path == "git_repos" + self.base_path = self._resolve_base_path(base_path) + self._ensure_base_path_exists() + # [/DEF:GitService_init:Function] + + # [DEF:_ensure_base_path_exists:Function] + # @PURPOSE: Ensure the repositories root directory exists and is a directory. + # @PRE: self.base_path is resolved to filesystem path. + # @POST: self.base_path exists as directory or raises ValueError. + def _ensure_base_path_exists(self) -> None: + base = Path(self.base_path) + if base.exists() and not base.is_dir(): + raise ValueError(f"Git repositories base path is not a directory: {self.base_path}") + try: + base.mkdir(parents=True, exist_ok=True) + except (PermissionError, OSError) as e: + logger.warning( + f"[_ensure_base_path_exists][Coherence:Failed] Cannot create Git repositories base path: {self.base_path}. Error: {e}" + ) + raise ValueError(f"Cannot create Git repositories base path: {self.base_path}. {e}") + # [/DEF:_ensure_base_path_exists:Function] + + # [DEF:_resolve_base_path:Function] + # @PURPOSE: Resolve base repository directory from explicit argument or global storage settings. + # @PRE: base_path is a string path. + # @POST: Returns absolute path for Git repositories root. + # @RETURN: str + def _resolve_base_path(self, base_path: str) -> str: + backend_root = Path(__file__).parents[3] + fallback_path = str((backend_root / base_path).resolve()) + if base_path != "git_repos": + return fallback_path + try: + session = SessionLocal() + try: + config_row = session.query(AppConfigRecord).filter(AppConfigRecord.id == "global").first() + finally: + session.close() + payload = (config_row.payload if config_row and config_row.payload else {}) if config_row else {} + storage_cfg = payload.get("settings", {}).get("storage", {}) if isinstance(payload, dict) else {} + root_path = str(storage_cfg.get("root_path", "")).strip() + repo_path = str(storage_cfg.get("repo_path", "")).strip() + if not root_path: + return fallback_path + project_root = Path(__file__).parents[4] + root = Path(root_path) + if not root.is_absolute(): + root = (project_root / root).resolve() + repo_root = Path(repo_path) if repo_path else Path("repositorys") + if repo_root.is_absolute(): + return str(repo_root.resolve()) + return str((root / repo_root).resolve()) + except Exception as e: + logger.warning(f"[_resolve_base_path][Coherence:Failed] Falling back to default path: {e}") + return fallback_path + # [/DEF:_resolve_base_path:Function] + + # [DEF:_normalize_repo_key:Function] + # @PURPOSE: Convert user/dashboard-provided key to safe filesystem directory name. + # @PRE: repo_key can be None/empty. + # @POST: Returns normalized non-empty key. + # @RETURN: str + def _normalize_repo_key(self, repo_key: Optional[str]) -> str: + raw_key = str(repo_key or "").strip().lower() + normalized = re.sub(r"[^a-z0-9._-]+", "-", raw_key).strip("._-") + return normalized or "dashboard" + # [/DEF:_normalize_repo_key:Function] + + # [DEF:_update_repo_local_path:Function] + # @PURPOSE: Persist repository local_path in GitRepository table when record exists. + # @PRE: dashboard_id is valid integer. + # @POST: local_path is updated for existing record. + def _update_repo_local_path(self, dashboard_id: int, local_path: str) -> None: + try: + session = SessionLocal() + try: + db_repo = ( + session.query(GitRepository) + .filter(GitRepository.dashboard_id == int(dashboard_id)) + .first() + ) + if db_repo: + db_repo.local_path = local_path + session.commit() + finally: + session.close() + except Exception as e: + logger.warning(f"[_update_repo_local_path][Coherence:Failed] {e}") + # [/DEF:_update_repo_local_path:Function] + + # [DEF:_migrate_repo_directory:Function] + # @PURPOSE: Move legacy repository directory to target path and sync DB metadata. + # @PRE: source_path exists. + # @POST: Repository content available at target_path. + # @RETURN: str + def _migrate_repo_directory(self, dashboard_id: int, source_path: str, target_path: str) -> str: + source_abs = os.path.abspath(source_path) + target_abs = os.path.abspath(target_path) + if source_abs == target_abs: + return source_abs + if os.path.exists(target_abs): + logger.warning(f"[_migrate_repo_directory][Action] Target already exists, keeping source path: {target_abs}") + return source_abs + Path(target_abs).parent.mkdir(parents=True, exist_ok=True) + try: + os.replace(source_abs, target_abs) + except OSError: + shutil.move(source_abs, target_abs) + self._update_repo_local_path(dashboard_id, target_abs) + logger.info( + f"[_migrate_repo_directory][Coherence:OK] Repository migrated for dashboard {dashboard_id}: {source_abs} -> {target_abs}" + ) + return target_abs + # [/DEF:_migrate_repo_directory:Function] + + # [DEF:_get_repo_path:Function] + # @PURPOSE: Resolves the local filesystem path for a dashboard's repository. + # @PARAM: dashboard_id (int) + # @PARAM: repo_key (Optional[str]) - Slug-like key used when DB local_path is absent. + # @PRE: dashboard_id is an integer. + # @POST: Returns DB-local_path when present, otherwise base_path/. + # @RETURN: str + def _get_repo_path(self, dashboard_id: int, repo_key: Optional[str] = None) -> str: + with belief_scope("GitService._get_repo_path"): + if dashboard_id is None: + raise ValueError("dashboard_id cannot be None") + self._ensure_base_path_exists() + fallback_key = repo_key if repo_key is not None else str(dashboard_id) + normalized_key = self._normalize_repo_key(fallback_key) + target_path = os.path.join(self.base_path, normalized_key) + if not self._uses_default_base_path: + return target_path + try: + session = SessionLocal() + try: + db_repo = ( + session.query(GitRepository) + .filter(GitRepository.dashboard_id == int(dashboard_id)) + .first() + ) + finally: + session.close() + if db_repo and db_repo.local_path: + db_path = os.path.abspath(db_repo.local_path) + if os.path.exists(db_path): + if ( + os.path.abspath(self.base_path) != os.path.abspath(self.legacy_base_path) + and db_path.startswith(os.path.abspath(self.legacy_base_path) + os.sep) + ): + return self._migrate_repo_directory(dashboard_id, db_path, target_path) + return db_path + except Exception as e: + logger.warning(f"[_get_repo_path][Coherence:Failed] Could not resolve local_path from DB: {e}") + legacy_id_path = os.path.join(self.legacy_base_path, str(dashboard_id)) + if os.path.exists(legacy_id_path) and not os.path.exists(target_path): + return self._migrate_repo_directory(dashboard_id, legacy_id_path, target_path) + if os.path.exists(target_path): + self._update_repo_local_path(dashboard_id, target_path) + return target_path + # [/DEF:_get_repo_path:Function] + + # [DEF:init_repo:Function] + # @PURPOSE: Initialize or clone a repository for a dashboard. + # @PARAM: dashboard_id (int), remote_url (str), pat (str), repo_key (Optional[str]). + # @PRE: dashboard_id is int, remote_url is valid Git URL, pat is provided. + # @POST: Repository is cloned or opened at the local path. + # @RETURN: Repo + def init_repo(self, dashboard_id: int, remote_url: str, pat: str, repo_key: Optional[str] = None) -> Repo: + with belief_scope("GitService.init_repo"): + self._ensure_base_path_exists() + repo_path = self._get_repo_path(dashboard_id, repo_key=repo_key or str(dashboard_id)) + Path(repo_path).parent.mkdir(parents=True, exist_ok=True) + if pat and "://" in remote_url: + proto, rest = remote_url.split("://", 1) + auth_url = f"{proto}://oauth2:{pat}@{rest}" + else: + auth_url = remote_url + if os.path.exists(repo_path): + logger.info(f"[init_repo][Action] Opening existing repo at {repo_path}") + try: + repo = Repo(repo_path) + except (InvalidGitRepositoryError, NoSuchPathError): + logger.warning(f"[init_repo][Action] Existing path is not a Git repository, recreating: {repo_path}") + stale_path = Path(repo_path) + if stale_path.exists(): + shutil.rmtree(stale_path, ignore_errors=True) + if stale_path.exists(): + try: + stale_path.unlink() + except Exception: + pass + repo = Repo.clone_from(auth_url, repo_path) + self._ensure_gitflow_branches(repo, dashboard_id) + return repo + logger.info(f"[init_repo][Action] Cloning {remote_url} to {repo_path}") + repo = Repo.clone_from(auth_url, repo_path) + self._ensure_gitflow_branches(repo, dashboard_id) + return repo + # [/DEF:init_repo:Function] + + # [DEF:delete_repo:Function] + # @PURPOSE: Remove local repository and DB binding for a dashboard. + # @PRE: dashboard_id is a valid integer. + # @POST: Local path is deleted when present and GitRepository row is removed. + def delete_repo(self, dashboard_id: int) -> None: + with belief_scope("GitService.delete_repo"): + repo_path = self._get_repo_path(dashboard_id) + removed_files = False + if os.path.exists(repo_path): + if os.path.isdir(repo_path): + shutil.rmtree(repo_path) + else: + os.remove(repo_path) + removed_files = True + session = SessionLocal() + try: + db_repo = ( + session.query(GitRepository) + .filter(GitRepository.dashboard_id == int(dashboard_id)) + .first() + ) + if db_repo: + session.delete(db_repo) + session.commit() + return + if removed_files: + return + raise HTTPException( + status_code=404, + detail=f"Repository for dashboard {dashboard_id} not found", + ) + except HTTPException: + session.rollback() + raise + except Exception as e: + session.rollback() + logger.error(f"[delete_repo][Coherence:Failed] Failed to delete repository for dashboard {dashboard_id}: {e}") + raise HTTPException(status_code=500, detail=f"Failed to delete repository: {str(e)}") + finally: + session.close() + # [/DEF:delete_repo:Function] + + # [DEF:get_repo:Function] + # @PURPOSE: Get Repo object for a dashboard. + # @PRE: Repository must exist on disk for the given dashboard_id. + # @POST: Returns a GitPython Repo instance for the dashboard. + # @RETURN: Repo + def get_repo(self, dashboard_id: int) -> Repo: + with belief_scope("GitService.get_repo"): + repo_path = self._get_repo_path(dashboard_id) + if not os.path.exists(repo_path): + logger.error(f"[get_repo][Coherence:Failed] Repository for dashboard {dashboard_id} does not exist") + raise HTTPException(status_code=404, detail=f"Repository for dashboard {dashboard_id} not found") + try: + return Repo(repo_path) + except Exception as e: + logger.error(f"[get_repo][Coherence:Failed] Failed to open repository at {repo_path}: {e}") + raise HTTPException(status_code=500, detail="Failed to open local Git repository") + # [/DEF:get_repo:Function] + + # [DEF:configure_identity:Function] + # @PURPOSE: Configure repository-local Git committer identity for user-scoped operations. + # @PRE: dashboard_id repository exists; git_username/git_email may be empty. + # @POST: Repository config has user.name and user.email when both identity values are provided. + def configure_identity(self, dashboard_id: int, git_username: Optional[str], git_email: Optional[str]) -> None: + with belief_scope("GitService.configure_identity"): + normalized_username = str(git_username or "").strip() + normalized_email = str(git_email or "").strip() + if not normalized_username or not normalized_email: + return + repo = self.get_repo(dashboard_id) + try: + with repo.config_writer(config_level="repository") as config_writer: + config_writer.set_value("user", "name", normalized_username) + config_writer.set_value("user", "email", normalized_email) + logger.info("[configure_identity][Action] Applied repository-local git identity for dashboard %s", dashboard_id) + except Exception as e: + logger.error(f"[configure_identity][Coherence:Failed] Failed to configure git identity: {e}") + raise HTTPException(status_code=500, detail=f"Failed to configure git identity: {str(e)}") + # [/DEF:configure_identity:Function] +# [/DEF:GitServiceBase:Class] +# [/DEF:GitServiceBase:Module] diff --git a/backend/src/services/git/_branch.py b/backend/src/services/git/_branch.py new file mode 100644 index 00000000..2122769d --- /dev/null +++ b/backend/src/services/git/_branch.py @@ -0,0 +1,192 @@ +# [DEF:GitServiceBranchMixin:Module] +# @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: git, branches, commits, checkout, gitflow +# @PURPOSE: Branch and commit operations for GitService — gitflow branches, list/create/checkout branches, commit changes. +# @RELATION: USED_BY -> [GitService] + +import os +from typing import Any, Dict, List, Optional +from datetime import datetime +from git import Repo +from git.exc import GitCommandError +from fastapi import HTTPException +from src.core.logger import logger, belief_scope + + +# [DEF:GitServiceBranchMixin:Class] +# @COMPLEXITY: 3 +# @PURPOSE: Mixin providing branch and commit operations for GitService. +class GitServiceBranchMixin: + # [DEF:_ensure_gitflow_branches:Function] + # @PURPOSE: Ensure standard GitFlow branches (main/dev/preprod) exist locally and on origin. + # @PRE: repo is a valid GitPython Repo instance. + # @POST: main, dev, preprod are available in local repository and pushed to origin when available. + def _ensure_gitflow_branches(self, repo: Repo, dashboard_id: int) -> None: + with belief_scope("GitService._ensure_gitflow_branches"): + required_branches = ["main", "dev", "preprod"] + local_heads = {head.name: head for head in getattr(repo, "heads", [])} + base_commit = None + try: + base_commit = repo.head.commit + except Exception: + base_commit = None + if "main" in local_heads: + base_commit = local_heads["main"].commit + if base_commit is None: + logger.warning( + f"[_ensure_gitflow_branches][Action] Skipping branch bootstrap for dashboard {dashboard_id}: repository has no commits" + ) + return + if "main" not in local_heads: + local_heads["main"] = repo.create_head("main", base_commit) + logger.info(f"[_ensure_gitflow_branches][Action] Created local branch main for dashboard {dashboard_id}") + for branch_name in ("dev", "preprod"): + if branch_name in local_heads: + continue + local_heads[branch_name] = repo.create_head(branch_name, local_heads["main"].commit) + logger.info( + f"[_ensure_gitflow_branches][Action] Created local branch {branch_name} for dashboard {dashboard_id}" + ) + try: + origin = repo.remote(name="origin") + except ValueError: + logger.info( + f"[_ensure_gitflow_branches][Action] Remote origin is not configured for dashboard {dashboard_id}; skipping remote branch creation" + ) + return + remote_branch_names = set() + try: + origin.fetch() + for ref in origin.refs: + remote_head = getattr(ref, "remote_head", None) + if remote_head: + remote_branch_names.add(str(remote_head)) + except Exception as e: + logger.warning(f"[_ensure_gitflow_branches][Action] Failed to fetch origin refs: {e}") + for branch_name in required_branches: + if branch_name in remote_branch_names: + continue + try: + origin.push(refspec=f"{branch_name}:{branch_name}") + logger.info( + f"[_ensure_gitflow_branches][Action] Pushed branch {branch_name} to origin for dashboard {dashboard_id}" + ) + except Exception as e: + logger.error(f"[_ensure_gitflow_branches][Coherence:Failed] Failed to push branch {branch_name} to origin: {e}") + raise HTTPException( + status_code=500, + detail=f"Failed to create default branch '{branch_name}' on remote: {str(e)}", + ) + try: + repo.git.checkout("dev") + logger.info(f"[_ensure_gitflow_branches][Action] Checked out default branch dev for dashboard {dashboard_id}") + except Exception as e: + logger.warning(f"[_ensure_gitflow_branches][Action] Could not checkout dev branch for dashboard {dashboard_id}: {e}") + # [/DEF:_ensure_gitflow_branches:Function] + + # [DEF:list_branches:Function] + # @PURPOSE: List all branches for a dashboard's repository. + # @PRE: Repository for dashboard_id exists. + # @POST: Returns a list of branch metadata dictionaries. + # @RETURN: List[dict] + def list_branches(self, dashboard_id: int) -> List[dict]: + with belief_scope("GitService.list_branches"): + repo = self.get_repo(dashboard_id) + logger.info(f"[list_branches][Action] Listing branches for {dashboard_id}. Refs: {repo.refs}") + branches = [] + for ref in repo.refs: + try: + name = ref.name.replace('refs/heads/', '').replace('refs/remotes/origin/', '') + if any(b['name'] == name for b in branches): + continue + branches.append({ + "name": name, + "commit_hash": ref.commit.hexsha if hasattr(ref, 'commit') else "0000000", + "is_remote": ref.is_remote() if hasattr(ref, 'is_remote') else False, + "last_updated": datetime.fromtimestamp(ref.commit.committed_date) if hasattr(ref, 'commit') else datetime.utcnow() + }) + except Exception as e: + logger.warning(f"[list_branches][Action] Skipping ref {ref}: {e}") + try: + active_name = repo.active_branch.name + if not any(b['name'] == active_name for b in branches): + branches.append({ + "name": active_name, "commit_hash": "0000000", + "is_remote": False, "last_updated": datetime.utcnow() + }) + except Exception as e: + logger.warning(f"[list_branches][Action] Could not determine active branch: {e}") + if not branches: + branches.append({ + "name": "dev", "commit_hash": "0000000", + "is_remote": False, "last_updated": datetime.utcnow() + }) + return branches + # [/DEF:list_branches:Function] + + # [DEF:create_branch:Function] + # @PURPOSE: Create a new branch from an existing one. + # @PARAM: name (str) - New branch name. + # @PARAM: from_branch (str) - Source branch. + # @PRE: Repository exists; name is valid; from_branch exists or repo is empty. + # @POST: A new branch is created in the repository. + def create_branch(self, dashboard_id: int, name: str, from_branch: str = "main"): + with belief_scope("GitService.create_branch"): + repo = self.get_repo(dashboard_id) + logger.info(f"[create_branch][Action] Creating branch {name} from {from_branch}") + if not repo.heads and not repo.remotes: + logger.warning("[create_branch][Action] Repository is empty. Creating initial commit to enable branching.") + readme_path = os.path.join(repo.working_dir, "README.md") + if not os.path.exists(readme_path): + with open(readme_path, "w") as f: + f.write(f"# Dashboard {dashboard_id}\nGit repository for Superset dashboard integration.") + repo.index.add(["README.md"]) + repo.index.commit("Initial commit") + try: + repo.commit(from_branch) + except Exception: + logger.warning(f"[create_branch][Action] Source branch {from_branch} not found, using HEAD") + from_branch = repo.head + try: + new_branch = repo.create_head(name, from_branch) + return new_branch + except Exception as e: + logger.error(f"[create_branch][Coherence:Failed] {e}") + raise + # [/DEF:create_branch:Function] + + # [DEF:checkout_branch:Function] + # @PURPOSE: Switch to a specific branch. + # @PRE: Repository exists and the specified branch name exists. + # @POST: The repository working directory is updated to the specified branch. + def checkout_branch(self, dashboard_id: int, name: str): + with belief_scope("GitService.checkout_branch"): + repo = self.get_repo(dashboard_id) + logger.info(f"[checkout_branch][Action] Checking out branch {name}") + repo.git.checkout(name) + # [/DEF:checkout_branch:Function] + + # [DEF:commit_changes:Function] + # @PURPOSE: Stage and commit changes. + # @PARAM: message (str) - Commit message. + # @PARAM: files (List[str]) - Optional list of specific files to stage. + # @PRE: Repository exists and has changes (dirty) or files are specified. + # @POST: Changes are staged and a new commit is created. + def commit_changes(self, dashboard_id: int, message: str, files: List[str] = None): + with belief_scope("GitService.commit_changes"): + repo = self.get_repo(dashboard_id) + if not repo.is_dirty(untracked_files=True) and not files: + logger.info(f"[commit_changes][Action] No changes to commit for dashboard {dashboard_id}") + return + if files: + logger.info(f"[commit_changes][Action] Staging files: {files}") + repo.index.add(files) + else: + logger.info("[commit_changes][Action] Staging all changes") + repo.git.add(A=True) + repo.index.commit(message) + logger.info(f"[commit_changes][Coherence:OK] Committed changes with message: {message}") + # [/DEF:commit_changes:Function] +# [/DEF:GitServiceBranchMixin:Class] +# [/DEF:GitServiceBranchMixin:Module] diff --git a/backend/src/services/git/_gitea.py b/backend/src/services/git/_gitea.py new file mode 100644 index 00000000..04fb90d3 --- /dev/null +++ b/backend/src/services/git/_gitea.py @@ -0,0 +1,262 @@ +# [DEF:GitServiceGiteaMixin:Module] +# @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: git, gitea, api, provider, connection_test +# @PURPOSE: Gitea API operations for GitService — connection testing, repository CRUD, and pull request creation. +# @RELATION: USED_BY -> [GitService] + +import httpx +from typing import Any, Dict, List, Optional +from urllib.parse import quote +from fastapi import HTTPException +from src.core.logger import logger, belief_scope +from src.models.git import GitProvider + + +# [DEF:GitServiceGiteaMixin:Class] +# @COMPLEXITY: 3 +# @PURPOSE: Mixin providing Gitea API operations for GitService. +class GitServiceGiteaMixin: + # [DEF:test_connection:Function] + # @PURPOSE: Test connection to Git provider using PAT. + # @PARAM: provider (GitProvider), url (str), pat (str) + # @PRE: provider is valid; url is a valid HTTP(S) URL; pat is provided. + # @POST: Returns True if connection to the provider's API succeeds. + # @RETURN: bool + async def test_connection(self, provider: GitProvider, url: str, pat: str) -> bool: + with belief_scope("GitService.test_connection"): + if ".local" in url or "localhost" in url: + logger.info("[test_connection][Action] Local/Offline mode detected for URL") + return True + if not url.startswith(('http://', 'https://')): + logger.error(f"[test_connection][Coherence:Failed] Invalid URL protocol: {url}") + return False + if not pat or not pat.strip(): + logger.error("[test_connection][Coherence:Failed] Git PAT is missing or empty") + return False + pat = pat.strip() + try: + async with httpx.AsyncClient() as client: + if provider == GitProvider.GITHUB: + headers = {"Authorization": f"token {pat}"} + api_url = "https://api.github.com/user" if "github.com" in url else f"{url.rstrip('/')}/api/v3/user" + resp = await client.get(api_url, headers=headers) + elif provider == GitProvider.GITLAB: + headers = {"PRIVATE-TOKEN": pat} + api_url = f"{url.rstrip('/')}/api/v4/user" + resp = await client.get(api_url, headers=headers) + elif provider == GitProvider.GITEA: + headers = {"Authorization": f"token {pat}"} + api_url = f"{url.rstrip('/')}/api/v1/user" + resp = await client.get(api_url, headers=headers) + else: + return False + if resp.status_code != 200: + logger.error(f"[test_connection][Coherence:Failed] Git connection test failed for {provider} at {api_url}. Status: {resp.status_code}") + return resp.status_code == 200 + except Exception as e: + logger.error(f"[test_connection][Coherence:Failed] Error testing git connection: {e}") + return False + # [/DEF:test_connection:Function] + + # [DEF:_gitea_headers:Function] + # @PURPOSE: Build Gitea API authorization headers. + # @PRE: pat is provided. + # @POST: Returns headers with token auth. + # @RETURN: Dict[str, str] + def _gitea_headers(self, pat: str) -> Dict[str, str]: + token = (pat or "").strip() + if not token: + raise HTTPException(status_code=400, detail="Git PAT is required for Gitea operations") + return { + "Authorization": f"token {token}", + "Content-Type": "application/json", + "Accept": "application/json", + } + # [/DEF:_gitea_headers:Function] + + # [DEF:_gitea_request:Function] + # @PURPOSE: Execute HTTP request against Gitea API with stable error mapping. + # @PRE: method and endpoint are valid. + # @POST: Returns decoded JSON payload. + # @RETURN: Any + async def _gitea_request( + self, method: str, server_url: str, pat: str, endpoint: str, + payload: Optional[Dict[str, Any]] = None, + ) -> Any: + base_url = self._normalize_git_server_url(server_url) + url = f"{base_url}/api/v1{endpoint}" + headers = self._gitea_headers(pat) + try: + async with httpx.AsyncClient(timeout=20.0) as client: + response = await client.request(method=method, url=url, headers=headers, json=payload) + except Exception as e: + logger.error(f"[gitea_request][Coherence:Failed] Network error: {e}") + raise HTTPException(status_code=503, detail=f"Gitea API is unavailable: {str(e)}") + if response.status_code >= 400: + detail = response.text + try: + parsed = response.json() + detail = parsed.get("message") or parsed.get("error") or detail + except Exception: + pass + logger.error(f"[gitea_request][Coherence:Failed] method={method} endpoint={endpoint} status={response.status_code} detail={detail}") + raise HTTPException(status_code=response.status_code, detail=f"Gitea API error: {detail}") + if response.status_code == 204: + return None + return response.json() + # [/DEF:_gitea_request:Function] + + # [DEF:get_gitea_current_user:Function] + # @PURPOSE: Resolve current Gitea user for PAT. + # @PRE: server_url and pat are valid. + # @POST: Returns current username. + # @RETURN: str + async def get_gitea_current_user(self, server_url: str, pat: str) -> str: + payload = await self._gitea_request("GET", server_url, pat, "/user") + username = payload.get("login") or payload.get("username") + if not username: + raise HTTPException(status_code=500, detail="Failed to resolve Gitea username") + return str(username) + # [/DEF:get_gitea_current_user:Function] + + # [DEF:list_gitea_repositories:Function] + # @PURPOSE: List repositories visible to authenticated Gitea user. + # @PRE: server_url and pat are valid. + # @POST: Returns repository list from Gitea. + # @RETURN: List[dict] + async def list_gitea_repositories(self, server_url: str, pat: str) -> List[dict]: + payload = await self._gitea_request("GET", server_url, pat, "/user/repos?limit=100&page=1") + if not isinstance(payload, list): + return [] + return payload + # [/DEF:list_gitea_repositories:Function] + + # [DEF:create_gitea_repository:Function] + # @PURPOSE: Create repository in Gitea for authenticated user. + # @PRE: name is non-empty and PAT has repo creation permission. + # @POST: Returns created repository payload. + # @RETURN: dict + async def create_gitea_repository( + self, server_url: str, pat: str, name: str, private: bool = True, + description: Optional[str] = None, auto_init: bool = True, default_branch: Optional[str] = "main", + ) -> Dict[str, Any]: + payload: Dict[str, Any] = {"name": name, "private": bool(private), "auto_init": bool(auto_init)} + if description: + payload["description"] = description + if default_branch: + payload["default_branch"] = default_branch + created = await self._gitea_request("POST", server_url, pat, "/user/repos", payload=payload) + if not isinstance(created, dict): + raise HTTPException(status_code=500, detail="Unexpected Gitea response while creating repository") + return created + # [/DEF:create_gitea_repository:Function] + + # [DEF:delete_gitea_repository:Function] + # @PURPOSE: Delete repository in Gitea. + # @PRE: owner and repo_name are non-empty. + # @POST: Repository deleted on Gitea server. + async def delete_gitea_repository(self, server_url: str, pat: str, owner: str, repo_name: str) -> None: + if not owner or not repo_name: + raise HTTPException(status_code=400, detail="owner and repo_name are required") + await self._gitea_request("DELETE", server_url, pat, f"/repos/{owner}/{repo_name}") + # [/DEF:delete_gitea_repository:Function] + + # [DEF:_gitea_branch_exists:Function] + # @PURPOSE: Check whether a branch exists in Gitea repository. + # @PRE: owner/repo/branch are non-empty. + # @POST: Returns True when branch exists, False when 404. + # @RETURN: bool + async def _gitea_branch_exists(self, server_url: str, pat: str, owner: str, repo: str, branch: str) -> bool: + if not owner or not repo or not branch: + return False + endpoint = f"/repos/{owner}/{repo}/branches/{quote(branch, safe='')}" + try: + await self._gitea_request("GET", server_url, pat, endpoint) + return True + except HTTPException as exc: + if exc.status_code == 404: + return False + raise + # [/DEF:_gitea_branch_exists:Function] + + # [DEF:_build_gitea_pr_404_detail:Function] + # @PURPOSE: Build actionable error detail for Gitea PR 404 responses. + # @PRE: owner/repo/from_branch/to_branch are provided. + # @POST: Returns specific branch-missing message when detected. + # @RETURN: Optional[str] + async def _build_gitea_pr_404_detail( + self, server_url: str, pat: str, owner: str, repo: str, from_branch: str, to_branch: str, + ) -> Optional[str]: + source_exists = await self._gitea_branch_exists( + server_url=server_url, pat=pat, owner=owner, repo=repo, branch=from_branch, + ) + target_exists = await self._gitea_branch_exists( + server_url=server_url, pat=pat, owner=owner, repo=repo, branch=to_branch, + ) + if not source_exists: + return f"Gitea branch not found: source branch '{from_branch}' in {owner}/{repo}" + if not target_exists: + return f"Gitea branch not found: target branch '{to_branch}' in {owner}/{repo}" + return None + # [/DEF:_build_gitea_pr_404_detail:Function] + + # [DEF:create_gitea_pull_request:Function] + # @PURPOSE: Create pull request in Gitea. + # @PRE: Config and remote URL are valid. + # @POST: Returns normalized PR metadata. + # @RETURN: Dict[str, Any] + async def create_gitea_pull_request( + self, server_url: str, pat: str, remote_url: str, from_branch: str, to_branch: str, + title: str, description: Optional[str] = None, + ) -> Dict[str, Any]: + identity = self._parse_remote_repo_identity(remote_url) + req_payload = {"title": title, "head": from_branch, "base": to_branch, "body": description or ""} + endpoint = f"/repos/{identity['owner']}/{identity['repo']}/pulls" + active_server_url = server_url + try: + data = await self._gitea_request("POST", active_server_url, pat, endpoint, payload=req_payload) + except HTTPException as exc: + fallback_url = self._derive_server_url_from_remote(remote_url) + normalized_primary = self._normalize_git_server_url(server_url) + should_retry_with_fallback = ( + exc.status_code == 404 and fallback_url and fallback_url != normalized_primary + ) + if should_retry_with_fallback: + logger.warning( + "[create_gitea_pull_request][Action] Primary Gitea URL not found, retrying with remote host: %s", + fallback_url, + ) + active_server_url = fallback_url + try: + data = await self._gitea_request("POST", active_server_url, pat, endpoint, payload=req_payload) + except HTTPException as retry_exc: + if retry_exc.status_code == 404: + branch_detail = await self._build_gitea_pr_404_detail( + server_url=active_server_url, pat=pat, + owner=identity["owner"], repo=identity["repo"], + from_branch=from_branch, to_branch=to_branch, + ) + if branch_detail: + raise HTTPException(status_code=400, detail=branch_detail) + raise + else: + if exc.status_code == 404: + branch_detail = await self._build_gitea_pr_404_detail( + server_url=active_server_url, pat=pat, + owner=identity["owner"], repo=identity["repo"], + from_branch=from_branch, to_branch=to_branch, + ) + if branch_detail: + raise HTTPException(status_code=400, detail=branch_detail) + raise + if not isinstance(data, dict): + raise HTTPException(status_code=500, detail="Unexpected Gitea response while creating pull request") + return { + "id": data.get("number") or data.get("id"), + "url": data.get("html_url") or data.get("url"), + "status": data.get("state") or "open", + } + # [/DEF:create_gitea_pull_request:Function] +# [/DEF:GitServiceGiteaMixin:Class] +# [/DEF:GitServiceGiteaMixin:Module] diff --git a/backend/src/services/git/_merge.py b/backend/src/services/git/_merge.py new file mode 100644 index 00000000..abc7379a --- /dev/null +++ b/backend/src/services/git/_merge.py @@ -0,0 +1,278 @@ +# [DEF:GitServiceMergeMixin:Module] +# @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: git, merge, conflicts, resolution, promote +# @PURPOSE: Merge operations for GitService — conflict detection, resolution, abort, continue, and direct promote. +# @RELATION: USED_BY -> [GitService] + +import os +from pathlib import Path +from typing import Any, Dict, List, Optional +from git import Repo +from git.exc import GitCommandError +from git.objects.blob import Blob +from fastapi import HTTPException +from src.core.logger import logger, belief_scope + + +# [DEF:GitServiceMergeMixin:Class] +# @COMPLEXITY: 3 +# @PURPOSE: Mixin providing merge operations for GitService. +class GitServiceMergeMixin: + # [DEF:_read_blob_text:Function] + # @PURPOSE: Read text from a Git blob. + def _read_blob_text(self, blob: Blob) -> str: + with belief_scope("GitService._read_blob_text"): + if blob is None: + return "" + try: + return blob.data_stream.read().decode("utf-8", errors="replace") + except Exception: + return "" + # [/DEF:_read_blob_text:Function] + + # [DEF:_get_unmerged_file_paths:Function] + # @PURPOSE: List files with merge conflicts. + def _get_unmerged_file_paths(self, repo: Repo) -> List[str]: + with belief_scope("GitService._get_unmerged_file_paths"): + try: + return sorted(list(repo.index.unmerged_blobs().keys())) + except Exception: + return [] + # [/DEF:_get_unmerged_file_paths:Function] + + # [DEF:_build_unfinished_merge_payload:Function] + # @PURPOSE: Build payload for unfinished merge state. + def _build_unfinished_merge_payload(self, repo: Repo) -> Dict[str, Any]: + with belief_scope("GitService._build_unfinished_merge_payload"): + merge_head_path = os.path.join(repo.git_dir, "MERGE_HEAD") + merge_head_value = "" + merge_msg_preview = "" + current_branch = "unknown" + try: + merge_head_value = Path(merge_head_path).read_text(encoding="utf-8").strip() + except Exception: + merge_head_value = "" + try: + merge_msg_path = os.path.join(repo.git_dir, "MERGE_MSG") + if os.path.exists(merge_msg_path): + merge_msg_preview = ( + Path(merge_msg_path).read_text(encoding="utf-8").strip().splitlines()[:1] or [""] + )[0] + except Exception: + merge_msg_preview = "" + try: + current_branch = repo.active_branch.name + except Exception: + current_branch = "detached_or_unknown" + conflicts_count = len(self._get_unmerged_file_paths(repo)) + return { + "error_code": "GIT_UNFINISHED_MERGE", + "message": ( + "\u0412 \u0440\u0435\u043f\u043e\u0437\u0438\u0442\u043e\u0440\u0438\u0438 \u0435\u0441\u0442\u044c \u043d\u0435\u0437\u0430\u0432\u0435\u0440\u0448\u0451\u043d\u043d\u043e\u0435 \u0441\u043b\u0438\u044f\u043d\u0438\u0435. " + "\u0417\u0430\u0432\u0435\u0440\u0448\u0438\u0442\u0435 \u0438\u043b\u0438 \u043e\u0442\u043c\u0435\u043d\u0438\u0442\u0435 \u0441\u043b\u0438\u044f\u043d\u0438\u0435 \u0432\u0440\u0443\u0447\u043d\u0443\u044e." + ), + "repository_path": repo.working_tree_dir, + "git_dir": repo.git_dir, + "current_branch": current_branch, + "merge_head": merge_head_value, + "merge_message_preview": merge_msg_preview, + "conflicts_count": conflicts_count, + "next_steps": [ + "\u041e\u0442\u043a\u0440\u043e\u0439\u0442\u0435 \u043b\u043e\u043a\u0430\u043b\u044c\u043d\u044b\u0439 \u0440\u0435\u043f\u043e\u0437\u0438\u0442\u043e\u0440\u0438\u0439 \u043f\u043e \u043f\u0443\u0442\u0438 repository_path", + "\u041f\u0440\u043e\u0432\u0435\u0440\u044c\u0442\u0435 \u0441\u043e\u0441\u0442\u043e\u044f\u043d\u0438\u0435: git status", + "\u0420\u0430\u0437\u0440\u0435\u0448\u0438\u0442\u0435 \u043a\u043e\u043d\u0444\u043b\u0438\u043a\u0442\u044b \u0438 \u0432\u044b\u043f\u043e\u043b\u043d\u0438\u0442\u0435 commit, \u043b\u0438\u0431\u043e \u043e\u0442\u043c\u0435\u043d\u0438\u0442\u0435: git merge --abort", + "\u041f\u043e\u0441\u043b\u0435 \u0437\u0430\u0432\u0435\u0440\u0448\u0435\u043d\u0438\u044f/\u043e\u0442\u043c\u0435\u043d\u044b \u0441\u043b\u0438\u044f\u043d\u0438\u044f \u043f\u043e\u0432\u0442\u043e\u0440\u0438\u0442\u0435 Pull \u0438\u0437 \u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441\u0430", + ], + "manual_commands": ["git status", "git add ", 'git commit -m "resolve merge conflicts"', "git merge --abort"], + } + # [/DEF:_build_unfinished_merge_payload:Function] + + # [DEF:get_merge_status:Function] + # @PURPOSE: Get current merge status for a dashboard repository. + def get_merge_status(self, dashboard_id: int) -> Dict[str, Any]: + with belief_scope("GitService.get_merge_status"): + repo = self.get_repo(dashboard_id) + merge_head_path = os.path.join(repo.git_dir, "MERGE_HEAD") + if not os.path.exists(merge_head_path): + current_branch = "unknown" + try: + current_branch = repo.active_branch.name + except Exception: + current_branch = "detached_or_unknown" + return { + "has_unfinished_merge": False, + "repository_path": repo.working_tree_dir, + "git_dir": repo.git_dir, + "current_branch": current_branch, + "merge_head": None, + "merge_message_preview": None, + "conflicts_count": 0, + } + payload = self._build_unfinished_merge_payload(repo) + return { + "has_unfinished_merge": True, + "repository_path": payload["repository_path"], + "git_dir": payload["git_dir"], + "current_branch": payload["current_branch"], + "merge_head": payload["merge_head"], + "merge_message_preview": payload["merge_message_preview"], + "conflicts_count": int(payload.get("conflicts_count") or 0), + } + # [/DEF:get_merge_status:Function] + + # [DEF:get_merge_conflicts:Function] + # @PURPOSE: List all files with conflicts and their contents. + def get_merge_conflicts(self, dashboard_id: int) -> List[Dict[str, Any]]: + with belief_scope("GitService.get_merge_conflicts"): + repo = self.get_repo(dashboard_id) + conflicts = [] + unmerged = repo.index.unmerged_blobs() + for file_path, stages in unmerged.items(): + mine_blob = None + theirs_blob = None + for stage, blob in stages: + if stage == 2: + mine_blob = blob + elif stage == 3: + theirs_blob = blob + conflicts.append({ + "file_path": file_path, + "mine": self._read_blob_text(mine_blob) if mine_blob else "", + "theirs": self._read_blob_text(theirs_blob) if theirs_blob else "", + }) + return sorted(conflicts, key=lambda item: item["file_path"]) + # [/DEF:get_merge_conflicts:Function] + + # [DEF:resolve_merge_conflicts:Function] + # @PURPOSE: Resolve conflicts using specified strategy. + def resolve_merge_conflicts(self, dashboard_id: int, resolutions: List[Dict[str, Any]]) -> List[str]: + with belief_scope("GitService.resolve_merge_conflicts"): + repo = self.get_repo(dashboard_id) + resolved_files: List[str] = [] + repo_root = os.path.abspath(str(repo.working_tree_dir or "")) + if not repo_root: + raise HTTPException(status_code=500, detail="Repository working tree directory is unavailable") + for item in resolutions or []: + file_path = str(item.get("file_path") or "").strip() + strategy = str(item.get("resolution") or "").strip().lower() + content = item.get("content") + if not file_path: + raise HTTPException(status_code=400, detail="resolution.file_path is required") + if strategy not in {"mine", "theirs", "manual"}: + raise HTTPException(status_code=400, detail=f"Unsupported resolution strategy: {strategy}") + if strategy == "mine": + repo.git.checkout("--ours", "--", file_path) + elif strategy == "theirs": + repo.git.checkout("--theirs", "--", file_path) + else: + abs_target = os.path.abspath(os.path.join(repo_root, file_path)) + if abs_target != repo_root and not abs_target.startswith(repo_root + os.sep): + raise HTTPException(status_code=400, detail=f"Invalid conflict file path: {file_path}") + os.makedirs(os.path.dirname(abs_target), exist_ok=True) + with open(abs_target, "w", encoding="utf-8") as file_obj: + file_obj.write(str(content or "")) + repo.git.add(file_path) + resolved_files.append(file_path) + return resolved_files + # [/DEF:resolve_merge_conflicts:Function] + + # [DEF:abort_merge:Function] + # @PURPOSE: Abort ongoing merge. + def abort_merge(self, dashboard_id: int) -> Dict[str, Any]: + with belief_scope("GitService.abort_merge"): + repo = self.get_repo(dashboard_id) + try: + repo.git.merge("--abort") + except GitCommandError as e: + details = str(e) + lowered = details.lower() + if "there is no merge to abort" in lowered or "no merge to abort" in lowered: + return {"status": "no_merge_in_progress"} + raise HTTPException(status_code=409, detail=f"Cannot abort merge: {details}") + return {"status": "aborted"} + # [/DEF:abort_merge:Function] + + # [DEF:continue_merge:Function] + # @PURPOSE: Finalize merge after conflict resolution. + def continue_merge(self, dashboard_id: int, message: Optional[str] = None) -> Dict[str, Any]: + with belief_scope("GitService.continue_merge"): + repo = self.get_repo(dashboard_id) + unmerged_files = self._get_unmerged_file_paths(repo) + if unmerged_files: + raise HTTPException( + status_code=409, + detail={ + "error_code": "GIT_MERGE_CONFLICTS_REMAIN", + "message": "\u041d\u0435\u0432\u043e\u0437\u043c\u043e\u0436\u043d\u043e \u0437\u0430\u0432\u0435\u0440\u0448\u0438\u0442\u044c merge: \u043e\u0441\u0442\u0430\u043b\u0438\u0441\u044c \u043d\u0435\u0440\u0430\u0437\u0440\u0435\u0448\u0451\u043d\u043d\u044b\u0435 \u043a\u043e\u043d\u0444\u043b\u0438\u043a\u0442\u044b.", + "unresolved_files": unmerged_files, + }, + ) + try: + normalized_message = str(message or "").strip() + if normalized_message: + repo.git.commit("-m", normalized_message) + else: + repo.git.commit("--no-edit") + except GitCommandError as e: + details = str(e) + lowered = details.lower() + if "nothing to commit" in lowered: + return {"status": "already_clean"} + raise HTTPException(status_code=409, detail=f"Cannot continue merge: {details}") + commit_hash = "" + try: + commit_hash = repo.head.commit.hexsha + except Exception: + commit_hash = "" + return {"status": "committed", "commit_hash": commit_hash} + # [/DEF:continue_merge:Function] + + # [DEF:promote_direct_merge:Function] + # @PURPOSE: Perform direct merge between branches in local repo and push target branch. + # @PRE: Repository exists and both branches are valid. + # @POST: Target branch contains merged changes from source branch. + # @RETURN: Dict[str, Any] + def promote_direct_merge(self, dashboard_id: int, from_branch: str, to_branch: str) -> Dict[str, Any]: + with belief_scope("GitService.promote_direct_merge"): + if not from_branch or not to_branch: + raise HTTPException(status_code=400, detail="from_branch and to_branch are required") + repo = self.get_repo(dashboard_id) + source = from_branch.strip() + target = to_branch.strip() + if source == target: + raise HTTPException(status_code=400, detail="from_branch and to_branch must be different") + try: + origin = repo.remote(name="origin") + except ValueError: + raise HTTPException(status_code=400, detail="Remote 'origin' not configured") + try: + origin.fetch() + if source not in [head.name for head in repo.heads]: + if f"origin/{source}" in [ref.name for ref in repo.refs]: + repo.git.checkout("-b", source, f"origin/{source}") + else: + raise HTTPException(status_code=404, detail=f"Source branch '{source}' not found") + if target in [head.name for head in repo.heads]: + repo.git.checkout(target) + elif f"origin/{target}" in [ref.name for ref in repo.refs]: + repo.git.checkout("-b", target, f"origin/{target}") + else: + raise HTTPException(status_code=404, detail=f"Target branch '{target}' not found") + try: + origin.pull(target) + except Exception: + pass + repo.git.merge(source, "--no-ff", "-m", f"chore(flow): promote {source} -> {target}") + origin.push(refspec=f"{target}:{target}") + except HTTPException: + raise + except Exception as e: + message = str(e) + if "CONFLICT" in message.upper(): + raise HTTPException(status_code=409, detail=f"Merge conflict during direct promote: {message}") + raise HTTPException(status_code=500, detail=f"Direct promote failed: {message}") + return {"mode": "direct", "from_branch": source, "to_branch": target, "status": "merged"} + # [/DEF:promote_direct_merge:Function] +# [/DEF:GitServiceMergeMixin:Class] +# [/DEF:GitServiceMergeMixin:Module] diff --git a/backend/src/services/git/_remote_providers.py b/backend/src/services/git/_remote_providers.py new file mode 100644 index 00000000..89846653 --- /dev/null +++ b/backend/src/services/git/_remote_providers.py @@ -0,0 +1,186 @@ +# [DEF:GitServiceRemoteMixin:Module] +# @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: git, github, gitlab, api, provider, repository, pull_request, merge_request +# @PURPOSE: GitHub and GitLab provider operations for GitService — repository creation and PR/MR creation. +# @RELATION: USED_BY -> [GitService] + +import httpx +from typing import Any, Dict, Optional +from urllib.parse import quote +from fastapi import HTTPException +from src.core.logger import logger, belief_scope + + +# [DEF:GitServiceGithubMixin:Class] +# @COMPLEXITY: 3 +# @PURPOSE: Mixin providing GitHub API operations for GitService. +class GitServiceGithubMixin: + # [DEF:create_github_repository:Function] + # @PURPOSE: Create repository in GitHub or GitHub Enterprise. + # @PRE: PAT has repository create permission. + # @POST: Returns created repository payload. + # @RETURN: dict + async def create_github_repository( + self, server_url: str, pat: str, name: str, private: bool = True, + description: Optional[str] = None, auto_init: bool = True, default_branch: Optional[str] = "main", + ) -> Dict[str, Any]: + base_url = self._normalize_git_server_url(server_url) + if "github.com" in base_url: + api_url = "https://api.github.com/user/repos" + else: + api_url = f"{base_url}/api/v3/user/repos" + headers = { + "Authorization": f"token {pat.strip()}", + "Content-Type": "application/json", + "Accept": "application/vnd.github+json", + } + payload: Dict[str, Any] = {"name": name, "private": bool(private), "auto_init": bool(auto_init)} + if description: + payload["description"] = description + if default_branch: + payload["default_branch"] = default_branch + try: + async with httpx.AsyncClient(timeout=20.0) as client: + response = await client.post(api_url, headers=headers, json=payload) + except Exception as e: + raise HTTPException(status_code=503, detail=f"GitHub API is unavailable: {str(e)}") + if response.status_code >= 400: + detail = response.text + try: + parsed = response.json() + detail = parsed.get("message") or detail + except Exception: + pass + raise HTTPException(status_code=response.status_code, detail=f"GitHub API error: {detail}") + return response.json() + # [/DEF:create_github_repository:Function] + + # [DEF:create_github_pull_request:Function] + # @PURPOSE: Create pull request in GitHub or GitHub Enterprise. + # @PRE: Config and remote URL are valid. + # @POST: Returns normalized PR metadata. + # @RETURN: Dict[str, Any] + async def create_github_pull_request( + self, server_url: str, pat: str, remote_url: str, from_branch: str, to_branch: str, + title: str, description: Optional[str] = None, draft: bool = False, + ) -> Dict[str, Any]: + identity = self._parse_remote_repo_identity(remote_url) + base_url = self._normalize_git_server_url(server_url) + if "github.com" in base_url: + api_url = f"https://api.github.com/repos/{identity['namespace']}/{identity['repo']}/pulls" + else: + api_url = f"{base_url}/api/v3/repos/{identity['namespace']}/{identity['repo']}/pulls" + headers = { + "Authorization": f"token {pat.strip()}", + "Content-Type": "application/json", + "Accept": "application/vnd.github+json", + } + payload = { + "title": title, "head": from_branch, "base": to_branch, + "body": description or "", "draft": bool(draft), + } + try: + async with httpx.AsyncClient(timeout=20.0) as client: + response = await client.post(api_url, headers=headers, json=payload) + except Exception as e: + raise HTTPException(status_code=503, detail=f"GitHub API is unavailable: {str(e)}") + if response.status_code >= 400: + detail = response.text + try: + detail = response.json().get("message") or detail + except Exception: + pass + raise HTTPException(status_code=response.status_code, detail=f"GitHub API error: {detail}") + data = response.json() + return {"id": data.get("number") or data.get("id"), "url": data.get("html_url") or data.get("url"), "status": data.get("state") or "open"} + # [/DEF:create_github_pull_request:Function] + + +# [DEF:GitServiceGitlabMixin:Class] +# @COMPLEXITY: 3 +# @PURPOSE: Mixin providing GitLab API operations for GitService. +class GitServiceGitlabMixin: + # [DEF:create_gitlab_repository:Function] + # @PURPOSE: Create repository(project) in GitLab. + # @PRE: PAT has api scope. + # @POST: Returns created repository payload. + # @RETURN: dict + async def create_gitlab_repository( + self, server_url: str, pat: str, name: str, private: bool = True, + description: Optional[str] = None, auto_init: bool = True, default_branch: Optional[str] = "main", + ) -> Dict[str, Any]: + base_url = self._normalize_git_server_url(server_url) + api_url = f"{base_url}/api/v4/projects" + headers = {"PRIVATE-TOKEN": pat.strip(), "Content-Type": "application/json", "Accept": "application/json"} + payload: Dict[str, Any] = { + "name": name, "visibility": "private" if private else "public", + "initialize_with_readme": bool(auto_init), + } + if description: + payload["description"] = description + if default_branch: + payload["default_branch"] = default_branch + try: + async with httpx.AsyncClient(timeout=20.0) as client: + response = await client.post(api_url, headers=headers, json=payload) + except Exception as e: + raise HTTPException(status_code=503, detail=f"GitLab API is unavailable: {str(e)}") + if response.status_code >= 400: + detail = response.text + try: + parsed = response.json() + if isinstance(parsed, dict): + detail = parsed.get("message") or detail + except Exception: + pass + raise HTTPException(status_code=response.status_code, detail=f"GitLab API error: {detail}") + data = response.json() + if "clone_url" not in data: + data["clone_url"] = data.get("http_url_to_repo") + if "html_url" not in data: + data["html_url"] = data.get("web_url") + if "ssh_url" not in data: + data["ssh_url"] = data.get("ssh_url_to_repo") + if "full_name" not in data: + data["full_name"] = data.get("path_with_namespace") or data.get("name") + return data + # [/DEF:create_gitlab_repository:Function] + + # [DEF:create_gitlab_merge_request:Function] + # @PURPOSE: Create merge request in GitLab. + # @PRE: Config and remote URL are valid. + # @POST: Returns normalized MR metadata. + # @RETURN: Dict[str, Any] + async def create_gitlab_merge_request( + self, server_url: str, pat: str, remote_url: str, from_branch: str, to_branch: str, + title: str, description: Optional[str] = None, remove_source_branch: bool = False, + ) -> Dict[str, Any]: + identity = self._parse_remote_repo_identity(remote_url) + base_url = self._normalize_git_server_url(server_url) + project_id = quote(identity["full_name"], safe="") + api_url = f"{base_url}/api/v4/projects/{project_id}/merge_requests" + headers = {"PRIVATE-TOKEN": pat.strip(), "Content-Type": "application/json", "Accept": "application/json"} + payload = { + "source_branch": from_branch, "target_branch": to_branch, "title": title, + "description": description or "", "remove_source_branch": bool(remove_source_branch), + } + try: + async with httpx.AsyncClient(timeout=20.0) as client: + response = await client.post(api_url, headers=headers, json=payload) + except Exception as e: + raise HTTPException(status_code=503, detail=f"GitLab API is unavailable: {str(e)}") + if response.status_code >= 400: + detail = response.text + try: + parsed = response.json() + if isinstance(parsed, dict): + detail = parsed.get("message") or detail + except Exception: + pass + raise HTTPException(status_code=response.status_code, detail=f"GitLab API error: {detail}") + data = response.json() + return {"id": data.get("iid") or data.get("id"), "url": data.get("web_url") or data.get("url"), "status": data.get("state") or "opened"} + # [/DEF:create_gitlab_merge_request:Function] +# [/DEF:GitServiceGitlabMixin:Class] +# [/DEF:GitServiceRemoteMixin:Module] diff --git a/backend/src/services/git/_status.py b/backend/src/services/git/_status.py new file mode 100644 index 00000000..303e0460 --- /dev/null +++ b/backend/src/services/git/_status.py @@ -0,0 +1,164 @@ +# [DEF:GitServiceStatusMixin:Module] +# @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: git, status, diff, history, porcelain +# @PURPOSE: Status, diff, and commit history operations for GitService using git status --porcelain to avoid --cached flag issues. +# @RELATION: USED_BY -> [GitService] + +from typing import Any, Dict, List +from datetime import datetime +from git import Repo +from src.core.logger import logger, belief_scope + + +# [DEF:GitServiceStatusMixin:Class] +# @COMPLEXITY: 3 +# @PURPOSE: Mixin providing repository status, diff, and commit history for GitService. +class GitServiceStatusMixin: + # [DEF:_parse_status_porcelain:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Parse git status --porcelain output into staged, modified, and untracked file lists. + # @PRE: `repo` is an open GitPython Repo instance. + # @POST: Returns (staged, modified, untracked) tuple of file path lists. + # @RATIONALE: Avoids repo.is_dirty() / repo.index.diff("HEAD") which internally + # call git diff --cached, a flag unsupported in some Git environments (exit 129). + # Using git status --porcelain is self-contained and avoids the --cached flag entirely. + def _parse_status_porcelain(self, repo) -> tuple[list[str], list[str], list[str]]: + with belief_scope("GitService._parse_status_porcelain"): + staged: list[str] = [] + modified: list[str] = [] + untracked: list[str] = [] + try: + output = repo.git.status("--porcelain") + except Exception: + logger.warning("[_parse_status_porcelain][Coherence:Failed] git status --porcelain failed") + return staged, modified, untracked + for line in output.split("\n"): + if not line: + continue + if line.startswith("??"): + untracked.append(line[2:].strip()) + continue + if line.startswith("!!"): + continue + if len(line) < 3: + continue + x = line[0] + y = line[1] + rest = line[3:] + path = rest.split(" -> ")[-1].strip() if " -> " in rest else rest.strip() + if x != " " and x != "?": + staged.append(path) + if y != " " and y != "?": + if path not in modified: + modified.append(path) + return staged, modified, untracked + # [/DEF:_parse_status_porcelain:Function] + + # [DEF:get_status:Function] + # @PURPOSE: Get current repository status (dirty files, untracked, etc.) + # @PRE: Repository for dashboard_id exists. + # @POST: Returns a dictionary representing the Git status. + # @RETURN: dict + def get_status(self, dashboard_id: int) -> dict: + with belief_scope("GitService.get_status"): + repo = self.get_repo(dashboard_id) + has_commits = False + try: + repo.head.commit + has_commits = True + except (ValueError, Exception): + has_commits = False + current_branch = repo.active_branch.name + tracking_branch = None + has_upstream = False + ahead_count = 0 + behind_count = 0 + try: + tracking_branch = repo.active_branch.tracking_branch() + has_upstream = tracking_branch is not None + except Exception: + tracking_branch = None + has_upstream = False + if has_upstream and tracking_branch is not None: + try: + ahead_count = sum(1 for _ in repo.iter_commits(f"{tracking_branch.name}..{current_branch}")) + behind_count = sum(1 for _ in repo.iter_commits(f"{current_branch}..{tracking_branch.name}")) + except Exception: + ahead_count = 0 + behind_count = 0 + staged_files, modified_files, untracked_files = self._parse_status_porcelain(repo) + is_dirty = bool(staged_files or modified_files or untracked_files) + is_diverged = ahead_count > 0 and behind_count > 0 + if is_diverged: + sync_state = "DIVERGED" + elif behind_count > 0: + sync_state = "BEHIND_REMOTE" + elif ahead_count > 0: + sync_state = "AHEAD_REMOTE" + elif is_dirty or modified_files or staged_files or untracked_files: + sync_state = "CHANGES" + else: + sync_state = "SYNCED" + return { + "is_dirty": is_dirty, + "untracked_files": untracked_files, + "modified_files": modified_files, + "staged_files": staged_files, + "current_branch": current_branch, + "upstream_branch": tracking_branch.name if tracking_branch is not None else None, + "has_upstream": has_upstream, + "ahead_count": ahead_count, + "behind_count": behind_count, + "is_diverged": is_diverged, + "sync_state": sync_state, + } + # [/DEF:get_status:Function] + + # [DEF:get_diff:Function] + # @PURPOSE: Generate diff for a file or the whole repository. + # @PARAM: file_path (str) - Optional specific file. + # @PARAM: staged (bool) - Whether to show staged changes. + # @PRE: Repository for dashboard_id exists. + # @POST: Returns the diff text as a string. + # @RETURN: str + def get_diff(self, dashboard_id: int, file_path: str = None, staged: bool = False) -> str: + with belief_scope("GitService.get_diff"): + repo = self.get_repo(dashboard_id) + diff_args = [] + if staged: + diff_args.append("--staged") + if file_path: + return repo.git.diff(*diff_args, "--", file_path) + return repo.git.diff(*diff_args) + # [/DEF:get_diff:Function] + + # [DEF:get_commit_history:Function] + # @PURPOSE: Retrieve commit history for a repository. + # @PARAM: limit (int) - Max number of commits to return. + # @PRE: Repository for dashboard_id exists. + # @POST: Returns a list of dictionaries for each commit in history. + # @RETURN: List[dict] + def get_commit_history(self, dashboard_id: int, limit: int = 50) -> List[dict]: + with belief_scope("GitService.get_commit_history"): + repo = self.get_repo(dashboard_id) + commits = [] + try: + if not repo.heads and not repo.remotes: + return [] + for commit in repo.iter_commits(max_count=limit): + commits.append({ + "hash": commit.hexsha, + "author": commit.author.name, + "email": commit.author.email, + "timestamp": datetime.fromtimestamp(commit.committed_date), + "message": commit.message.strip(), + "files_changed": list(commit.stats.files.keys()) + }) + except Exception as e: + logger.warning(f"[get_commit_history][Action] Could not retrieve commit history for dashboard {dashboard_id}: {e}") + return [] + return commits + # [/DEF:get_commit_history:Function] +# [/DEF:GitServiceStatusMixin:Class] +# [/DEF:GitServiceStatusMixin:Module] diff --git a/backend/src/services/git/_sync.py b/backend/src/services/git/_sync.py new file mode 100644 index 00000000..4a340990 --- /dev/null +++ b/backend/src/services/git/_sync.py @@ -0,0 +1,177 @@ +# [DEF:GitServiceSyncMixin:Module] +# @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: git, push, pull, sync, remote +# @PURPOSE: Push and pull operations for GitService with origin host auto-alignment. +# @RELATION: USED_BY -> [GitService] +# @RELATION: DEPENDS_ON -> [GitServiceUrlMixin] + +import os +from typing import Optional +from git import Repo +from git.exc import GitCommandError +from fastapi import HTTPException +from src.core.database import SessionLocal +from src.core.logger import logger, belief_scope +from src.models.git import GitRepository, GitServerConfig + + +# [DEF:GitServiceSyncMixin:Class] +# @COMPLEXITY: 3 +# @PURPOSE: Mixin providing push and pull operations with origin host alignment. +class GitServiceSyncMixin: + # [DEF:push_changes:Function] + # @PURPOSE: Push local commits to remote. + # @PRE: Repository exists and has an 'origin' remote. + # @POST: Local branch commits are pushed to origin. + def push_changes(self, dashboard_id: int): + with belief_scope("GitService.push_changes"): + repo = self.get_repo(dashboard_id) + if not repo.heads: + logger.warning(f"[push_changes][Coherence:Failed] No local branches to push for dashboard {dashboard_id}") + return + try: + origin = repo.remote(name='origin') + except ValueError: + logger.error(f"[push_changes][Coherence:Failed] Remote 'origin' not found for dashboard {dashboard_id}") + raise HTTPException(status_code=400, detail="Remote 'origin' not configured") + try: + origin_urls = list(origin.urls) + except Exception: + origin_urls = [] + binding_remote_url = None + binding_config_id = None + binding_config_url = None + try: + session = SessionLocal() + try: + db_repo = ( + session.query(GitRepository) + .filter(GitRepository.dashboard_id == int(dashboard_id)) + .first() + ) + if db_repo: + binding_remote_url = db_repo.remote_url + binding_config_id = db_repo.config_id + db_config = ( + session.query(GitServerConfig) + .filter(GitServerConfig.id == db_repo.config_id) + .first() + ) + if db_config: + binding_config_url = db_config.url + finally: + session.close() + except Exception as diag_error: + logger.warning( + "[push_changes][Action] Failed to load repository binding diagnostics for dashboard %s: %s", + dashboard_id, diag_error, + ) + realigned_origin_url = self._align_origin_host_with_config( + dashboard_id=dashboard_id, + origin=origin, + config_url=binding_config_url, + current_origin_url=(origin_urls[0] if origin_urls else None), + binding_remote_url=binding_remote_url, + ) + try: + origin_urls = list(origin.urls) + except Exception: + origin_urls = [] + logger.info( + "[push_changes][Action] Push diagnostics dashboard=%s config_id=%s config_url=%s binding_remote_url=%s origin_urls=%s origin_realigned=%s", + dashboard_id, binding_config_id, binding_config_url, binding_remote_url, origin_urls, bool(realigned_origin_url), + ) + try: + current_branch = repo.active_branch + logger.info(f"[push_changes][Action] Pushing branch {current_branch.name} to origin") + tracking_branch = None + try: + tracking_branch = current_branch.tracking_branch() + except Exception: + tracking_branch = None + if tracking_branch is None: + repo.git.push("--set-upstream", "origin", f"{current_branch.name}:{current_branch.name}") + else: + push_info = origin.push(refspec=f'{current_branch.name}:{current_branch.name}') + for info in push_info: + if info.flags & info.ERROR: + logger.error(f"[push_changes][Coherence:Failed] Error pushing ref {info.remote_ref_string}: {info.summary}") + raise Exception(f"Git push error for {info.remote_ref_string}: {info.summary}") + except GitCommandError as e: + details = str(e) + lowered = details.lower() + if "non-fast-forward" in lowered or "rejected" in lowered: + raise HTTPException( + status_code=409, + detail="Push rejected: remote branch contains newer commits. Run Pull first, resolve conflicts if any, then push again.", + ) + logger.error(f"[push_changes][Coherence:Failed] Failed to push changes: {e}") + raise HTTPException(status_code=500, detail=f"Git push failed: {details}") + except Exception as e: + logger.error(f"[push_changes][Coherence:Failed] Failed to push changes: {e}") + raise HTTPException(status_code=500, detail=f"Git push failed: {str(e)}") + # [/DEF:push_changes:Function] + + # [DEF:pull_changes:Function] + # @PURPOSE: Pull changes from remote. + # @PRE: Repository exists and has an 'origin' remote. + # @POST: Changes from origin are pulled and merged into the active branch. + def pull_changes(self, dashboard_id: int): + with belief_scope("GitService.pull_changes"): + repo = self.get_repo(dashboard_id) + merge_head_path = os.path.join(repo.git_dir, "MERGE_HEAD") + if os.path.exists(merge_head_path): + payload = self._build_unfinished_merge_payload(repo) + logger.warning( + "[pull_changes][Action] Unfinished merge detected for dashboard %s (repo_path=%s git_dir=%s branch=%s merge_head=%s merge_msg=%s)", + dashboard_id, payload["repository_path"], payload["git_dir"], + payload["current_branch"], payload["merge_head"], payload["merge_message_preview"], + ) + raise HTTPException(status_code=409, detail=payload) + try: + origin = repo.remote(name='origin') + current_branch = repo.active_branch.name + try: + origin_urls = list(origin.urls) + except Exception: + origin_urls = [] + logger.info( + "[pull_changes][Action] Pull diagnostics dashboard=%s repo_path=%s branch=%s origin_urls=%s", + dashboard_id, repo.working_tree_dir, current_branch, origin_urls, + ) + origin.fetch(prune=True) + remote_ref = f"origin/{current_branch}" + has_remote_branch = any(ref.name == remote_ref for ref in repo.refs) + logger.info( + "[pull_changes][Action] Pull remote branch check dashboard=%s branch=%s remote_ref=%s exists=%s", + dashboard_id, current_branch, remote_ref, has_remote_branch, + ) + if not has_remote_branch: + raise HTTPException( + status_code=409, + detail=f"Remote branch '{current_branch}' does not exist yet. Push this branch first.", + ) + logger.info(f"[pull_changes][Action] Pulling changes from origin/{current_branch}") + repo.git.pull("--no-rebase", "origin", current_branch) + except ValueError: + logger.error(f"[pull_changes][Coherence:Failed] Remote 'origin' not found for dashboard {dashboard_id}") + raise HTTPException(status_code=400, detail="Remote 'origin' not configured") + except GitCommandError as e: + details = str(e) + lowered = details.lower() + if "conflict" in lowered or "not possible to fast-forward" in lowered: + raise HTTPException( + status_code=409, + detail="Pull requires conflict resolution. Resolve conflicts in repository and repeat operation.", + ) + logger.error(f"[pull_changes][Coherence:Failed] Failed to pull changes: {e}") + raise HTTPException(status_code=500, detail=f"Git pull failed: {details}") + except HTTPException: + raise + except Exception as e: + logger.error(f"[pull_changes][Coherence:Failed] Failed to pull changes: {e}") + raise HTTPException(status_code=500, detail=f"Git pull failed: {str(e)}") + # [/DEF:pull_changes:Function] +# [/DEF:GitServiceSyncMixin:Class] +# [/DEF:GitServiceSyncMixin:Module] diff --git a/backend/src/services/git/_url.py b/backend/src/services/git/_url.py new file mode 100644 index 00000000..5d0ec135 --- /dev/null +++ b/backend/src/services/git/_url.py @@ -0,0 +1,212 @@ +# [DEF:GitServiceUrlMixin:Module] +# @COMPLEXITY: 3 +# @LAYER: Infra +# @SEMANTICS: git, url, parsing, normalization, host_alignment +# @PURPOSE: URL helper mixin for GitService — parse, normalize, align, and strip credentials from Git URLs. +# @RELATION: USED_BY -> [GitServiceSyncMixin] +# @RELATION: USED_BY -> [GitServiceGiteaMixin] +# @RELATION: USED_BY -> [GitServiceRemoteMixin] + +import os +from typing import Any, Dict, List, Optional +from urllib.parse import quote, urlparse +from fastapi import HTTPException +from src.core.database import SessionLocal +from src.core.logger import logger, belief_scope +from src.models.git import GitRepository, GitServerConfig + +# [DEF:GitServiceUrlMixin:Class] +# @COMPLEXITY: 3 +# @PURPOSE: URL helper methods for GitService — extract host, strip credentials, replace host, align origin, parse remote identity, derive server URL, normalize server URL. +class GitServiceUrlMixin: + # [DEF:_extract_http_host:Function] + # @PURPOSE: Extract normalized host[:port] from HTTP(S) URL. + # @PRE: url_value may be empty. + # @POST: Returns lowercase host token or None. + # @RETURN: Optional[str] + def _extract_http_host(self, url_value: Optional[str]) -> Optional[str]: + normalized = str(url_value or "").strip() + if not normalized: + return None + try: + parsed = urlparse(normalized) + except Exception: + return None + if parsed.scheme not in {"http", "https"}: + return None + host = parsed.hostname + if not host: + return None + if parsed.port: + return f"{host.lower()}:{parsed.port}" + return host.lower() + # [/DEF:_extract_http_host:Function] + + # [DEF:_strip_url_credentials:Function] + # @PURPOSE: Remove credentials from URL while preserving scheme/host/path. + # @PRE: url_value may contain credentials. + # @POST: Returns URL without username/password. + # @RETURN: str + def _strip_url_credentials(self, url_value: str) -> str: + normalized = str(url_value or "").strip() + if not normalized: + return normalized + try: + parsed = urlparse(normalized) + except Exception: + return normalized + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + return normalized + host = parsed.hostname + if parsed.port: + host = f"{host}:{parsed.port}" + return parsed._replace(netloc=host).geturl() + # [/DEF:_strip_url_credentials:Function] + + # [DEF:_replace_host_in_url:Function] + # @PURPOSE: Replace source URL host with host from configured server URL. + # @PRE: source_url and config_url are HTTP(S) URLs. + # @POST: Returns source URL with updated host (credentials preserved) or None. + # @RETURN: Optional[str] + def _replace_host_in_url(self, source_url: Optional[str], config_url: Optional[str]) -> Optional[str]: + source = str(source_url or "").strip() + config = str(config_url or "").strip() + if not source or not config: + return None + try: + source_parsed = urlparse(source) + config_parsed = urlparse(config) + except Exception: + return None + if source_parsed.scheme not in {"http", "https"} or config_parsed.scheme not in {"http", "https"}: + return None + if not source_parsed.hostname or not config_parsed.hostname: + return None + target_host = config_parsed.hostname + if config_parsed.port: + target_host = f"{target_host}:{config_parsed.port}" + auth_part = "" + if source_parsed.username: + auth_part = quote(source_parsed.username, safe="") + if source_parsed.password is not None: + auth_part = f"{auth_part}:{quote(source_parsed.password, safe='')}" + auth_part = f"{auth_part}@" + new_netloc = f"{auth_part}{target_host}" + return source_parsed._replace(netloc=new_netloc).geturl() + # [/DEF:_replace_host_in_url:Function] + + # [DEF:_align_origin_host_with_config:Function] + # @PURPOSE: Auto-align local origin host to configured Git server host when they drift. + # @PRE: origin remote exists. + # @POST: origin URL host updated and DB binding normalized when mismatch detected. + # @RETURN: Optional[str] + def _align_origin_host_with_config( + self, + dashboard_id: int, + origin, + config_url: Optional[str], + current_origin_url: Optional[str], + binding_remote_url: Optional[str], + ) -> Optional[str]: + config_host = self._extract_http_host(config_url) + source_origin_url = str(current_origin_url or "").strip() or str(binding_remote_url or "").strip() + origin_host = self._extract_http_host(source_origin_url) + if not config_host or not origin_host: + return None + if config_host == origin_host: + return None + aligned_url = self._replace_host_in_url(source_origin_url, config_url) + if not aligned_url: + return None + logger.warning( + "[_align_origin_host_with_config][Action] Host mismatch for dashboard %s: config_host=%s origin_host=%s, applying origin.set_url", + dashboard_id, config_host, origin_host, + ) + try: + origin.set_url(aligned_url) + except Exception as e: + logger.warning( + "[_align_origin_host_with_config][Coherence:Failed] Failed to set origin URL for dashboard %s: %s", + dashboard_id, e, + ) + return None + try: + session = SessionLocal() + try: + db_repo = ( + session.query(GitRepository) + .filter(GitRepository.dashboard_id == int(dashboard_id)) + .first() + ) + if db_repo: + db_repo.remote_url = self._strip_url_credentials(aligned_url) + session.commit() + finally: + session.close() + except Exception as e: + logger.warning( + "[_align_origin_host_with_config][Action] Failed to persist aligned remote_url for dashboard %s: %s", + dashboard_id, e, + ) + return aligned_url + # [/DEF:_align_origin_host_with_config:Function] + + # [DEF:_parse_remote_repo_identity:Function] + # @PURPOSE: Parse owner/repo from remote URL for Git server API operations. + # @PRE: remote_url is a valid git URL. + # @POST: Returns owner/repo tokens. + # @RETURN: Dict[str, str] + def _parse_remote_repo_identity(self, remote_url: str) -> Dict[str, str]: + normalized = str(remote_url or "").strip() + if not normalized: + raise HTTPException(status_code=400, detail="Repository remote_url is empty") + if normalized.startswith("git@"): + path = normalized.split(":", 1)[1] if ":" in normalized else "" + else: + parsed = urlparse(normalized) + path = parsed.path or "" + path = path.strip("/") + if path.endswith(".git"): + path = path[:-4] + parts = [segment for segment in path.split("/") if segment] + if len(parts) < 2: + raise HTTPException(status_code=400, detail=f"Cannot parse repository owner/name from remote URL: {remote_url}") + owner = parts[0] + repo = parts[-1] + namespace = "/".join(parts[:-1]) + return {"owner": owner, "repo": repo, "namespace": namespace, "full_name": f"{namespace}/{repo}"} + # [/DEF:_parse_remote_repo_identity:Function] + + # [DEF:_derive_server_url_from_remote:Function] + # @PURPOSE: Build API base URL from remote repository URL without credentials. + # @PRE: remote_url may be any git URL. + # @POST: Returns normalized http(s) base URL or None when derivation is impossible. + # @RETURN: Optional[str] + def _derive_server_url_from_remote(self, remote_url: str) -> Optional[str]: + normalized = str(remote_url or "").strip() + if not normalized or normalized.startswith("git@"): + return None + parsed = urlparse(normalized) + if parsed.scheme not in {"http", "https"}: + return None + if not parsed.hostname: + return None + netloc = parsed.hostname + if parsed.port: + netloc = f"{netloc}:{parsed.port}" + return f"{parsed.scheme}://{netloc}".rstrip("/") + # [/DEF:_derive_server_url_from_remote:Function] + + # [DEF:_normalize_git_server_url:Function] + # @PURPOSE: Normalize Git server URL for provider API calls. + # @PRE: raw_url is non-empty. + # @POST: Returns URL without trailing slash. + # @RETURN: str + def _normalize_git_server_url(self, raw_url: str) -> str: + normalized = (raw_url or "").strip() + if not normalized: + raise HTTPException(status_code=400, detail="Git server URL is required") + return normalized.rstrip("/") + # [/DEF:_normalize_git_server_url:Function] +# [/DEF:GitServiceUrlMixin:Class] +# [/DEF:GitServiceUrlMixin:Module] diff --git a/backend/src/services/git_service.py b/backend/src/services/git_service.py index 36fe3d4b..8a8084a5 100644 --- a/backend/src/services/git_service.py +++ b/backend/src/services/git_service.py @@ -1,2101 +1,11 @@ -# [DEF:git_service:Module] -# -# @COMPLEXITY: 3 -# @SEMANTICS: git, service, gitpython, repository, version_control -# @PURPOSE: Core Git logic using GitPython to manage dashboard repositories. -# @LAYER: Service -# @RELATION: USED_BY -> [GitApi] -# @RELATION: USED_BY -> [GitPluginModule] -# @RELATION: DEPENDS_ON -> [SessionLocal] -# @RELATION: DEPENDS_ON -> [AppConfigRecord] -# @RELATION: DEPENDS_ON -> [GitRepository] -# -# @INVARIANT: All Git operations must be performed on a valid local directory. - -import os -import httpx -import re -import shutil -from git import Repo -from git.exc import GitCommandError -from git.exc import InvalidGitRepositoryError, NoSuchPathError -from git.objects.blob import Blob -from fastapi import HTTPException -from typing import Any, Dict, List, Optional -from datetime import datetime -from pathlib import Path -from urllib.parse import quote, urlparse -from src.core.logger import logger, belief_scope -from src.models.git import GitProvider -from src.models.git import GitRepository, GitServerConfig -from src.models.config import AppConfigRecord -from src.core.database import SessionLocal - -# [DEF:GitService:Class] -# @COMPLEXITY: 3 -# @PURPOSE: Wrapper for GitPython operations with semantic logging and error handling. -# @RELATION: DEPENDS_ON -> git -class GitService: - """ - Wrapper for GitPython operations. - """ - - # [DEF:GitService_init:Function] - # @PURPOSE: Initializes the GitService with a base path for repositories. - # @PARAM: base_path (str) - Root directory for all Git clones. - # @PRE: base_path is a valid string path. - # @POST: GitService is initialized; base_path directory exists. - # @RELATION: CALLS -> [GitService._resolve_base_path] - # @RELATION: CALLS -> [GitService._ensure_base_path_exists] - def __init__(self, base_path: str = "git_repos"): - with belief_scope("GitService.__init__"): - backend_root = Path(__file__).parents[2] - self.legacy_base_path = str((backend_root / "git_repos").resolve()) - self._uses_default_base_path = base_path == "git_repos" - self.base_path = self._resolve_base_path(base_path) - self._ensure_base_path_exists() - # [/DEF:GitService_init:Function] - - # [DEF:_ensure_base_path_exists:Function] - # @PURPOSE: Ensure the repositories root directory exists and is a directory. - # @PRE: self.base_path is resolved to filesystem path. - # @POST: self.base_path exists as directory or raises ValueError. - # @RETURN: None - # @RELATION: USES -> [self.base_path] - def _ensure_base_path_exists(self) -> None: - base = Path(self.base_path) - if base.exists() and not base.is_dir(): - raise ValueError(f"Git repositories base path is not a directory: {self.base_path}") - try: - base.mkdir(parents=True, exist_ok=True) - except (PermissionError, OSError) as e: - logger.warning( - f"[_ensure_base_path_exists][Coherence:Failed] Cannot create Git repositories base path: {self.base_path}. Error: {e}" - ) - raise ValueError(f"Cannot create Git repositories base path: {self.base_path}. {e}") - # [/DEF:_ensure_base_path_exists:Function] - - # [DEF:_resolve_base_path:Function] - # @PURPOSE: Resolve base repository directory from explicit argument or global storage settings. - # @PRE: base_path is a string path. - # @POST: Returns absolute path for Git repositories root. - # @RETURN: str - # @RELATION: USES -> [AppConfigRecord] - def _resolve_base_path(self, base_path: str) -> str: - # Resolve relative to backend directory for backward compatibility. - backend_root = Path(__file__).parents[2] - fallback_path = str((backend_root / base_path).resolve()) - - if base_path != "git_repos": - return fallback_path - - try: - session = SessionLocal() - try: - config_row = session.query(AppConfigRecord).filter(AppConfigRecord.id == "global").first() - finally: - session.close() - - payload = (config_row.payload if config_row and config_row.payload else {}) if config_row else {} - storage_cfg = payload.get("settings", {}).get("storage", {}) if isinstance(payload, dict) else {} - - root_path = str(storage_cfg.get("root_path", "")).strip() - repo_path = str(storage_cfg.get("repo_path", "")).strip() - if not root_path: - return fallback_path - - project_root = Path(__file__).parents[3] - root = Path(root_path) - if not root.is_absolute(): - root = (project_root / root).resolve() - - repo_root = Path(repo_path) if repo_path else Path("repositorys") - if repo_root.is_absolute(): - return str(repo_root.resolve()) - return str((root / repo_root).resolve()) - except Exception as e: - logger.warning(f"[_resolve_base_path][Coherence:Failed] Falling back to default path: {e}") - return fallback_path - # [/DEF:_resolve_base_path:Function] - - # [DEF:_normalize_repo_key:Function] - # @PURPOSE: Convert user/dashboard-provided key to safe filesystem directory name. - # @PRE: repo_key can be None/empty. - # @POST: Returns normalized non-empty key. - # @RETURN: str - # @RELATION: USES -> [re.sub] - def _normalize_repo_key(self, repo_key: Optional[str]) -> str: - raw_key = str(repo_key or "").strip().lower() - normalized = re.sub(r"[^a-z0-9._-]+", "-", raw_key).strip("._-") - return normalized or "dashboard" - # [/DEF:_normalize_repo_key:Function] - - # [DEF:_update_repo_local_path:Function] - # @PURPOSE: Persist repository local_path in GitRepository table when record exists. - # @PRE: dashboard_id is valid integer. - # @POST: local_path is updated for existing record. - # @RETURN: None - # @RELATION: USES -> [GitRepository] - def _update_repo_local_path(self, dashboard_id: int, local_path: str) -> None: - try: - session = SessionLocal() - try: - db_repo = ( - session.query(GitRepository) - .filter(GitRepository.dashboard_id == int(dashboard_id)) - .first() - ) - if db_repo: - db_repo.local_path = local_path - session.commit() - finally: - session.close() - except Exception as e: - logger.warning(f"[_update_repo_local_path][Coherence:Failed] {e}") - # [/DEF:_update_repo_local_path:Function] - - # [DEF:_migrate_repo_directory:Function] - # @PURPOSE: Move legacy repository directory to target path and sync DB metadata. - # @PRE: source_path exists. - # @POST: Repository content available at target_path. - # @RETURN: str - # @RELATION: CALLS -> [GitService._update_repo_local_path] - def _migrate_repo_directory(self, dashboard_id: int, source_path: str, target_path: str) -> str: - source_abs = os.path.abspath(source_path) - target_abs = os.path.abspath(target_path) - if source_abs == target_abs: - return source_abs - - if os.path.exists(target_abs): - logger.warning( - f"[_migrate_repo_directory][Action] Target already exists, keeping source path: {target_abs}" - ) - return source_abs - - Path(target_abs).parent.mkdir(parents=True, exist_ok=True) - try: - os.replace(source_abs, target_abs) - except OSError: - shutil.move(source_abs, target_abs) - - self._update_repo_local_path(dashboard_id, target_abs) - logger.info( - f"[_migrate_repo_directory][Coherence:OK] Repository migrated for dashboard {dashboard_id}: {source_abs} -> {target_abs}" - ) - return target_abs - # [/DEF:_migrate_repo_directory:Function] - - # [DEF:_ensure_gitflow_branches:Function] - # @PURPOSE: Ensure standard GitFlow branches (main/dev/preprod) exist locally and on origin. - # @PRE: repo is a valid GitPython Repo instance. - # @POST: main, dev, preprod are available in local repository and pushed to origin when available. - # @RETURN: None - # @RELATION: USES -> [Repo] - def _ensure_gitflow_branches(self, repo: Repo, dashboard_id: int) -> None: - with belief_scope("GitService._ensure_gitflow_branches"): - required_branches = ["main", "dev", "preprod"] - local_heads = {head.name: head for head in getattr(repo, "heads", [])} - - base_commit = None - try: - base_commit = repo.head.commit - except Exception: - base_commit = None - - if "main" in local_heads: - base_commit = local_heads["main"].commit - - if base_commit is None: - logger.warning( - f"[_ensure_gitflow_branches][Action] Skipping branch bootstrap for dashboard {dashboard_id}: repository has no commits" - ) - return - - if "main" not in local_heads: - local_heads["main"] = repo.create_head("main", base_commit) - logger.info(f"[_ensure_gitflow_branches][Action] Created local branch main for dashboard {dashboard_id}") - - for branch_name in ("dev", "preprod"): - if branch_name in local_heads: - continue - local_heads[branch_name] = repo.create_head(branch_name, local_heads["main"].commit) - logger.info( - f"[_ensure_gitflow_branches][Action] Created local branch {branch_name} for dashboard {dashboard_id}" - ) - - try: - origin = repo.remote(name="origin") - except ValueError: - logger.info( - f"[_ensure_gitflow_branches][Action] Remote origin is not configured for dashboard {dashboard_id}; skipping remote branch creation" - ) - return - - remote_branch_names = set() - try: - origin.fetch() - for ref in origin.refs: - remote_head = getattr(ref, "remote_head", None) - if remote_head: - remote_branch_names.add(str(remote_head)) - except Exception as e: - logger.warning(f"[_ensure_gitflow_branches][Action] Failed to fetch origin refs: {e}") - - for branch_name in required_branches: - if branch_name in remote_branch_names: - continue - try: - origin.push(refspec=f"{branch_name}:{branch_name}") - logger.info( - f"[_ensure_gitflow_branches][Action] Pushed branch {branch_name} to origin for dashboard {dashboard_id}" - ) - except Exception as e: - logger.error( - f"[_ensure_gitflow_branches][Coherence:Failed] Failed to push branch {branch_name} to origin: {e}" - ) - raise HTTPException( - status_code=500, - detail=f"Failed to create default branch '{branch_name}' on remote: {str(e)}", - ) - - # Keep default working branch on DEV for day-to-day changes. - try: - repo.git.checkout("dev") - logger.info( - f"[_ensure_gitflow_branches][Action] Checked out default branch dev for dashboard {dashboard_id}" - ) - except Exception as e: - logger.warning( - f"[_ensure_gitflow_branches][Action] Could not checkout dev branch for dashboard {dashboard_id}: {e}" - ) - # [/DEF:_ensure_gitflow_branches:Function] - - # [DEF:_get_repo_path:Function] - # @PURPOSE: Resolves the local filesystem path for a dashboard's repository. - # @PARAM: dashboard_id (int) - # @PARAM: repo_key (Optional[str]) - Slug-like key used when DB local_path is absent. - # @PRE: dashboard_id is an integer. - # @POST: Returns DB-local_path when present, otherwise base_path/. - # @RETURN: str - # @RELATION: CALLS -> [GitService._normalize_repo_key] - # @RELATION: CALLS -> [GitService._migrate_repo_directory] - # @RELATION: CALLS -> [GitService._update_repo_local_path] - def _get_repo_path(self, dashboard_id: int, repo_key: Optional[str] = None) -> str: - with belief_scope("GitService._get_repo_path"): - if dashboard_id is None: - raise ValueError("dashboard_id cannot be None") - self._ensure_base_path_exists() - fallback_key = repo_key if repo_key is not None else str(dashboard_id) - normalized_key = self._normalize_repo_key(fallback_key) - target_path = os.path.join(self.base_path, normalized_key) - - if not self._uses_default_base_path: - return target_path - - try: - session = SessionLocal() - try: - db_repo = ( - session.query(GitRepository) - .filter(GitRepository.dashboard_id == int(dashboard_id)) - .first() - ) - finally: - session.close() - if db_repo and db_repo.local_path: - db_path = os.path.abspath(db_repo.local_path) - if os.path.exists(db_path): - if ( - os.path.abspath(self.base_path) != os.path.abspath(self.legacy_base_path) - and db_path.startswith(os.path.abspath(self.legacy_base_path) + os.sep) - ): - return self._migrate_repo_directory(dashboard_id, db_path, target_path) - return db_path - except Exception as e: - logger.warning(f"[_get_repo_path][Coherence:Failed] Could not resolve local_path from DB: {e}") - - legacy_id_path = os.path.join(self.legacy_base_path, str(dashboard_id)) - if os.path.exists(legacy_id_path) and not os.path.exists(target_path): - return self._migrate_repo_directory(dashboard_id, legacy_id_path, target_path) - - if os.path.exists(target_path): - self._update_repo_local_path(dashboard_id, target_path) - - return target_path - # [/DEF:_get_repo_path:Function] - - # [DEF:init_repo:Function] - # @PURPOSE: Initialize or clone a repository for a dashboard. - # @PARAM: dashboard_id (int) - # @PARAM: remote_url (str) - # @PARAM: pat (str) - Personal Access Token for authentication. - # @PARAM: repo_key (Optional[str]) - Slug-like key for deterministic folder naming on first init. - # @PRE: dashboard_id is int, remote_url is valid Git URL, pat is provided. - # @POST: Repository is cloned or opened at the local path. - # @RETURN: Repo - GitPython Repo object. - # @RELATION: CALLS -> [GitService._get_repo_path] - # @RELATION: CALLS -> [GitService._ensure_gitflow_branches] - def init_repo(self, dashboard_id: int, remote_url: str, pat: str, repo_key: Optional[str] = None) -> Repo: - with belief_scope("GitService.init_repo"): - self._ensure_base_path_exists() - repo_path = self._get_repo_path(dashboard_id, repo_key=repo_key or str(dashboard_id)) - Path(repo_path).parent.mkdir(parents=True, exist_ok=True) - - # Inject PAT into remote URL if needed - if pat and "://" in remote_url: - proto, rest = remote_url.split("://", 1) - auth_url = f"{proto}://oauth2:{pat}@{rest}" - else: - auth_url = remote_url - - if os.path.exists(repo_path): - logger.info(f"[init_repo][Action] Opening existing repo at {repo_path}") - try: - repo = Repo(repo_path) - except (InvalidGitRepositoryError, NoSuchPathError): - logger.warning( - f"[init_repo][Action] Existing path is not a Git repository, recreating: {repo_path}" - ) - stale_path = Path(repo_path) - if stale_path.exists(): - shutil.rmtree(stale_path, ignore_errors=True) - if stale_path.exists(): - try: - stale_path.unlink() - except Exception: - pass - repo = Repo.clone_from(auth_url, repo_path) - self._ensure_gitflow_branches(repo, dashboard_id) - return repo - - logger.info(f"[init_repo][Action] Cloning {remote_url} to {repo_path}") - repo = Repo.clone_from(auth_url, repo_path) - self._ensure_gitflow_branches(repo, dashboard_id) - return repo - # [/DEF:init_repo:Function] - - # [DEF:delete_repo:Function] - # @PURPOSE: Remove local repository and DB binding for a dashboard. - # @PRE: dashboard_id is a valid integer. - # @POST: Local path is deleted when present and GitRepository row is removed. - # @RETURN: None - # @RELATION: CALLS -> [GitService._get_repo_path] - def delete_repo(self, dashboard_id: int) -> None: - with belief_scope("GitService.delete_repo"): - repo_path = self._get_repo_path(dashboard_id) - removed_files = False - if os.path.exists(repo_path): - if os.path.isdir(repo_path): - shutil.rmtree(repo_path) - else: - os.remove(repo_path) - removed_files = True - - session = SessionLocal() - try: - db_repo = ( - session.query(GitRepository) - .filter(GitRepository.dashboard_id == int(dashboard_id)) - .first() - ) - if db_repo: - session.delete(db_repo) - session.commit() - return - - if removed_files: - return - - raise HTTPException( - status_code=404, - detail=f"Repository for dashboard {dashboard_id} not found", - ) - except HTTPException: - session.rollback() - raise - except Exception as e: - session.rollback() - logger.error( - f"[delete_repo][Coherence:Failed] Failed to delete repository for dashboard {dashboard_id}: {e}" - ) - raise HTTPException(status_code=500, detail=f"Failed to delete repository: {str(e)}") - finally: - session.close() - # [/DEF:delete_repo:Function] - - # [DEF:get_repo:Function] - # @PURPOSE: Get Repo object for a dashboard. - # @PRE: Repository must exist on disk for the given dashboard_id. - # @POST: Returns a GitPython Repo instance for the dashboard. - # @RETURN: Repo - # @RELATION: CALLS -> [GitService._get_repo_path] - def get_repo(self, dashboard_id: int) -> Repo: - with belief_scope("GitService.get_repo"): - repo_path = self._get_repo_path(dashboard_id) - if not os.path.exists(repo_path): - logger.error(f"[get_repo][Coherence:Failed] Repository for dashboard {dashboard_id} does not exist") - raise HTTPException(status_code=404, detail=f"Repository for dashboard {dashboard_id} not found") - try: - return Repo(repo_path) - except Exception as e: - logger.error(f"[get_repo][Coherence:Failed] Failed to open repository at {repo_path}: {e}") - raise HTTPException(status_code=500, detail="Failed to open local Git repository") - # [/DEF:get_repo:Function] - - # [DEF:configure_identity:Function] - # @PURPOSE: Configure repository-local Git committer identity for user-scoped operations. - # @PRE: dashboard_id repository exists; git_username/git_email may be empty. - # @POST: Repository config has user.name and user.email when both identity values are provided. - # @RETURN: None - # @RELATION: CALLS -> [GitService.get_repo] - def configure_identity( - self, - dashboard_id: int, - git_username: Optional[str], - git_email: Optional[str], - ) -> None: - with belief_scope("GitService.configure_identity"): - normalized_username = str(git_username or "").strip() - normalized_email = str(git_email or "").strip() - if not normalized_username or not normalized_email: - return - - repo = self.get_repo(dashboard_id) - try: - with repo.config_writer(config_level="repository") as config_writer: - config_writer.set_value("user", "name", normalized_username) - config_writer.set_value("user", "email", normalized_email) - logger.info( - "[configure_identity][Action] Applied repository-local git identity for dashboard %s", - dashboard_id, - ) - except Exception as e: - logger.error(f"[configure_identity][Coherence:Failed] Failed to configure git identity: {e}") - raise HTTPException(status_code=500, detail=f"Failed to configure git identity: {str(e)}") - # [/DEF:configure_identity:Function] - - # [DEF:list_branches:Function] - # @PURPOSE: List all branches for a dashboard's repository. - # @PRE: Repository for dashboard_id exists. - # @POST: Returns a list of branch metadata dictionaries. - # @RETURN: List[dict] - # @RELATION: CALLS -> [GitService.get_repo] - def list_branches(self, dashboard_id: int) -> List[dict]: - with belief_scope("GitService.list_branches"): - repo = self.get_repo(dashboard_id) - logger.info(f"[list_branches][Action] Listing branches for {dashboard_id}. Refs: {repo.refs}") - branches = [] - - # Add existing refs - for ref in repo.refs: - try: - # Strip prefixes for UI - name = ref.name.replace('refs/heads/', '').replace('refs/remotes/origin/', '') - - # Avoid duplicates (e.g. local and remote with same name) - if any(b['name'] == name for b in branches): - continue - - branches.append({ - "name": name, - "commit_hash": ref.commit.hexsha if hasattr(ref, 'commit') else "0000000", - "is_remote": ref.is_remote() if hasattr(ref, 'is_remote') else False, - "last_updated": datetime.fromtimestamp(ref.commit.committed_date) if hasattr(ref, 'commit') else datetime.utcnow() - }) - except Exception as e: - logger.warning(f"[list_branches][Action] Skipping ref {ref}: {e}") - - # Ensure the current active branch is in the list even if it has no commits or refs - try: - active_name = repo.active_branch.name - if not any(b['name'] == active_name for b in branches): - branches.append({ - "name": active_name, - "commit_hash": "0000000", - "is_remote": False, - "last_updated": datetime.utcnow() - }) - except Exception as e: - logger.warning(f"[list_branches][Action] Could not determine active branch: {e}") - # If everything else failed and list is still empty, add default - if not branches: - branches.append({ - "name": "dev", - "commit_hash": "0000000", - "is_remote": False, - "last_updated": datetime.utcnow() - }) - - return branches - # [/DEF:list_branches:Function] - - # [DEF:create_branch:Function] - # @PURPOSE: Create a new branch from an existing one. - # @PARAM: name (str) - New branch name. - # @PARAM: from_branch (str) - Source branch. - # @PRE: Repository exists; name is valid; from_branch exists or repo is empty. - # @POST: A new branch is created in the repository. - # @RELATION: CALLS -> [GitService.get_repo] - def create_branch(self, dashboard_id: int, name: str, from_branch: str = "main"): - with belief_scope("GitService.create_branch"): - repo = self.get_repo(dashboard_id) - logger.info(f"[create_branch][Action] Creating branch {name} from {from_branch}") - - # Handle empty repository case (no commits) - if not repo.heads and not repo.remotes: - logger.warning("[create_branch][Action] Repository is empty. Creating initial commit to enable branching.") - readme_path = os.path.join(repo.working_dir, "README.md") - if not os.path.exists(readme_path): - with open(readme_path, "w") as f: - f.write(f"# Dashboard {dashboard_id}\nGit repository for Superset dashboard integration.") - repo.index.add(["README.md"]) - repo.index.commit("Initial commit") - - # Verify source branch exists - try: - repo.commit(from_branch) - except Exception: - logger.warning(f"[create_branch][Action] Source branch {from_branch} not found, using HEAD") - from_branch = repo.head - - try: - new_branch = repo.create_head(name, from_branch) - return new_branch - except Exception as e: - logger.error(f"[create_branch][Coherence:Failed] {e}") - raise - # [/DEF:create_branch:Function] - - # [DEF:checkout_branch:Function] - # @PURPOSE: Switch to a specific branch. - # @PRE: Repository exists and the specified branch name exists. - # @POST: The repository working directory is updated to the specified branch. - # @RELATION: CALLS -> [GitService.get_repo] - def checkout_branch(self, dashboard_id: int, name: str): - with belief_scope("GitService.checkout_branch"): - repo = self.get_repo(dashboard_id) - logger.info(f"[checkout_branch][Action] Checking out branch {name}") - repo.git.checkout(name) - # [/DEF:checkout_branch:Function] - - # [DEF:commit_changes:Function] - # @PURPOSE: Stage and commit changes. - # @PARAM: message (str) - Commit message. - # @PARAM: files (List[str]) - Optional list of specific files to stage. - # @PRE: Repository exists and has changes (dirty) or files are specified. - # @POST: Changes are staged and a new commit is created. - # @RELATION: CALLS -> [GitService.get_repo] - def commit_changes(self, dashboard_id: int, message: str, files: List[str] = None): - with belief_scope("GitService.commit_changes"): - repo = self.get_repo(dashboard_id) - - # Check if there are any changes to commit - if not repo.is_dirty(untracked_files=True) and not files: - logger.info(f"[commit_changes][Action] No changes to commit for dashboard {dashboard_id}") - return - - if files: - logger.info(f"[commit_changes][Action] Staging files: {files}") - repo.index.add(files) - else: - logger.info("[commit_changes][Action] Staging all changes") - repo.git.add(A=True) - - repo.index.commit(message) - logger.info(f"[commit_changes][Coherence:OK] Committed changes with message: {message}") - # [/DEF:commit_changes:Function] - - # [DEF:_extract_http_host:Function] - # @PURPOSE: Extract normalized host[:port] from HTTP(S) URL. - # @PRE: url_value may be empty. - # @POST: Returns lowercase host token or None. - # @RETURN: Optional[str] - # @RELATION: USES -> [urlparse] - def _extract_http_host(self, url_value: Optional[str]) -> Optional[str]: - normalized = str(url_value or "").strip() - if not normalized: - return None - try: - parsed = urlparse(normalized) - except Exception: - return None - if parsed.scheme not in {"http", "https"}: - return None - host = parsed.hostname - if not host: - return None - if parsed.port: - return f"{host.lower()}:{parsed.port}" - return host.lower() - # [/DEF:_extract_http_host:Function] - - # [DEF:_strip_url_credentials:Function] - # @PURPOSE: Remove credentials from URL while preserving scheme/host/path. - # @PRE: url_value may contain credentials. - # @POST: Returns URL without username/password. - # @RETURN: str - # @RELATION: USES -> [urlparse] - def _strip_url_credentials(self, url_value: str) -> str: - normalized = str(url_value or "").strip() - if not normalized: - return normalized - try: - parsed = urlparse(normalized) - except Exception: - return normalized - if parsed.scheme not in {"http", "https"} or not parsed.hostname: - return normalized - host = parsed.hostname - if parsed.port: - host = f"{host}:{parsed.port}" - return parsed._replace(netloc=host).geturl() - # [/DEF:_strip_url_credentials:Function] - - # [DEF:_replace_host_in_url:Function] - # @PURPOSE: Replace source URL host with host from configured server URL. - # @PRE: source_url and config_url are HTTP(S) URLs. - # @POST: Returns source URL with updated host (credentials preserved) or None. - # @RETURN: Optional[str] - # @RELATION: USES -> [urlparse] - def _replace_host_in_url(self, source_url: Optional[str], config_url: Optional[str]) -> Optional[str]: - source = str(source_url or "").strip() - config = str(config_url or "").strip() - if not source or not config: - return None - try: - source_parsed = urlparse(source) - config_parsed = urlparse(config) - except Exception: - return None - - if source_parsed.scheme not in {"http", "https"}: - return None - if config_parsed.scheme not in {"http", "https"}: - return None - if not source_parsed.hostname or not config_parsed.hostname: - return None - - target_host = config_parsed.hostname - if config_parsed.port: - target_host = f"{target_host}:{config_parsed.port}" - - auth_part = "" - if source_parsed.username: - auth_part = quote(source_parsed.username, safe="") - if source_parsed.password is not None: - auth_part = f"{auth_part}:{quote(source_parsed.password, safe='')}" - auth_part = f"{auth_part}@" - - new_netloc = f"{auth_part}{target_host}" - return source_parsed._replace(netloc=new_netloc).geturl() - # [/DEF:_replace_host_in_url:Function] - - # [DEF:_align_origin_host_with_config:Function] - # @PURPOSE: Auto-align local origin host to configured Git server host when they drift. - # @PRE: origin remote exists. - # @POST: origin URL host updated and DB binding normalized when mismatch detected. - # @RETURN: Optional[str] - # @RELATION: CALLS -> [GitService._extract_http_host] - # @RELATION: CALLS -> [GitService._replace_host_in_url] - # @RELATION: CALLS -> [GitService._strip_url_credentials] - def _align_origin_host_with_config( - self, - dashboard_id: int, - origin, - config_url: Optional[str], - current_origin_url: Optional[str], - binding_remote_url: Optional[str], - ) -> Optional[str]: - config_host = self._extract_http_host(config_url) - source_origin_url = str(current_origin_url or "").strip() or str(binding_remote_url or "").strip() - origin_host = self._extract_http_host(source_origin_url) - - if not config_host or not origin_host: - return None - if config_host == origin_host: - return None - - aligned_url = self._replace_host_in_url(source_origin_url, config_url) - if not aligned_url: - return None - - logger.warning( - "[_align_origin_host_with_config][Action] Host mismatch for dashboard %s: config_host=%s origin_host=%s, applying origin.set_url", - dashboard_id, - config_host, - origin_host, - ) - - try: - origin.set_url(aligned_url) - except Exception as e: - logger.warning( - "[_align_origin_host_with_config][Coherence:Failed] Failed to set origin URL for dashboard %s: %s", - dashboard_id, - e, - ) - return None - - try: - session = SessionLocal() - try: - db_repo = ( - session.query(GitRepository) - .filter(GitRepository.dashboard_id == int(dashboard_id)) - .first() - ) - if db_repo: - db_repo.remote_url = self._strip_url_credentials(aligned_url) - session.commit() - finally: - session.close() - except Exception as e: - logger.warning( - "[_align_origin_host_with_config][Action] Failed to persist aligned remote_url for dashboard %s: %s", - dashboard_id, - e, - ) - - return aligned_url - # [/DEF:_align_origin_host_with_config:Function] - - # [DEF:push_changes:Function] - # @PURPOSE: Push local commits to remote. - # @PRE: Repository exists and has an 'origin' remote. - # @POST: Local branch commits are pushed to origin. - # @RELATION: CALLS -> [GitService.get_repo] - # @RELATION: CALLS -> [GitService._align_origin_host_with_config] - def push_changes(self, dashboard_id: int): - with belief_scope("GitService.push_changes"): - repo = self.get_repo(dashboard_id) - - # Ensure we have something to push - if not repo.heads: - logger.warning(f"[push_changes][Coherence:Failed] No local branches to push for dashboard {dashboard_id}") - return - - try: - origin = repo.remote(name='origin') - except ValueError: - logger.error(f"[push_changes][Coherence:Failed] Remote 'origin' not found for dashboard {dashboard_id}") - raise HTTPException(status_code=400, detail="Remote 'origin' not configured") - - # Emit diagnostic context to verify config-url vs repository-origin mismatch. - try: - origin_urls = list(origin.urls) - except Exception: - origin_urls = [] - binding_remote_url = None - binding_config_id = None - binding_config_url = None - try: - session = SessionLocal() - try: - db_repo = ( - session.query(GitRepository) - .filter(GitRepository.dashboard_id == int(dashboard_id)) - .first() - ) - if db_repo: - binding_remote_url = db_repo.remote_url - binding_config_id = db_repo.config_id - db_config = ( - session.query(GitServerConfig) - .filter(GitServerConfig.id == db_repo.config_id) - .first() - ) - if db_config: - binding_config_url = db_config.url - finally: - session.close() - except Exception as diag_error: - logger.warning( - "[push_changes][Action] Failed to load repository binding diagnostics for dashboard %s: %s", - dashboard_id, - diag_error, - ) - - realigned_origin_url = self._align_origin_host_with_config( - dashboard_id=dashboard_id, - origin=origin, - config_url=binding_config_url, - current_origin_url=(origin_urls[0] if origin_urls else None), - binding_remote_url=binding_remote_url, - ) - try: - origin_urls = list(origin.urls) - except Exception: - origin_urls = [] - - logger.info( - "[push_changes][Action] Push diagnostics dashboard=%s config_id=%s config_url=%s binding_remote_url=%s origin_urls=%s origin_realigned=%s", - dashboard_id, - binding_config_id, - binding_config_url, - binding_remote_url, - origin_urls, - bool(realigned_origin_url), - ) - - # Check if current branch has an upstream - try: - current_branch = repo.active_branch - logger.info(f"[push_changes][Action] Pushing branch {current_branch.name} to origin") - tracking_branch = None - try: - tracking_branch = current_branch.tracking_branch() - except Exception: - tracking_branch = None - - # First push for a new branch must set upstream, otherwise future pull fails. - if tracking_branch is None: - repo.git.push("--set-upstream", "origin", f"{current_branch.name}:{current_branch.name}") - else: - push_info = origin.push(refspec=f'{current_branch.name}:{current_branch.name}') - for info in push_info: - if info.flags & info.ERROR: - logger.error(f"[push_changes][Coherence:Failed] Error pushing ref {info.remote_ref_string}: {info.summary}") - raise Exception(f"Git push error for {info.remote_ref_string}: {info.summary}") - except GitCommandError as e: - details = str(e) - lowered = details.lower() - if "non-fast-forward" in lowered or "rejected" in lowered: - raise HTTPException( - status_code=409, - detail=( - "Push rejected: remote branch contains newer commits. " - "Run Pull first, resolve conflicts if any, then push again." - ), - ) - logger.error(f"[push_changes][Coherence:Failed] Failed to push changes: {e}") - raise HTTPException(status_code=500, detail=f"Git push failed: {details}") - except Exception as e: - logger.error(f"[push_changes][Coherence:Failed] Failed to push changes: {e}") - raise HTTPException(status_code=500, detail=f"Git push failed: {str(e)}") - # [/DEF:push_changes:Function] - - # [DEF:_read_blob_text:Function] - # @PURPOSE: Read text from a Git blob. - # @RELATION: USES -> [Blob] - def _read_blob_text(self, blob: Blob) -> str: - with belief_scope("GitService._read_blob_text"): - if blob is None: - return "" - try: - return blob.data_stream.read().decode("utf-8", errors="replace") - except Exception: - return "" - # [/DEF:_read_blob_text:Function] - - # [DEF:_get_unmerged_file_paths:Function] - # @PURPOSE: List files with merge conflicts. - # @RELATION: USES -> [Repo] - def _get_unmerged_file_paths(self, repo: Repo) -> List[str]: - with belief_scope("GitService._get_unmerged_file_paths"): - try: - return sorted(list(repo.index.unmerged_blobs().keys())) - except Exception: - return [] - # [/DEF:_get_unmerged_file_paths:Function] - - # [DEF:_build_unfinished_merge_payload:Function] - # @PURPOSE: Build payload for unfinished merge state. - # @RELATION: CALLS -> [GitService._get_unmerged_file_paths] - def _build_unfinished_merge_payload(self, repo: Repo) -> Dict[str, Any]: - with belief_scope("GitService._build_unfinished_merge_payload"): - merge_head_path = os.path.join(repo.git_dir, "MERGE_HEAD") - merge_head_value = "" - merge_msg_preview = "" - current_branch = "unknown" - try: - merge_head_value = Path(merge_head_path).read_text(encoding="utf-8").strip() - except Exception: - merge_head_value = "" - try: - merge_msg_path = os.path.join(repo.git_dir, "MERGE_MSG") - if os.path.exists(merge_msg_path): - merge_msg_preview = ( - Path(merge_msg_path).read_text(encoding="utf-8").strip().splitlines()[:1] or [""] - )[0] - except Exception: - merge_msg_preview = "" - try: - current_branch = repo.active_branch.name - except Exception: - current_branch = "detached_or_unknown" - - conflicts_count = len(self._get_unmerged_file_paths(repo)) - return { - "error_code": "GIT_UNFINISHED_MERGE", - "message": ( - "В репозитории есть незавершённое слияние. " - "Завершите или отмените слияние вручную." - ), - "repository_path": repo.working_tree_dir, - "git_dir": repo.git_dir, - "current_branch": current_branch, - "merge_head": merge_head_value, - "merge_message_preview": merge_msg_preview, - "conflicts_count": conflicts_count, - "next_steps": [ - "Откройте локальный репозиторий по пути repository_path", - "Проверьте состояние: git status", - "Разрешите конфликты и выполните commit, либо отмените: git merge --abort", - "После завершения/отмены слияния повторите Pull из интерфейса", - ], - "manual_commands": [ - "git status", - "git add ", - "git commit -m \"resolve merge conflicts\"", - "git merge --abort", - ], - } - # [/DEF:_build_unfinished_merge_payload:Function] - - # [DEF:get_merge_status:Function] - # @PURPOSE: Get current merge status for a dashboard repository. - # @RELATION: CALLS -> [GitService.get_repo] - # @RELATION: CALLS -> [GitService._build_unfinished_merge_payload] - def get_merge_status(self, dashboard_id: int) -> Dict[str, Any]: - with belief_scope("GitService.get_merge_status"): - repo = self.get_repo(dashboard_id) - merge_head_path = os.path.join(repo.git_dir, "MERGE_HEAD") - if not os.path.exists(merge_head_path): - current_branch = "unknown" - try: - current_branch = repo.active_branch.name - except Exception: - current_branch = "detached_or_unknown" - return { - "has_unfinished_merge": False, - "repository_path": repo.working_tree_dir, - "git_dir": repo.git_dir, - "current_branch": current_branch, - "merge_head": None, - "merge_message_preview": None, - "conflicts_count": 0, - } - payload = self._build_unfinished_merge_payload(repo) - return { - "has_unfinished_merge": True, - "repository_path": payload["repository_path"], - "git_dir": payload["git_dir"], - "current_branch": payload["current_branch"], - "merge_head": payload["merge_head"], - "merge_message_preview": payload["merge_message_preview"], - "conflicts_count": int(payload.get("conflicts_count") or 0), - } - # [/DEF:get_merge_status:Function] - - # [DEF:get_merge_conflicts:Function] - # @PURPOSE: List all files with conflicts and their contents. - # @RELATION: CALLS -> [GitService.get_repo] - # @RELATION: CALLS -> [GitService._read_blob_text] - def get_merge_conflicts(self, dashboard_id: int) -> List[Dict[str, Any]]: - with belief_scope("GitService.get_merge_conflicts"): - repo = self.get_repo(dashboard_id) - conflicts = [] - unmerged = repo.index.unmerged_blobs() - for file_path, stages in unmerged.items(): - mine_blob = None - theirs_blob = None - for stage, blob in stages: - if stage == 2: - mine_blob = blob - elif stage == 3: - theirs_blob = blob - conflicts.append( - { - "file_path": file_path, - "mine": self._read_blob_text(mine_blob) if mine_blob else "", - "theirs": self._read_blob_text(theirs_blob) if theirs_blob else "", - } - ) - return sorted(conflicts, key=lambda item: item["file_path"]) - # [/DEF:get_merge_conflicts:Function] - - # [DEF:resolve_merge_conflicts:Function] - # @PURPOSE: Resolve conflicts using specified strategy. - # @RELATION: CALLS -> [GitService.get_repo] - def resolve_merge_conflicts(self, dashboard_id: int, resolutions: List[Dict[str, Any]]) -> List[str]: - with belief_scope("GitService.resolve_merge_conflicts"): - repo = self.get_repo(dashboard_id) - resolved_files: List[str] = [] - repo_root = os.path.abspath(str(repo.working_tree_dir or "")) - if not repo_root: - raise HTTPException(status_code=500, detail="Repository working tree directory is unavailable") - - for item in resolutions or []: - file_path = str(item.get("file_path") or "").strip() - strategy = str(item.get("resolution") or "").strip().lower() - content = item.get("content") - if not file_path: - raise HTTPException(status_code=400, detail="resolution.file_path is required") - if strategy not in {"mine", "theirs", "manual"}: - raise HTTPException(status_code=400, detail=f"Unsupported resolution strategy: {strategy}") - - if strategy == "mine": - repo.git.checkout("--ours", "--", file_path) - elif strategy == "theirs": - repo.git.checkout("--theirs", "--", file_path) - else: - abs_target = os.path.abspath(os.path.join(repo_root, file_path)) - if abs_target != repo_root and not abs_target.startswith(repo_root + os.sep): - raise HTTPException(status_code=400, detail=f"Invalid conflict file path: {file_path}") - os.makedirs(os.path.dirname(abs_target), exist_ok=True) - with open(abs_target, "w", encoding="utf-8") as file_obj: - file_obj.write(str(content or "")) - - repo.git.add(file_path) - resolved_files.append(file_path) - - return resolved_files - # [/DEF:resolve_merge_conflicts:Function] - - # [DEF:abort_merge:Function] - # @PURPOSE: Abort ongoing merge. - # @RELATION: CALLS -> [GitService.get_repo] - def abort_merge(self, dashboard_id: int) -> Dict[str, Any]: - with belief_scope("GitService.abort_merge"): - repo = self.get_repo(dashboard_id) - try: - repo.git.merge("--abort") - except GitCommandError as e: - details = str(e) - lowered = details.lower() - if "there is no merge to abort" in lowered or "no merge to abort" in lowered: - return {"status": "no_merge_in_progress"} - raise HTTPException(status_code=409, detail=f"Cannot abort merge: {details}") - return {"status": "aborted"} - # [/DEF:abort_merge:Function] - - # [DEF:continue_merge:Function] - # @PURPOSE: Finalize merge after conflict resolution. - # @RELATION: CALLS -> [GitService.get_repo] - # @RELATION: CALLS -> [GitService._get_unmerged_file_paths] - def continue_merge(self, dashboard_id: int, message: Optional[str] = None) -> Dict[str, Any]: - with belief_scope("GitService.continue_merge"): - repo = self.get_repo(dashboard_id) - unmerged_files = self._get_unmerged_file_paths(repo) - if unmerged_files: - raise HTTPException( - status_code=409, - detail={ - "error_code": "GIT_MERGE_CONFLICTS_REMAIN", - "message": "Невозможно завершить merge: остались неразрешённые конфликты.", - "unresolved_files": unmerged_files, - }, - ) - try: - normalized_message = str(message or "").strip() - if normalized_message: - repo.git.commit("-m", normalized_message) - else: - repo.git.commit("--no-edit") - except GitCommandError as e: - details = str(e) - lowered = details.lower() - if "nothing to commit" in lowered: - return {"status": "already_clean"} - raise HTTPException(status_code=409, detail=f"Cannot continue merge: {details}") - - commit_hash = "" - try: - commit_hash = repo.head.commit.hexsha - except Exception: - commit_hash = "" - return {"status": "committed", "commit_hash": commit_hash} - # [/DEF:continue_merge:Function] - - # [DEF:pull_changes:Function] - # @PURPOSE: Pull changes from remote. - # @PRE: Repository exists and has an 'origin' remote. - # @POST: Changes from origin are pulled and merged into the active branch. - # @RELATION: CALLS -> [GitService.get_repo] - # @RELATION: CALLS -> [GitService._build_unfinished_merge_payload] - def pull_changes(self, dashboard_id: int): - with belief_scope("GitService.pull_changes"): - repo = self.get_repo(dashboard_id) - - # Check for unfinished merge (MERGE_HEAD exists) - merge_head_path = os.path.join(repo.git_dir, "MERGE_HEAD") - if os.path.exists(merge_head_path): - payload = self._build_unfinished_merge_payload(repo) - - logger.warning( - "[pull_changes][Action] Unfinished merge detected for dashboard %s " - "(repo_path=%s git_dir=%s branch=%s merge_head=%s merge_msg=%s)", - dashboard_id, - payload["repository_path"], - payload["git_dir"], - payload["current_branch"], - payload["merge_head"], - payload["merge_message_preview"], - ) - raise HTTPException(status_code=409, detail=payload) - - try: - origin = repo.remote(name='origin') - current_branch = repo.active_branch.name - try: - origin_urls = list(origin.urls) - except Exception: - origin_urls = [] - - logger.info( - "[pull_changes][Action] Pull diagnostics dashboard=%s repo_path=%s branch=%s origin_urls=%s", - dashboard_id, - repo.working_tree_dir, - current_branch, - origin_urls, - ) - - origin.fetch(prune=True) - remote_ref = f"origin/{current_branch}" - has_remote_branch = any(ref.name == remote_ref for ref in repo.refs) - logger.info( - "[pull_changes][Action] Pull remote branch check dashboard=%s branch=%s remote_ref=%s exists=%s", - dashboard_id, - current_branch, - remote_ref, - has_remote_branch, - ) - if not has_remote_branch: - raise HTTPException( - status_code=409, - detail=f"Remote branch '{current_branch}' does not exist yet. Push this branch first.", - ) - - logger.info(f"[pull_changes][Action] Pulling changes from origin/{current_branch}") - # Force deterministic merge strategy for modern git versions. - repo.git.pull("--no-rebase", "origin", current_branch) - except ValueError: - logger.error(f"[pull_changes][Coherence:Failed] Remote 'origin' not found for dashboard {dashboard_id}") - raise HTTPException(status_code=400, detail="Remote 'origin' not configured") - except GitCommandError as e: - details = str(e) - lowered = details.lower() - if "conflict" in lowered or "not possible to fast-forward" in lowered: - raise HTTPException( - status_code=409, - detail=( - "Pull requires conflict resolution. Resolve conflicts in repository " - "and repeat operation." - ), - ) - logger.error(f"[pull_changes][Coherence:Failed] Failed to pull changes: {e}") - raise HTTPException(status_code=500, detail=f"Git pull failed: {details}") - except HTTPException: - raise - except Exception as e: - logger.error(f"[pull_changes][Coherence:Failed] Failed to pull changes: {e}") - raise HTTPException(status_code=500, detail=f"Git pull failed: {str(e)}") - # [/DEF:pull_changes:Function] - - # [DEF:_parse_status_porcelain:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Parse git status --porcelain output into staged, modified, and untracked file lists. - # @PRE: `repo` is an open GitPython Repo instance. - # @POST: Returns (staged, modified, untracked) tuple of file path lists. - # @RATIONALE: Avoids repo.is_dirty() / repo.index.diff("HEAD") which internally - # call git diff --cached, a flag unsupported in some Git environments (exit 129). - # Using git status --porcelain is self-contained and avoids the --cached flag entirely. - # @RELATION: CALLED_BY -> [GitService.get_status] - def _parse_status_porcelain(self, repo) -> tuple[list[str], list[str], list[str]]: - with belief_scope("GitService._parse_status_porcelain"): - staged: list[str] = [] - modified: list[str] = [] - untracked: list[str] = [] - try: - output = repo.git.status("--porcelain") - except Exception: - logger.warning("[_parse_status_porcelain][Coherence:Failed] git status --porcelain failed") - return staged, modified, untracked - - for line in output.split("\n"): - # Do NOT strip the line — porcelain v1 format "X Y PATH" uses - # leading space (X=' ') to indicate "unmodified in index". - # Stripping would shift column alignment. - if not line: - continue - # Untracked: "?? path" - if line.startswith("??"): - untracked.append(line[2:].strip()) - continue - # Ignored: "!! path" (skip) - if line.startswith("!!"): - continue - # Normal entry: "XY path" or "XY orig -> dest" for renames - if len(line) < 3: - continue - x = line[0] # index (staged) status - y = line[1] # work-tree status - rest = line[3:] # strip "XY " - # For renames/copies: "R orig -> dest" — use the destination - path = rest.split(" -> ")[-1].strip() if " -> " in rest else rest.strip() - if x != " " and x != "?": - staged.append(path) - if y != " " and y != "?": - if path not in modified: - modified.append(path) - return staged, modified, untracked - # [/DEF:_parse_status_porcelain:Function] - - # [DEF:get_status:Function] - # @PURPOSE: Get current repository status (dirty files, untracked, etc.) - # @PRE: Repository for dashboard_id exists. - # @POST: Returns a dictionary representing the Git status. - # @RETURN: dict - # @RELATION: CALLS -> [GitService.get_repo] - # @RELATION: CALLS -> [GitService._parse_status_porcelain] - def get_status(self, dashboard_id: int) -> dict: - with belief_scope("GitService.get_status"): - repo = self.get_repo(dashboard_id) - - # Handle empty repository (no commits) - has_commits = False - try: - repo.head.commit - has_commits = True - except (ValueError, Exception): - has_commits = False - - current_branch = repo.active_branch.name - tracking_branch = None - has_upstream = False - ahead_count = 0 - behind_count = 0 - - try: - tracking_branch = repo.active_branch.tracking_branch() - has_upstream = tracking_branch is not None - except Exception: - tracking_branch = None - has_upstream = False - - if has_upstream and tracking_branch is not None: - try: - # Commits present locally but not in upstream. - ahead_count = sum( - 1 for _ in repo.iter_commits(f"{tracking_branch.name}..{current_branch}") - ) - # Commits present in upstream but not local. - behind_count = sum( - 1 for _ in repo.iter_commits(f"{current_branch}..{tracking_branch.name}") - ) - except Exception: - ahead_count = 0 - behind_count = 0 - - # Use git status --porcelain to gather file state safely. - # Avoids repo.is_dirty() and repo.index.diff("HEAD") which internally - # call git diff --cached -- a flag unsupported in some Git environments. - staged_files, modified_files, untracked_files = self._parse_status_porcelain(repo) - is_dirty = bool(staged_files or modified_files or untracked_files) - is_diverged = ahead_count > 0 and behind_count > 0 - - if is_diverged: - sync_state = "DIVERGED" - elif behind_count > 0: - sync_state = "BEHIND_REMOTE" - elif ahead_count > 0: - sync_state = "AHEAD_REMOTE" - elif is_dirty or modified_files or staged_files or untracked_files: - sync_state = "CHANGES" - else: - sync_state = "SYNCED" - - return { - "is_dirty": is_dirty, - "untracked_files": untracked_files, - "modified_files": modified_files, - "staged_files": staged_files, - "current_branch": current_branch, - "upstream_branch": tracking_branch.name if tracking_branch is not None else None, - "has_upstream": has_upstream, - "ahead_count": ahead_count, - "behind_count": behind_count, - "is_diverged": is_diverged, - "sync_state": sync_state, - } - # [/DEF:get_status:Function] - - # [DEF:get_diff:Function] - # @PURPOSE: Generate diff for a file or the whole repository. - # @PARAM: file_path (str) - Optional specific file. - # @PARAM: staged (bool) - Whether to show staged changes. - # @PRE: Repository for dashboard_id exists. - # @POST: Returns the diff text as a string. - # @RETURN: str - # @RELATION: CALLS -> [GitService.get_repo] - def get_diff(self, dashboard_id: int, file_path: str = None, staged: bool = False) -> str: - with belief_scope("GitService.get_diff"): - repo = self.get_repo(dashboard_id) - diff_args = [] - if staged: - diff_args.append("--staged") - - if file_path: - return repo.git.diff(*diff_args, "--", file_path) - return repo.git.diff(*diff_args) - # [/DEF:get_diff:Function] - - # [DEF:get_commit_history:Function] - # @PURPOSE: Retrieve commit history for a repository. - # @PARAM: limit (int) - Max number of commits to return. - # @PRE: Repository for dashboard_id exists. - # @POST: Returns a list of dictionaries for each commit in history. - # @RETURN: List[dict] - # @RELATION: CALLS -> [GitService.get_repo] - def get_commit_history(self, dashboard_id: int, limit: int = 50) -> List[dict]: - with belief_scope("GitService.get_commit_history"): - repo = self.get_repo(dashboard_id) - commits = [] - try: - # Check if there are any commits at all - if not repo.heads and not repo.remotes: - return [] - - for commit in repo.iter_commits(max_count=limit): - commits.append({ - "hash": commit.hexsha, - "author": commit.author.name, - "email": commit.author.email, - "timestamp": datetime.fromtimestamp(commit.committed_date), - "message": commit.message.strip(), - "files_changed": list(commit.stats.files.keys()) - }) - except Exception as e: - logger.warning(f"[get_commit_history][Action] Could not retrieve commit history for dashboard {dashboard_id}: {e}") - return [] - return commits - # [/DEF:get_commit_history:Function] - - # [DEF:test_connection:Function] - # @PURPOSE: Test connection to Git provider using PAT. - # @PARAM: provider (GitProvider) - # @PARAM: url (str) - # @PARAM: pat (str) - # @PRE: provider is valid; url is a valid HTTP(S) URL; pat is provided. - # @POST: Returns True if connection to the provider's API succeeds. - # @RETURN: bool - # @RELATION: USES -> [httpx.AsyncClient] - async def test_connection(self, provider: GitProvider, url: str, pat: str) -> bool: - with belief_scope("GitService.test_connection"): - # Check for offline mode or local-only URLs - if ".local" in url or "localhost" in url: - logger.info("[test_connection][Action] Local/Offline mode detected for URL") - return True - - if not url.startswith(('http://', 'https://')): - logger.error(f"[test_connection][Coherence:Failed] Invalid URL protocol: {url}") - return False - - if not pat or not pat.strip(): - logger.error("[test_connection][Coherence:Failed] Git PAT is missing or empty") - return False - - pat = pat.strip() - - try: - async with httpx.AsyncClient() as client: - if provider == GitProvider.GITHUB: - headers = {"Authorization": f"token {pat}"} - api_url = "https://api.github.com/user" if "github.com" in url else f"{url.rstrip('/')}/api/v3/user" - resp = await client.get(api_url, headers=headers) - elif provider == GitProvider.GITLAB: - headers = {"PRIVATE-TOKEN": pat} - api_url = f"{url.rstrip('/')}/api/v4/user" - resp = await client.get(api_url, headers=headers) - elif provider == GitProvider.GITEA: - headers = {"Authorization": f"token {pat}"} - api_url = f"{url.rstrip('/')}/api/v1/user" - resp = await client.get(api_url, headers=headers) - else: - return False - - if resp.status_code != 200: - logger.error(f"[test_connection][Coherence:Failed] Git connection test failed for {provider} at {api_url}. Status: {resp.status_code}") - return resp.status_code == 200 - except Exception as e: - logger.error(f"[test_connection][Coherence:Failed] Error testing git connection: {e}") - return False - # [/DEF:test_connection:Function] - - # [DEF:_normalize_git_server_url:Function] - # @PURPOSE: Normalize Git server URL for provider API calls. - # @PRE: raw_url is non-empty. - # @POST: Returns URL without trailing slash. - # @RETURN: str - def _normalize_git_server_url(self, raw_url: str) -> str: - normalized = (raw_url or "").strip() - if not normalized: - raise HTTPException(status_code=400, detail="Git server URL is required") - return normalized.rstrip("/") - # [/DEF:_normalize_git_server_url:Function] - - # [DEF:_gitea_headers:Function] - # @PURPOSE: Build Gitea API authorization headers. - # @PRE: pat is provided. - # @POST: Returns headers with token auth. - # @RETURN: Dict[str, str] - def _gitea_headers(self, pat: str) -> Dict[str, str]: - token = (pat or "").strip() - if not token: - raise HTTPException(status_code=400, detail="Git PAT is required for Gitea operations") - return { - "Authorization": f"token {token}", - "Content-Type": "application/json", - "Accept": "application/json", - } - # [/DEF:_gitea_headers:Function] - - # [DEF:_gitea_request:Function] - # @PURPOSE: Execute HTTP request against Gitea API with stable error mapping. - # @PRE: method and endpoint are valid. - # @POST: Returns decoded JSON payload. - # @RETURN: Any - # @RELATION: CALLS -> [GitService._normalize_git_server_url] - # @RELATION: CALLS -> [GitService._gitea_headers] - async def _gitea_request( - self, - method: str, - server_url: str, - pat: str, - endpoint: str, - payload: Optional[Dict[str, Any]] = None, - ) -> Any: - base_url = self._normalize_git_server_url(server_url) - url = f"{base_url}/api/v1{endpoint}" - headers = self._gitea_headers(pat) - try: - async with httpx.AsyncClient(timeout=20.0) as client: - response = await client.request( - method=method, - url=url, - headers=headers, - json=payload, - ) - except Exception as e: - logger.error(f"[gitea_request][Coherence:Failed] Network error: {e}") - raise HTTPException(status_code=503, detail=f"Gitea API is unavailable: {str(e)}") - - if response.status_code >= 400: - detail = response.text - try: - parsed = response.json() - detail = parsed.get("message") or parsed.get("error") or detail - except Exception: - pass - logger.error( - f"[gitea_request][Coherence:Failed] method={method} endpoint={endpoint} status={response.status_code} detail={detail}" - ) - raise HTTPException( - status_code=response.status_code, - detail=f"Gitea API error: {detail}", - ) - - if response.status_code == 204: - return None - return response.json() - # [/DEF:_gitea_request:Function] - - # [DEF:get_gitea_current_user:Function] - # @PURPOSE: Resolve current Gitea user for PAT. - # @PRE: server_url and pat are valid. - # @POST: Returns current username. - # @RETURN: str - # @RELATION: CALLS -> [GitService._gitea_request] - async def get_gitea_current_user(self, server_url: str, pat: str) -> str: - payload = await self._gitea_request("GET", server_url, pat, "/user") - username = payload.get("login") or payload.get("username") - if not username: - raise HTTPException(status_code=500, detail="Failed to resolve Gitea username") - return str(username) - # [/DEF:get_gitea_current_user:Function] - - # [DEF:list_gitea_repositories:Function] - # @PURPOSE: List repositories visible to authenticated Gitea user. - # @PRE: server_url and pat are valid. - # @POST: Returns repository list from Gitea. - # @RETURN: List[dict] - # @RELATION: CALLS -> [GitService._gitea_request] - async def list_gitea_repositories(self, server_url: str, pat: str) -> List[dict]: - payload = await self._gitea_request( - "GET", - server_url, - pat, - "/user/repos?limit=100&page=1", - ) - if not isinstance(payload, list): - return [] - return payload - # [/DEF:list_gitea_repositories:Function] - - # [DEF:create_gitea_repository:Function] - # @PURPOSE: Create repository in Gitea for authenticated user. - # @PRE: name is non-empty and PAT has repo creation permission. - # @POST: Returns created repository payload. - # @RETURN: dict - # @RELATION: CALLS -> [GitService._gitea_request] - async def create_gitea_repository( - self, - server_url: str, - pat: str, - name: str, - private: bool = True, - description: Optional[str] = None, - auto_init: bool = True, - default_branch: Optional[str] = "main", - ) -> Dict[str, Any]: - payload = { - "name": name, - "private": bool(private), - "auto_init": bool(auto_init), - } - if description: - payload["description"] = description - if default_branch: - payload["default_branch"] = default_branch - created = await self._gitea_request( - "POST", - server_url, - pat, - "/user/repos", - payload=payload, - ) - if not isinstance(created, dict): - raise HTTPException(status_code=500, detail="Unexpected Gitea response while creating repository") - return created - # [/DEF:create_gitea_repository:Function] - - # [DEF:delete_gitea_repository:Function] - # @PURPOSE: Delete repository in Gitea. - # @PRE: owner and repo_name are non-empty. - # @POST: Repository deleted on Gitea server. - # @RELATION: CALLS -> [GitService._gitea_request] - async def delete_gitea_repository( - self, - server_url: str, - pat: str, - owner: str, - repo_name: str, - ) -> None: - if not owner or not repo_name: - raise HTTPException(status_code=400, detail="owner and repo_name are required") - await self._gitea_request( - "DELETE", - server_url, - pat, - f"/repos/{owner}/{repo_name}", - ) - # [/DEF:delete_gitea_repository:Function] - - # [DEF:_gitea_branch_exists:Function] - # @PURPOSE: Check whether a branch exists in Gitea repository. - # @PRE: owner/repo/branch are non-empty. - # @POST: Returns True when branch exists, False when 404. - # @RETURN: bool - # @RELATION: CALLS -> [GitService._gitea_request] - async def _gitea_branch_exists( - self, - server_url: str, - pat: str, - owner: str, - repo: str, - branch: str, - ) -> bool: - if not owner or not repo or not branch: - return False - endpoint = f"/repos/{owner}/{repo}/branches/{quote(branch, safe='')}" - try: - await self._gitea_request("GET", server_url, pat, endpoint) - return True - except HTTPException as exc: - if exc.status_code == 404: - return False - raise - # [/DEF:_gitea_branch_exists:Function] - - # [DEF:_build_gitea_pr_404_detail:Function] - # @PURPOSE: Build actionable error detail for Gitea PR 404 responses. - # @PRE: owner/repo/from_branch/to_branch are provided. - # @POST: Returns specific branch-missing message when detected. - # @RETURN: Optional[str] - # @RELATION: CALLS -> [GitService._gitea_branch_exists] - async def _build_gitea_pr_404_detail( - self, - server_url: str, - pat: str, - owner: str, - repo: str, - from_branch: str, - to_branch: str, - ) -> Optional[str]: - source_exists = await self._gitea_branch_exists( - server_url=server_url, - pat=pat, - owner=owner, - repo=repo, - branch=from_branch, - ) - target_exists = await self._gitea_branch_exists( - server_url=server_url, - pat=pat, - owner=owner, - repo=repo, - branch=to_branch, - ) - if not source_exists: - return f"Gitea branch not found: source branch '{from_branch}' in {owner}/{repo}" - if not target_exists: - return f"Gitea branch not found: target branch '{to_branch}' in {owner}/{repo}" - return None - # [/DEF:_build_gitea_pr_404_detail:Function] - - # [DEF:create_github_repository:Function] - # @PURPOSE: Create repository in GitHub or GitHub Enterprise. - # @PRE: PAT has repository create permission. - # @POST: Returns created repository payload. - # @RETURN: dict - # @RELATION: CALLS -> [GitService._normalize_git_server_url] - async def create_github_repository( - self, - server_url: str, - pat: str, - name: str, - private: bool = True, - description: Optional[str] = None, - auto_init: bool = True, - default_branch: Optional[str] = "main", - ) -> Dict[str, Any]: - base_url = self._normalize_git_server_url(server_url) - if "github.com" in base_url: - api_url = "https://api.github.com/user/repos" - else: - api_url = f"{base_url}/api/v3/user/repos" - headers = { - "Authorization": f"token {pat.strip()}", - "Content-Type": "application/json", - "Accept": "application/vnd.github+json", - } - payload: Dict[str, Any] = { - "name": name, - "private": bool(private), - "auto_init": bool(auto_init), - } - if description: - payload["description"] = description - # GitHub API does not reliably support setting default branch on create without template/import. - if default_branch: - payload["default_branch"] = default_branch - try: - async with httpx.AsyncClient(timeout=20.0) as client: - response = await client.post(api_url, headers=headers, json=payload) - except Exception as e: - raise HTTPException(status_code=503, detail=f"GitHub API is unavailable: {str(e)}") - - if response.status_code >= 400: - detail = response.text - try: - parsed = response.json() - detail = parsed.get("message") or detail - except Exception: - pass - raise HTTPException(status_code=response.status_code, detail=f"GitHub API error: {detail}") - return response.json() - # [/DEF:create_github_repository:Function] - - # [DEF:create_gitlab_repository:Function] - # @PURPOSE: Create repository(project) in GitLab. - # @PRE: PAT has api scope. - # @POST: Returns created repository payload. - # @RETURN: dict - # @RELATION: CALLS -> [GitService._normalize_git_server_url] - async def create_gitlab_repository( - self, - server_url: str, - pat: str, - name: str, - private: bool = True, - description: Optional[str] = None, - auto_init: bool = True, - default_branch: Optional[str] = "main", - ) -> Dict[str, Any]: - base_url = self._normalize_git_server_url(server_url) - api_url = f"{base_url}/api/v4/projects" - headers = { - "PRIVATE-TOKEN": pat.strip(), - "Content-Type": "application/json", - "Accept": "application/json", - } - payload: Dict[str, Any] = { - "name": name, - "visibility": "private" if private else "public", - "initialize_with_readme": bool(auto_init), - } - if description: - payload["description"] = description - if default_branch: - payload["default_branch"] = default_branch - try: - async with httpx.AsyncClient(timeout=20.0) as client: - response = await client.post(api_url, headers=headers, json=payload) - except Exception as e: - raise HTTPException(status_code=503, detail=f"GitLab API is unavailable: {str(e)}") - - if response.status_code >= 400: - detail = response.text - try: - parsed = response.json() - if isinstance(parsed, dict): - detail = parsed.get("message") or detail - except Exception: - pass - raise HTTPException(status_code=response.status_code, detail=f"GitLab API error: {detail}") - - data = response.json() - # Normalize clone URL key to keep route response stable. - if "clone_url" not in data: - data["clone_url"] = data.get("http_url_to_repo") - if "html_url" not in data: - data["html_url"] = data.get("web_url") - if "ssh_url" not in data: - data["ssh_url"] = data.get("ssh_url_to_repo") - if "full_name" not in data: - data["full_name"] = data.get("path_with_namespace") or data.get("name") - return data - # [/DEF:create_gitlab_repository:Function] - - # [DEF:_parse_remote_repo_identity:Function] - # @PURPOSE: Parse owner/repo from remote URL for Git server API operations. - # @PRE: remote_url is a valid git URL. - # @POST: Returns owner/repo tokens. - # @RETURN: Dict[str, str] - # @RELATION: USES -> [urlparse] - def _parse_remote_repo_identity(self, remote_url: str) -> Dict[str, str]: - normalized = str(remote_url or "").strip() - if not normalized: - raise HTTPException(status_code=400, detail="Repository remote_url is empty") - - if normalized.startswith("git@"): - # git@host:owner/repo.git - path = normalized.split(":", 1)[1] if ":" in normalized else "" - else: - parsed = urlparse(normalized) - path = parsed.path or "" - - path = path.strip("/") - if path.endswith(".git"): - path = path[:-4] - parts = [segment for segment in path.split("/") if segment] - if len(parts) < 2: - raise HTTPException(status_code=400, detail=f"Cannot parse repository owner/name from remote URL: {remote_url}") - - owner = parts[0] - repo = parts[-1] - namespace = "/".join(parts[:-1]) - return { - "owner": owner, - "repo": repo, - "namespace": namespace, - "full_name": f"{namespace}/{repo}", - } - # [/DEF:_parse_remote_repo_identity:Function] - - # [DEF:_derive_server_url_from_remote:Function] - # @PURPOSE: Build API base URL from remote repository URL without credentials. - # @PRE: remote_url may be any git URL. - # @POST: Returns normalized http(s) base URL or None when derivation is impossible. - # @RETURN: Optional[str] - # @RELATION: USES -> [urlparse] - def _derive_server_url_from_remote(self, remote_url: str) -> Optional[str]: - normalized = str(remote_url or "").strip() - if not normalized or normalized.startswith("git@"): - return None - - parsed = urlparse(normalized) - if parsed.scheme not in {"http", "https"}: - return None - if not parsed.hostname: - return None - - netloc = parsed.hostname - if parsed.port: - netloc = f"{netloc}:{parsed.port}" - return f"{parsed.scheme}://{netloc}".rstrip("/") - # [/DEF:_derive_server_url_from_remote:Function] - - # [DEF:promote_direct_merge:Function] - # @PURPOSE: Perform direct merge between branches in local repo and push target branch. - # @PRE: Repository exists and both branches are valid. - # @POST: Target branch contains merged changes from source branch. - # @RETURN: Dict[str, Any] - # @RELATION: CALLS -> [GitService.get_repo] - def promote_direct_merge( - self, - dashboard_id: int, - from_branch: str, - to_branch: str, - ) -> Dict[str, Any]: - with belief_scope("GitService.promote_direct_merge"): - if not from_branch or not to_branch: - raise HTTPException(status_code=400, detail="from_branch and to_branch are required") - repo = self.get_repo(dashboard_id) - source = from_branch.strip() - target = to_branch.strip() - if source == target: - raise HTTPException(status_code=400, detail="from_branch and to_branch must be different") - - try: - origin = repo.remote(name="origin") - except ValueError: - raise HTTPException(status_code=400, detail="Remote 'origin' not configured") - - try: - origin.fetch() - # Ensure local source branch exists. - if source not in [head.name for head in repo.heads]: - if f"origin/{source}" in [ref.name for ref in repo.refs]: - repo.git.checkout("-b", source, f"origin/{source}") - else: - raise HTTPException(status_code=404, detail=f"Source branch '{source}' not found") - - # Ensure local target branch exists and is checked out. - if target in [head.name for head in repo.heads]: - repo.git.checkout(target) - elif f"origin/{target}" in [ref.name for ref in repo.refs]: - repo.git.checkout("-b", target, f"origin/{target}") - else: - raise HTTPException(status_code=404, detail=f"Target branch '{target}' not found") - - # Bring target up to date and merge source into target. - try: - origin.pull(target) - except Exception: - pass - repo.git.merge(source, "--no-ff", "-m", f"chore(flow): promote {source} -> {target}") - origin.push(refspec=f"{target}:{target}") - except HTTPException: - raise - except Exception as e: - message = str(e) - if "CONFLICT" in message.upper(): - raise HTTPException(status_code=409, detail=f"Merge conflict during direct promote: {message}") - raise HTTPException(status_code=500, detail=f"Direct promote failed: {message}") - - return { - "mode": "direct", - "from_branch": source, - "to_branch": target, - "status": "merged", - } - # [/DEF:promote_direct_merge:Function] - - # [DEF:create_gitea_pull_request:Function] - # @PURPOSE: Create pull request in Gitea. - # @PRE: Config and remote URL are valid. - # @POST: Returns normalized PR metadata. - # @RETURN: Dict[str, Any] - # @RELATION: CALLS -> [GitService._parse_remote_repo_identity] - # @RELATION: CALLS -> [GitService._gitea_request] - # @RELATION: CALLS -> [GitService._derive_server_url_from_remote] - # @RELATION: CALLS -> [GitService._normalize_git_server_url] - # @RELATION: CALLS -> [GitService._build_gitea_pr_404_detail] - async def create_gitea_pull_request( - self, - server_url: str, - pat: str, - remote_url: str, - from_branch: str, - to_branch: str, - title: str, - description: Optional[str] = None, - ) -> Dict[str, Any]: - identity = self._parse_remote_repo_identity(remote_url) - payload = { - "title": title, - "head": from_branch, - "base": to_branch, - "body": description or "", - } - endpoint = f"/repos/{identity['owner']}/{identity['repo']}/pulls" - active_server_url = server_url - try: - data = await self._gitea_request( - "POST", - active_server_url, - pat, - endpoint, - payload=payload, - ) - except HTTPException as exc: - fallback_url = self._derive_server_url_from_remote(remote_url) - normalized_primary = self._normalize_git_server_url(server_url) - should_retry_with_fallback = ( - exc.status_code == 404 and fallback_url and fallback_url != normalized_primary - ) - if should_retry_with_fallback: - logger.warning( - "[create_gitea_pull_request][Action] Primary Gitea URL not found, retrying with remote host: %s", - fallback_url, - ) - active_server_url = fallback_url - try: - data = await self._gitea_request( - "POST", - active_server_url, - pat, - endpoint, - payload=payload, - ) - except HTTPException as retry_exc: - if retry_exc.status_code == 404: - branch_detail = await self._build_gitea_pr_404_detail( - server_url=active_server_url, - pat=pat, - owner=identity["owner"], - repo=identity["repo"], - from_branch=from_branch, - to_branch=to_branch, - ) - if branch_detail: - raise HTTPException(status_code=400, detail=branch_detail) - raise - else: - if exc.status_code == 404: - branch_detail = await self._build_gitea_pr_404_detail( - server_url=active_server_url, - pat=pat, - owner=identity["owner"], - repo=identity["repo"], - from_branch=from_branch, - to_branch=to_branch, - ) - if branch_detail: - raise HTTPException(status_code=400, detail=branch_detail) - raise - - if not isinstance(data, dict): - raise HTTPException(status_code=500, detail="Unexpected Gitea response while creating pull request") - return { - "id": data.get("number") or data.get("id"), - "url": data.get("html_url") or data.get("url"), - "status": data.get("state") or "open", - } - # [/DEF:create_gitea_pull_request:Function] - - # [DEF:create_github_pull_request:Function] - # @PURPOSE: Create pull request in GitHub or GitHub Enterprise. - # @PRE: Config and remote URL are valid. - # @POST: Returns normalized PR metadata. - # @RETURN: Dict[str, Any] - # @RELATION: CALLS -> [GitService._parse_remote_repo_identity] - # @RELATION: CALLS -> [GitService._normalize_git_server_url] - async def create_github_pull_request( - self, - server_url: str, - pat: str, - remote_url: str, - from_branch: str, - to_branch: str, - title: str, - description: Optional[str] = None, - draft: bool = False, - ) -> Dict[str, Any]: - identity = self._parse_remote_repo_identity(remote_url) - base_url = self._normalize_git_server_url(server_url) - if "github.com" in base_url: - api_url = f"https://api.github.com/repos/{identity['namespace']}/{identity['repo']}/pulls" - else: - api_url = f"{base_url}/api/v3/repos/{identity['namespace']}/{identity['repo']}/pulls" - headers = { - "Authorization": f"token {pat.strip()}", - "Content-Type": "application/json", - "Accept": "application/vnd.github+json", - } - payload = { - "title": title, - "head": from_branch, - "base": to_branch, - "body": description or "", - "draft": bool(draft), - } - try: - async with httpx.AsyncClient(timeout=20.0) as client: - response = await client.post(api_url, headers=headers, json=payload) - except Exception as e: - raise HTTPException(status_code=503, detail=f"GitHub API is unavailable: {str(e)}") - if response.status_code >= 400: - detail = response.text - try: - detail = response.json().get("message") or detail - except Exception: - pass - raise HTTPException(status_code=response.status_code, detail=f"GitHub API error: {detail}") - data = response.json() - return { - "id": data.get("number") or data.get("id"), - "url": data.get("html_url") or data.get("url"), - "status": data.get("state") or "open", - } - # [/DEF:create_github_pull_request:Function] - - # [DEF:create_gitlab_merge_request:Function] - # @PURPOSE: Create merge request in GitLab. - # @PRE: Config and remote URL are valid. - # @POST: Returns normalized MR metadata. - # @RETURN: Dict[str, Any] - # @RELATION: CALLS -> [GitService._parse_remote_repo_identity] - # @RELATION: CALLS -> [GitService._normalize_git_server_url] - async def create_gitlab_merge_request( - self, - server_url: str, - pat: str, - remote_url: str, - from_branch: str, - to_branch: str, - title: str, - description: Optional[str] = None, - remove_source_branch: bool = False, - ) -> Dict[str, Any]: - identity = self._parse_remote_repo_identity(remote_url) - base_url = self._normalize_git_server_url(server_url) - project_id = quote(identity["full_name"], safe="") - api_url = f"{base_url}/api/v4/projects/{project_id}/merge_requests" - headers = { - "PRIVATE-TOKEN": pat.strip(), - "Content-Type": "application/json", - "Accept": "application/json", - } - payload = { - "source_branch": from_branch, - "target_branch": to_branch, - "title": title, - "description": description or "", - "remove_source_branch": bool(remove_source_branch), - } - try: - async with httpx.AsyncClient(timeout=20.0) as client: - response = await client.post(api_url, headers=headers, json=payload) - except Exception as e: - raise HTTPException(status_code=503, detail=f"GitLab API is unavailable: {str(e)}") - if response.status_code >= 400: - detail = response.text - try: - parsed = response.json() - if isinstance(parsed, dict): - detail = parsed.get("message") or detail - except Exception: - pass - raise HTTPException(status_code=response.status_code, detail=f"GitLab API error: {detail}") - data = response.json() - return { - "id": data.get("iid") or data.get("id"), - "url": data.get("web_url") or data.get("url"), - "status": data.get("state") or "opened", - } - # [/DEF:create_gitlab_merge_request:Function] - -# [/DEF:GitService:Class] -# [/DEF:git_service:Module] +# [DEF:git_service:Module:Tombstone] +# @COMPLEXITY: 1 +# @PURPOSE: Re-export shim — GitService has been decomposed into services/git/ package. +# All consumers continue to import from this same path without changes. +# @RELATION: REDIRECTS_TO -> [GitServiceModule] +# @RATIONALE: Monolithic GitService (2101 lines) was decomposed into 8 domain-specific mixins +# under services/git/ to satisfy INV_7 (< 400 lines per module). This shim preserves +# the original import path for all 5 consumers. +# @REJECTED: Breaking 5 consumer imports to remove this shim — unacceptable migration cost. +from src.services.git import GitService # noqa: F401 +# [/DEF:git_service:Module:Tombstone] diff --git a/backend/tests/core/test_defensive_guards.py b/backend/tests/core/test_defensive_guards.py index a9f4257f..39a9d2cc 100644 --- a/backend/tests/core/test_defensive_guards.py +++ b/backend/tests/core/test_defensive_guards.py @@ -64,7 +64,7 @@ def test_git_service_init_repo_reclones_when_path_is_not_a_git_repo(): (target_path / "placeholder.txt").write_text("not a git repo", encoding="utf-8") clone_result = MagicMock() - with patch("src.services.git_service.Repo") as repo_ctor: + with patch("src.services.git._base.Repo") as repo_ctor: repo_ctor.side_effect = InvalidGitRepositoryError("invalid repo") repo_ctor.clone_from.return_value = clone_result result = service.init_repo(10, "https://example.com/org/repo.git", "token", repo_key="covid")