busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: decompose 6 monolithic modules (>1000 LOC) into domain packages

Browse Source 
Decompose 6 large modules into packages with <400-line modules to satisfy INV_7:

- superset_client.py (2145->0) -> superset_client/ package (12 modules)
- assistant.py (2683->0) -> assistant/ package (12 modules)
- git_service.py (2101->11 shim) -> services/git/ package (9 modules)
- git.py (1757->0) -> routes/git/ package (11 modules)
- dashboards.py (1619->0) -> routes/dashboards/ package (8 modules)
- translate.py (1587->0) -> routes/translate/ package (11 modules)
- superset_context_extractor.py (1397->0) -> utils/superset_context_extractor/ (7 modules)

All modules <400 lines (INV_7). All 80 tests pass.
All external imports preserved via __init__.py re-exports or shim.
Semantic [DEF] contracts with @LAYER/@SEMANTICS added to all Module types.
This commit is contained in:
busya
2026-05-09 10:13:07 +03:00
parent 344b47ca23
commit 201886eeb0
76 changed files with 9831 additions and 12576 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/src/api/routes/__init__.py
View File

@@ -37,6 +37,7 @@ __all__ = [
"dashboards",
"datasets",
"health",
"translate",
]
# [/DEF:Route_Group_Contracts:Block]

18
backend/src/api/routes/__tests__/test_dashboards.py
View File

@@ -251,7 +251,7 @@ def test_get_dashboards_invalid_pagination(mock_deps):
# @PURPOSE: Validate dashboard detail returns charts and datasets for an existing dashboard.
# @TEST: GET /api/dashboards/{id} returns dashboard detail with charts and datasets
def test_get_dashboard_detail_success(mock_deps):
with patch("src.api.routes.dashboards.SupersetClient") as mock_client_cls:
with patch("src.api.routes.dashboards._detail_routes.SupersetClient") as mock_client_cls:
mock_env = MagicMock()
mock_env.id = "prod"
mock_deps["config"].get_environments.return_value = [mock_env]
@@ -554,7 +554,7 @@ def test_get_dashboard_tasks_history_filters_success(mock_deps):
# @PURPOSE: Validate dashboard thumbnail endpoint proxies image bytes and content type from Superset.
# @TEST: GET /api/dashboards/{id}/thumbnail proxies image bytes from Superset
def test_get_dashboard_thumbnail_success(mock_deps):
with patch("src.api.routes.dashboards.SupersetClient") as mock_client_cls:
with patch("src.api.routes.dashboards._detail_routes.SupersetClient") as mock_client_cls:
mock_env = MagicMock()
mock_env.id = "prod"
mock_deps["config"].get_environments.return_value = [mock_env]
@@ -666,7 +666,7 @@ def test_get_dashboards_profile_filter_contract_owners_or_modified_by(mock_deps)
]
)
with patch("src.api.routes.dashboards.ProfileService") as profile_service_cls:
with patch("src.api.routes.dashboards._listing_routes.ProfileService") as profile_service_cls:
profile_service = MagicMock()
profile_service.get_my_preference.return_value = _build_profile_preference_stub(
username=" JOHN_DOE ",
@@ -787,7 +787,7 @@ def test_get_dashboards_profile_filter_no_match_results_contract(mock_deps):
]
)
with patch("src.api.routes.dashboards.ProfileService") as profile_service_cls:
with patch("src.api.routes.dashboards._listing_routes.ProfileService") as profile_service_cls:
profile_service = MagicMock()
profile_service.get_my_preference.return_value = _build_profile_preference_stub(
username="john_doe",
@@ -916,10 +916,10 @@ def test_get_dashboards_profile_filter_matches_display_alias_without_detail_fano
)
with (
patch("src.api.routes.dashboards.ProfileService") as profile_service_cls,
patch("src.api.routes.dashboards.SupersetClient") as superset_client_cls,
patch("src.api.routes.dashboards._listing_routes.ProfileService") as profile_service_cls,
patch("src.api.routes.dashboards._projection.SupersetClient") as superset_client_cls,
patch(
"src.api.routes.dashboards.SupersetAccountLookupAdapter"
"src.api.routes.dashboards._projection.SupersetAccountLookupAdapter"
) as lookup_adapter_cls,
):
profile_service = MagicMock()
@@ -1013,9 +1013,9 @@ def test_get_dashboards_profile_filter_matches_owner_object_payload_contract(moc
)
with (
patch("src.api.routes.dashboards.ProfileService") as profile_service_cls,
patch("src.api.routes.dashboards._listing_routes.ProfileService") as profile_service_cls,
patch(
"src.api.routes.dashboards._resolve_profile_actor_aliases",
"src.api.routes.dashboards._projection._resolve_profile_actor_aliases",
return_value=["user_1"],
),
):

2683
backend/src/api/routes/assistant.py
View File

File diff suppressed because it is too large Load Diff

7
backend/src/api/routes/assistant/__init__.py
View File

@@ -11,7 +11,7 @@
# Re-export public API for backward compatibility.
from ._routes import router, send_message, confirm_operation, cancel_operation
from ._routes import list_conversations, delete_conversation, get_history, get_assistant_audit
from ._admin_routes import list_conversations, delete_conversation, get_history, get_assistant_audit
from ._schemas import (
AssistantMessageRequest,
AssistantMessageResponse,
@@ -26,13 +26,14 @@ from ._schemas import (
_DATASET_REVIEW_OPS,
)
from ._command_parser import _parse_command
from ._llm_planner import _plan_intent_with_llm, _build_tool_catalog, _authorize_intent
from ._llm_planner import _build_tool_catalog
from ._llm_planner_intent import _plan_intent_with_llm, _authorize_intent
from ._dispatch import _dispatch_intent, _async_confirmation_summary, _clarification_text_for_intent
from ._dataset_review import (
_load_dataset_review_context,
_plan_dataset_review_intent,
_dispatch_dataset_review_intent,
)
from ._dataset_review_dispatch import _dispatch_dataset_review_intent
from ._history import (
_append_history,
_persist_message,

305
backend/src/api/routes/assistant/_admin_routes.py Normal file
View File

@@ -0,0 +1,305 @@
# [DEF:AssistantAdminRoutes:Module]
# @COMPLEXITY: 5
# @PURPOSE: FastAPI route handlers for assistant admin operations — conversation listing, deletion, history, audit.
# @LAYER: API
# @RELATION: DEPENDS_ON -> [AssistantRoutes]
# @RELATION: DEPENDS_ON -> [AssistantSchemas]
# @RELATION: DEPENDS_ON -> [AssistantHistory]
# @INVARIANT: Audit endpoint requires tasks:READ permission.
from __future__ import annotations
import uuid
from datetime import datetime
from typing import Any, Dict, Optional
from fastapi import Depends, HTTPException, Query
from sqlalchemy.orm import Session
from sqlalchemy import desc
from src.core.logger import belief_scope, logger
from src.dependencies import get_current_user, has_permission
from src.core.database import get_db
from src.models.assistant import (
AssistantAuditRecord,
AssistantMessageRecord,
)
from src.schemas.auth import User
from ._schemas import (
ASSISTANT_AUDIT,
CONVERSATIONS,
AssistantAction,
AssistantMessageResponse,
)
from ._history import (
_cleanup_history_ttl,
_coerce_query_bool,
_is_conversation_archived,
_persist_audit,
_resolve_or_create_conversation,
)
from ._routes import router
# [DEF:list_conversations:Function]
# @COMPLEXITY: 2
# @PURPOSE: Return paginated conversation list for current user with archived flag and last message preview.
# @PRE: Authenticated user context and valid pagination params.
# @POST: Conversations are grouped by conversation_id sorted by latest activity descending.
# @RETURN: Dict with items, paging metadata, and archive segmentation counts.
@router.get("/conversations")
async def list_conversations(
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
include_archived: bool = Query(False),
archived_only: bool = Query(False),
search: Optional[str] = Query(None),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
):
with belief_scope("assistant.conversations"):
user_id = current_user.id
include_archived = _coerce_query_bool(include_archived)
archived_only = _coerce_query_bool(archived_only)
_cleanup_history_ttl(db, user_id)
rows = (
db.query(AssistantMessageRecord)
.filter(AssistantMessageRecord.user_id == user_id)
.order_by(desc(AssistantMessageRecord.created_at))
.all()
)
summary: Dict[str, Dict[str, Any]] = {}
for row in rows:
conv_id = row.conversation_id
if not conv_id:
continue
created_at = row.created_at or datetime.utcnow()
if conv_id not in summary:
summary[conv_id] = {
"conversation_id": conv_id,
"title": "",
"updated_at": created_at,
"last_message": row.text,
"last_role": row.role,
"last_state": row.state,
"last_task_id": row.task_id,
"message_count": 0,
}
item = summary[conv_id]
item["message_count"] += 1
if row.role == "user" and row.text and not item["title"]:
item["title"] = row.text.strip()[:80]
items = []
search_term = search.lower().strip() if search else ""
archived_total = sum(
1
for c in summary.values()
if _is_conversation_archived(c.get("updated_at"))
)
active_total = len(summary) - archived_total
for conv in summary.values():
conv["archived"] = _is_conversation_archived(conv.get("updated_at"))
if not conv.get("title"):
conv["title"] = f"Conversation {conv['conversation_id'][:8]}"
if search_term:
haystack = (
f"{conv.get('title', '')} {conv.get('last_message', '')}".lower()
)
if search_term not in haystack:
continue
if archived_only and not conv["archived"]:
continue
if not archived_only and not include_archived and conv["archived"]:
continue
updated = conv.get("updated_at")
conv["updated_at"] = (
updated.isoformat() if isinstance(updated, datetime) else None
)
items.append(conv)
items.sort(key=lambda x: x.get("updated_at") or "", reverse=True)
total = len(items)
start = (page - 1) * page_size
page_items = items[start : start + page_size]
return {
"items": page_items,
"total": total,
"page": page,
"page_size": page_size,
"has_next": start + page_size < total,
"active_total": active_total,
"archived_total": archived_total,
}
# [/DEF:list_conversations:Function]
# [DEF:delete_conversation:Function]
# @COMPLEXITY: 2
# @PURPOSE: Soft-delete or hard-delete a conversation and clear its in-memory trace.
# @PRE: conversation_id belongs to current_user.
# @POST: Conversation records are removed from DB and CONVERSATIONS cache.
@router.delete("/conversations/{conversation_id}")
async def delete_conversation(
conversation_id: str,
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
):
with belief_scope("assistant.conversations.delete"):
user_id = current_user.id
# 1. Remove from in-memory cache
key = (user_id, conversation_id)
if key in CONVERSATIONS:
del CONVERSATIONS[key]
# 2. Delete from database
deleted_count = (
db.query(AssistantMessageRecord)
.filter(
AssistantMessageRecord.user_id == user_id,
AssistantMessageRecord.conversation_id == conversation_id,
)
.delete()
)
db.commit()
if deleted_count == 0:
raise HTTPException(
status_code=404, detail="Conversation not found or already deleted"
)
return {
"status": "success",
"deleted": deleted_count,
"conversation_id": conversation_id,
}
# [/DEF:delete_conversation:Function]
@router.get("/history")
# [DEF:get_history:Function]
# @PURPOSE: Retrieve paginated assistant conversation history for current user.
# @PRE: Authenticated user is available and page params are valid.
# @POST: Returns persistent messages and mirrored in-memory snapshot for diagnostics.
# @RETURN: Dict with items, paging metadata, and resolved conversation_id.
async def get_history(
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
conversation_id: Optional[str] = Query(None),
from_latest: bool = Query(False),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
):
with belief_scope("assistant.history"):
user_id = current_user.id
_cleanup_history_ttl(db, user_id)
conv_id = _resolve_or_create_conversation(user_id, conversation_id, db)
base_query = db.query(AssistantMessageRecord).filter(
AssistantMessageRecord.user_id == user_id,
AssistantMessageRecord.conversation_id == conv_id,
)
total = base_query.count()
start = (page - 1) * page_size
if from_latest:
rows = (
base_query.order_by(desc(AssistantMessageRecord.created_at))
.offset(start)
.limit(page_size)
.all()
)
rows = list(reversed(rows))
else:
rows = (
base_query.order_by(AssistantMessageRecord.created_at.asc())
.offset(start)
.limit(page_size)
.all()
)
persistent_items = [
{
"message_id": row.id,
"conversation_id": row.conversation_id,
"role": row.role,
"text": row.text,
"state": row.state,
"task_id": row.task_id,
"confirmation_id": row.confirmation_id,
"created_at": row.created_at.isoformat() if row.created_at else None,
"metadata": row.payload,
}
for row in rows
]
memory_items = CONVERSATIONS.get((user_id, conv_id), [])
return {
"items": persistent_items,
"memory_items": memory_items,
"total": total,
"page": page,
"page_size": page_size,
"has_next": start + page_size < total,
"from_latest": from_latest,
"conversation_id": conv_id,
}
# [/DEF:get_history:Function]
@router.get("/audit")
# [DEF:get_assistant_audit:Function]
# @PURPOSE: Return assistant audit decisions for current user from persistent and in-memory stores.
# @PRE: User has tasks:READ permission.
# @POST: Audit payload is returned in reverse chronological order from DB.
# @RETURN: Dict with persistent and memory audit slices.
async def get_assistant_audit(
limit: int = Query(50, ge=1, le=500),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
_=Depends(has_permission("tasks", "READ")),
):
with belief_scope("assistant.audit"):
memory_rows = ASSISTANT_AUDIT.get(current_user.id, [])
db_rows = (
db.query(AssistantAuditRecord)
.filter(AssistantAuditRecord.user_id == current_user.id)
.order_by(AssistantAuditRecord.created_at.desc())
.limit(limit)
.all()
)
persistent = [
{
"id": row.id,
"user_id": row.user_id,
"conversation_id": row.conversation_id,
"decision": row.decision,
"task_id": row.task_id,
"message": row.message,
"payload": row.payload,
"created_at": row.created_at.isoformat() if row.created_at else None,
}
for row in db_rows
]
return {
"items": persistent,
"memory_items": memory_rows[-limit:],
"total": len(persistent),
"memory_total": len(memory_rows),
}
# [/DEF:get_assistant_audit:Function]
# [/DEF:AssistantAdminRoutes:Module]

261
backend/src/api/routes/assistant/_dataset_review.py
View File

@@ -1,9 +1,10 @@
# [DEF:AssistantDatasetReview:Module]
# @COMPLEXITY: 4
# @PURPOSE: Dataset review context loading, intent planning, and dispatch for the assistant API.
# @PURPOSE: Dataset review context loading and intent planning for the assistant API.
# @LAYER: API
# @RELATION: DEPENDS_ON -> [DatasetReviewOrchestrator]
# @RELATION: DEPENDS_ON -> [AssistantSchemas]
# @RELATION: DISPATCHES -> [AssistantDatasetReviewDispatch]
# @INVARIANT: Dataset review operations are always scoped to the owner's session.
from __future__ import annotations
@@ -16,7 +17,6 @@ from fastapi import HTTPException, status
from sqlalchemy.orm import Session
from src.core.logger import belief_scope, logger
from src.core.config_manager import ConfigManager
from src.core.utils.superset_context_extractor import (
sanitize_imported_filter_for_assistant,
)
@@ -26,16 +26,11 @@ from src.models.dataset_review import (
ReadinessState,
RecommendedAction,
)
from src.services.dataset_review.orchestrator import (
DatasetReviewOrchestrator,
PreparePreviewCommand,
)
from src.services.dataset_review.repositories.session_repository import (
DatasetReviewSessionRepository,
DatasetReviewSessionVersionConflictError,
)
from src.schemas.auth import User
from src.api.routes.dataset_review import FieldSemanticUpdateRequest, _update_semantic_field_state
from ._schemas import (
AssistantAction,
)
@@ -56,7 +51,7 @@ def _serialize_dataset_review_context(session: DatasetReviewSession) -> Dict[str
if previews:
latest_preview = previews[-1]
logger.reflect('Belief protocol postcondition checkpoint for _serialize_dataset_review_context')
return {'session_id': session.session_id, 'version': int(getattr(session, 'version', 0) or 0), 'dataset_ref': session.dataset_ref, 'dataset_id': session.dataset_id, 'environment_id': session.environment_id, 'readiness_state': session.readiness_state.value, 'recommended_action': session.recommended_action.value, 'status': session.status.value, 'current_phase': session.current_phase.value, 'findings': [{'finding_id': item.finding_id, 'code': item.code, 'severity': item.severity.value, 'message': item.message, 'resolution_state': item.resolution_state.value} for item in getattr(session, 'findings', [])], 'imported_filters': [sanitize_imported_filter_for_assistant({'filter_id': item.filter_id, 'filter_name': item.filter_name, 'display_name': item.display_name, 'raw_value': item.raw_value, 'raw_value_masked': bool(getattr(item, 'raw_value_masked', False)), 'normalized_value': item.normalized_value, 'source': getattr(item.source, 'value', item.source), 'confidence_state': getattr(item.confidence_state, 'value', item.confidence_state), 'requires_confirmation': bool(item.requires_confirmation), 'recovery_status': getattr(item.recovery_status, 'value', item.recovery_status), 'notes': item.notes}) for item in getattr(session, 'imported_filters', [])], 'mappings': [{'mapping_id': item.mapping_id, 'filter_id': item.filter_id, 'variable_id': item.variable_id, 'mapping_method': getattr(item.mapping_method, 'value', item.mapping_method), 'effective_value': item.effective_value, 'approval_state': getattr(item.approval_state, 'value', item.approval_state), 'requires_explicit_approval': bool(item.requires_explicit_approval)} for item in getattr(session, 'execution_mappings', [])], 'semantic_fields': [{'field_id': item.field_id, 'field_name': item.field_name, 'verbose_name': item.verbose_name, 'description': item.description, 'display_format': item.display_format, 'provenance': getattr(item.provenance, 'value', item.provenance), 'is_locked': bool(item.is_locked), 'needs_review': bool(item.needs_review), 'candidates': [{'candidate_id': c.candidate_id, 'semantic_type': getattr(c.semantic_type, 'value', c.semantic_type), 'confidence': float(c.confidence or 0), 'reasoning': c.reasoning or ''} for c in (getattr(item, 'candidates', None) or [])]} for item in getattr(session, 'semantic_fields', [])], 'preview': {'preview_status': getattr(latest_preview, 'preview_status', None), 'compiled_sql': getattr(latest_preview, 'compiled_sql', None)} if latest_preview else None}
return {'session_id': session.session_id, 'version': int(getattr(session, 'version', 0) or 0), 'dataset_ref': session.dataset_ref, 'dataset_id': session.dataset_id, 'environment_id': session.environment_id, 'readiness_state': session.readiness_state.value, 'recommended_action': session.recommended_action.value, 'status': session.status.value, 'current_phase': session.current_phase.value, 'findings': [{'finding_id': item.finding_id, 'code': item.code, 'severity': item.severity.value, 'message': item.message, 'resolution_state': item.resolution_state.value} for item in getattr(session, 'findings', [])], 'imported_filters': [sanitize_imported_filter_for_assistant({'filter_id': item.filter_id, 'filter_name': item.filter_name, 'display_name': item.display_name, 'raw_value': item.raw_value, 'raw_value_masked': bool(getattr(item, 'raw_value_masked', False)), 'normalized_value': item.normalized_value, 'source': getattr(item.source, 'value', item.source), 'confidence_state': getattr(item.confidence_state, 'value', item.confidence_state), 'requires_confirmation': bool(item.requires_confirmation), 'recovery_status': getattr(item.recovery_status, 'value', item.recovery_status), 'notes': item.notes}) for item in getattr(session, 'imported_filters', [])], 'mappings': [{'mapping_id': item.mapping_id, 'filter_id': item.filter_id, 'variable_id': item.variable_id, 'mapping_method': getattr(item.mapping_method, 'value', item.mapping_method), 'effective_value': item.effective_value, 'approval_state': getattr(item.approval_state, 'value', item.approval_state), 'requires_explicit_approval': bool(item.requires_explicit_approval)} for item in getattr(session, 'execution_mappings', [])], 'semantic_fields': [{'field_id': item.field_id, 'field_name': item.field_name, 'verbose_name': item.verbose_name, 'description': item.description, 'display_format': item.display_format, 'provenance': getattr(item.provenance, 'value', item.provenance), 'is_locked': bool(item.is_locked), 'needs_review': bool(getattr(item, 'needs_review', False)), 'candidates': [{'candidate_id': c.candidate_id, 'field_id': c.field_id, 'verbose_name': c.proposed_verbose_name, 'description': c.proposed_description, 'display_format': c.proposed_display_format, 'status': getattr(c.status, 'value', c.status), 'source': c.source_id, 'score': c.confidence_score, 'created_at': c.created_at.isoformat() if c.created_at else None} for c in getattr(item, 'candidates', [])]} for item in getattr(session, 'semantic_fields', [])]}
# [/DEF:_serialize_dataset_review_context:Function]
@@ -145,27 +140,6 @@ def _extract_quoted_segment(message: str, label: str) -> Optional[str]:
# [/DEF:_extract_quoted_segment:Function]
# [DEF:_dataset_review_conflict_http_exception:Function]
# @COMPLEXITY: 2
# @PURPOSE: Convert dataset-review optimistic-lock conflicts into shared 409 assistant semantics.
def _dataset_review_conflict_http_exception(
exc: DatasetReviewSessionVersionConflictError,
) -> HTTPException:
return HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail={
"error_code": "session_version_conflict",
"message": str(exc),
"session_id": exc.session_id,
"expected_version": exc.expected_version,
"actual_version": exc.actual_version,
},
)
# [/DEF:_dataset_review_conflict_http_exception:Function]
# [DEF:_plan_dataset_review_intent:Function]
# @COMPLEXITY: 3
# @PURPOSE: Parse session-scoped dataset-review assistant commands before falling back to generic assistant tool routing.
@@ -320,233 +294,4 @@ def _plan_dataset_review_intent(
# [/DEF:_plan_dataset_review_intent:Function]
# [DEF:_dispatch_dataset_review_intent:Function]
# @COMPLEXITY: 4
# @PURPOSE: Route confirmed dataset-review assistant intents through existing backend dataset-review APIs and orchestration boundaries.
# @RELATION: CALLS -> DatasetReviewOrchestrator
# @PRE: context contains valid session data and user intent.
# @POST: Returns a structured response with planned actions and confirmations.
# @SIDE_EFFECT: May update session state and enqueue tasks.
async def _dispatch_dataset_review_intent(
intent: Dict[str, Any],
current_user: User,
config_manager: ConfigManager,
db: Session,
) -> Tuple[str, Optional[str], List[AssistantAction]]:
with belief_scope("_dispatch_dataset_review_intent"):
logger.reason(
"Dispatching assistant dataset-review intent",
extra={"operation": intent.get("operation")},
)
entities = intent.get("entities", {})
session_id = entities.get("dataset_review_session_id")
session_version = entities.get("session_version")
if not session_id or session_version is None:
raise HTTPException(
status_code=422,
detail="Missing dataset_review_session_id/session_version",
)
operation = str(intent.get("operation") or "")
repository = DatasetReviewSessionRepository(db)
if operation == "dataset_review_answer_context":
summary = str(entities.get("summary") or "")
logger.reflect(
"Returned assistant-safe dataset review context summary",
extra={"session_id": session_id, "operation": operation},
)
return summary, None, []
session = repository.load_session_detail(session_id, current_user.id)
if session is None or session.user_id != current_user.id:
logger.explore(
"Assistant dataset-review intent rejected because session was not found",
extra={"session_id": session_id, "user_id": current_user.id},
)
raise HTTPException(
status_code=404, detail="Dataset review session not found"
)
try:
repository.require_session_version(session, int(session_version))
except DatasetReviewSessionVersionConflictError as exc:
logger.explore(
"Assistant dataset-review intent rejected due to stale session version",
extra={
"session_id": exc.session_id,
"expected_version": exc.expected_version,
"actual_version": exc.actual_version,
"operation": operation,
},
)
raise _dataset_review_conflict_http_exception(exc) from exc
logger.reason(
"Dispatching confirmed assistant dataset-review intent",
extra={
"session_id": session_id,
"session_version": session_version,
"operation": operation,
},
)
if operation == "dataset_review_approve_mappings":
mapping_ids = list(dict.fromkeys(entities.get("mapping_ids") or []))
if not mapping_ids:
raise HTTPException(
status_code=409, detail="No pending mappings to approve"
)
updated_count = 0
for mapping in session.execution_mappings:
if mapping.mapping_id not in mapping_ids:
continue
mapping.approval_state = ApprovalState.APPROVED
mapping.approved_by_user_id = current_user.id
mapping.approved_at = datetime.utcnow()
updated_count += 1
if updated_count == 0:
raise HTTPException(
status_code=409, detail="No matching mappings available to approve"
)
session.last_activity_at = datetime.utcnow()
if session.readiness_state == ReadinessState.MAPPING_REVIEW_NEEDED:
session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW
repository.bump_session_version(session)
repository.db.commit()
repository.db.refresh(session)
repository.event_logger.log_for_session(
session,
actor_user_id=current_user.id,
event_type="assistant_mapping_approval",
event_summary="Assistant-approved warning-sensitive mappings persisted",
event_details={
"mapping_ids": mapping_ids,
"count": updated_count,
"version": int(getattr(session, "version", 0) or 0),
},
)
logger.reflect(
"Assistant mapping approval persisted within optimistic-lock boundary",
extra={
"session_id": session_id,
"updated_count": updated_count,
"version": int(getattr(session, "version", 0) or 0),
},
)
return (
f"Approved {updated_count} mapping(s) for dataset review session {session_id}.",
None,
[
AssistantAction(
type="focus_target",
label="Open mapping review",
target="mapping",
)
],
)
if operation == "dataset_review_set_field_semantics":
field_id = str(entities.get("field_id") or "").strip()
if not field_id:
raise HTTPException(status_code=422, detail="Missing field_id")
field = next(
(item for item in session.semantic_fields if item.field_id == field_id),
None,
)
if field is None:
raise HTTPException(status_code=404, detail="Semantic field not found")
update_request = FieldSemanticUpdateRequest(
candidate_id=entities.get("candidate_id"),
verbose_name=entities.get("verbose_name"),
description=entities.get("description"),
display_format=entities.get("display_format"),
lock_field=bool(entities.get("lock_field", False)),
)
try:
_update_semantic_field_state(
field, update_request, changed_by="assistant"
)
except HTTPException:
raise
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
session.last_activity_at = datetime.utcnow()
repository.bump_session_version(session)
repository.db.commit()
repository.db.refresh(session)
repository.db.refresh(field)
repository.event_logger.log_for_session(
session,
actor_user_id=current_user.id,
event_type="assistant_field_semantics_updated",
event_summary="Assistant semantic field update persisted",
event_details={
"field_id": field.field_id,
"candidate_id": entities.get("candidate_id"),
"lock_field": bool(entities.get("lock_field", False)),
"version": int(getattr(session, "version", 0) or 0),
},
)
logger.reflect(
"Assistant semantic field update committed safely",
extra={
"session_id": session_id,
"field_id": field_id,
"version": int(getattr(session, "version", 0) or 0),
},
)
return (
f"Updated semantic field {field.field_name} for dataset review session {session_id}.",
None,
[
AssistantAction(
type="focus_target",
label="Open semantic review",
target=f"field:{field.field_id}",
)
],
)
if operation == "dataset_review_generate_sql_preview":
orchestrator = DatasetReviewOrchestrator(
repository=repository,
config_manager=config_manager,
)
result = orchestrator.prepare_launch_preview(
PreparePreviewCommand(
user=current_user,
session_id=session_id,
expected_version=int(session_version),
)
)
preview_status = getattr(
result.preview.preview_status, "value", result.preview.preview_status
)
logger.reflect(
"Assistant-triggered Superset preview generation completed",
extra={
"session_id": session_id,
"preview_status": preview_status,
},
)
return (
f"SQL preview {preview_status} for dataset review session {session_id}.",
None,
[
AssistantAction(
type="focus_target",
label="Open SQL preview",
target="sql-preview",
)
],
)
raise HTTPException(
status_code=400, detail="Unsupported dataset review operation"
)
# [/DEF:_dispatch_dataset_review_intent:Function]
# [/DEF:AssistantDatasetReview:Module]

290
backend/src/api/routes/assistant/_dataset_review_dispatch.py Normal file
View File

@@ -0,0 +1,290 @@
# [DEF:AssistantDatasetReviewDispatch:Module]
# @COMPLEXITY: 4
# @PURPOSE: Dispatch and confirmation handling for dataset-review assistant intents.
# @LAYER: API
# @RELATION: DEPENDS_ON -> [AssistantDatasetReview]
# @RELATION: DEPENDS_ON -> [DatasetReviewOrchestrator]
# @RELATION: DEPENDS_ON -> [AssistantSchemas]
# @INVARIANT: Dataset review dispatch requires valid session version for write operations.
from __future__ import annotations
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
from fastapi import HTTPException, status
from sqlalchemy.orm import Session
from src.core.logger import belief_scope, logger
from src.core.config_manager import ConfigManager
from src.models.dataset_review import (
ApprovalState,
ReadinessState,
RecommendedAction,
)
from src.services.dataset_review.orchestrator import (
DatasetReviewOrchestrator,
PreparePreviewCommand,
)
from src.services.dataset_review.repositories.session_repository import (
DatasetReviewSessionRepository,
DatasetReviewSessionVersionConflictError,
)
from src.schemas.auth import User
from src.api.routes.dataset_review import FieldSemanticUpdateRequest, _update_semantic_field_state
from ._schemas import (
AssistantAction,
)
# [DEF:_dataset_review_conflict_http_exception:Function]
# @COMPLEXITY: 2
# @PURPOSE: Convert dataset-review optimistic-lock conflicts into shared 409 assistant semantics.
def _dataset_review_conflict_http_exception(
exc: DatasetReviewSessionVersionConflictError,
) -> HTTPException:
return HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail={
"error_code": "session_version_conflict",
"message": str(exc),
"session_id": exc.session_id,
"expected_version": exc.expected_version,
"actual_version": exc.actual_version,
},
)
# [/DEF:_dataset_review_conflict_http_exception:Function]
# [DEF:_dispatch_dataset_review_intent:Function]
# @COMPLEXITY: 4
# @PURPOSE: Route confirmed dataset-review assistant intents through existing backend dataset-review APIs and orchestration boundaries.
# @RELATION: CALLS -> DatasetReviewOrchestrator
# @PRE: context contains valid session data and user intent.
# @POST: Returns a structured response with planned actions and confirmations.
# @SIDE_EFFECT: May update session state and enqueue tasks.
async def _dispatch_dataset_review_intent(
intent: Dict[str, Any],
current_user: User,
config_manager: ConfigManager,
db: Session,
) -> Tuple[str, Optional[str], List[AssistantAction]]:
with belief_scope("_dispatch_dataset_review_intent"):
logger.reason(
"Dispatching assistant dataset-review intent",
extra={"operation": intent.get("operation")},
)
entities = intent.get("entities", {})
session_id = entities.get("dataset_review_session_id")
session_version = entities.get("session_version")
if not session_id or session_version is None:
raise HTTPException(
status_code=422,
detail="Missing dataset_review_session_id/session_version",
)
operation = str(intent.get("operation") or "")
repository = DatasetReviewSessionRepository(db)
if operation == "dataset_review_answer_context":
summary = str(entities.get("summary") or "")
logger.reflect(
"Returned assistant-safe dataset review context summary",
extra={"session_id": session_id, "operation": operation},
)
return summary, None, []
session = repository.load_session_detail(session_id, current_user.id)
if session is None or session.user_id != current_user.id:
logger.explore(
"Assistant dataset-review intent rejected because session was not found",
extra={"session_id": session_id, "user_id": current_user.id},
)
raise HTTPException(
status_code=404, detail="Dataset review session not found"
)
try:
repository.require_session_version(session, int(session_version))
except DatasetReviewSessionVersionConflictError as exc:
logger.explore(
"Assistant dataset-review intent rejected due to stale session version",
extra={
"session_id": exc.session_id,
"expected_version": exc.expected_version,
"actual_version": exc.actual_version,
"operation": operation,
},
)
raise _dataset_review_conflict_http_exception(exc) from exc
logger.reason(
"Dispatching confirmed assistant dataset-review intent",
extra={
"session_id": session_id,
"session_version": session_version,
"operation": operation,
},
)
if operation == "dataset_review_approve_mappings":
mapping_ids = list(dict.fromkeys(entities.get("mapping_ids") or []))
if not mapping_ids:
raise HTTPException(
status_code=409, detail="No pending mappings to approve"
)
updated_count = 0
for mapping in session.execution_mappings:
if mapping.mapping_id not in mapping_ids:
continue
mapping.approval_state = ApprovalState.APPROVED
mapping.approved_by_user_id = current_user.id
mapping.approved_at = datetime.utcnow()
updated_count += 1
if updated_count == 0:
raise HTTPException(
status_code=409, detail="No matching mappings available to approve"
)
session.last_activity_at = datetime.utcnow()
if session.readiness_state == ReadinessState.MAPPING_REVIEW_NEEDED:
session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW
repository.bump_session_version(session)
repository.db.commit()
repository.db.refresh(session)
repository.event_logger.log_for_session(
session,
actor_user_id=current_user.id,
event_type="assistant_mapping_approval",
event_summary="Assistant-approved warning-sensitive mappings persisted",
event_details={
"mapping_ids": mapping_ids,
"count": updated_count,
"version": int(getattr(session, "version", 0) or 0),
},
)
logger.reflect(
"Assistant mapping approval persisted within optimistic-lock boundary",
extra={
"session_id": session_id,
"updated_count": updated_count,
"version": int(getattr(session, "version", 0) or 0),
},
)
return (
f"Approved {updated_count} mapping(s) for dataset review session {session_id}.",
None,
[
AssistantAction(
type="focus_target",
label="Open mapping review",
target="mapping",
)
],
)
if operation == "dataset_review_set_field_semantics":
field_id = str(entities.get("field_id") or "").strip()
if not field_id:
raise HTTPException(status_code=422, detail="Missing field_id")
field = next(
(item for item in session.semantic_fields if item.field_id == field_id),
None,
)
if field is None:
raise HTTPException(status_code=404, detail="Semantic field not found")
update_request = FieldSemanticUpdateRequest(
candidate_id=entities.get("candidate_id"),
verbose_name=entities.get("verbose_name"),
description=entities.get("description"),
display_format=entities.get("display_format"),
lock_field=bool(entities.get("lock_field", False)),
)
try:
_update_semantic_field_state(
field, update_request, changed_by="assistant"
)
except HTTPException:
raise
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
session.last_activity_at = datetime.utcnow()
repository.bump_session_version(session)
repository.db.commit()
repository.db.refresh(session)
repository.db.refresh(field)
repository.event_logger.log_for_session(
session,
actor_user_id=current_user.id,
event_type="assistant_field_semantics_updated",
event_summary="Assistant semantic field update persisted",
event_details={
"field_id": field.field_id,
"candidate_id": entities.get("candidate_id"),
"lock_field": bool(entities.get("lock_field", False)),
"version": int(getattr(session, "version", 0) or 0),
},
)
logger.reflect(
"Assistant semantic field update committed safely",
extra={
"session_id": session_id,
"field_id": field_id,
"version": int(getattr(session, "version", 0) or 0),
},
)
return (
f"Updated semantic field {field.field_name} for dataset review session {session_id}.",
None,
[
AssistantAction(
type="focus_target",
label="Open semantic review",
target=f"field:{field.field_id}",
)
],
)
if operation == "dataset_review_generate_sql_preview":
orchestrator = DatasetReviewOrchestrator(
repository=repository,
config_manager=config_manager,
)
result = orchestrator.prepare_launch_preview(
PreparePreviewCommand(
user=current_user,
session_id=session_id,
expected_version=int(session_version),
)
)
preview_status = getattr(
result.preview.preview_status, "value", result.preview.preview_status
)
logger.reflect(
"Assistant-triggered Superset preview generation completed",
extra={
"session_id": session_id,
"preview_status": preview_status,
},
)
return (
f"SQL preview {preview_status} for dataset review session {session_id}.",
None,
[
AssistantAction(
type="focus_target",
label="Open SQL preview",
target="sql-preview",
)
],
)
raise HTTPException(
status_code=400, detail="Unsupported dataset review operation"
)
# [/DEF:_dispatch_dataset_review_intent:Function]
# [/DEF:AssistantDatasetReviewDispatch:Module]

2
backend/src/api/routes/assistant/_dispatch.py
View File

@@ -36,7 +36,7 @@ from ._resolvers import (
)
from ._history import _coerce_query_bool
from ._llm_planner import _check_any_permission
from ._dataset_review import _dispatch_dataset_review_intent
from ._dataset_review_dispatch import _dispatch_dataset_review_intent
git_service = GitService()

147
backend/src/api/routes/assistant/_llm_planner.py
View File

@@ -4,11 +4,11 @@
# @LAYER: API
# @RELATION: DEPENDS_ON -> [AssistantSchemas]
# @RELATION: DEPENDS_ON -> [AssistantResolvers]
# @RELATION: DISPATCHES -> [AssistantLlmPlannerIntent]
# @INVARIANT: Tool catalog is filtered by user permissions before being sent to LLM.
from __future__ import annotations
import json
from typing import Any, Dict, List, Optional, Tuple
from fastapi import HTTPException
@@ -19,20 +19,13 @@ from src.core.config_manager import ConfigManager
from src.dependencies import has_permission
from src.schemas.auth import User
from src.services.llm_provider import LLMProviderService
from src.services.llm_prompt_templates import (
normalize_llm_settings,
resolve_bound_provider_id,
)
from src.plugins.llm_analysis.service import LLMClient
from src.plugins.llm_analysis.models import LLMProviderType
from src.services.llm_prompt_templates import resolve_bound_provider_id
from ._schemas import (
INTENT_PERMISSION_CHECKS,
AssistantAction,
)
from ._resolvers import (
_get_default_environment_id,
_is_production_env,
_resolve_provider_id,
)
@@ -302,140 +295,4 @@ def _coerce_intent_entities(intent: Dict[str, Any]) -> Dict[str, Any]:
# [/DEF:_coerce_intent_entities:Function]
# [DEF:_plan_intent_with_llm:Function]
# @COMPLEXITY: 2
# @PURPOSE: Use active LLM provider to select best tool/operation from dynamic catalog.
# @PRE: tools list contains allowed operations for current user.
# @POST: Returns normalized intent dict when planning succeeds; otherwise None.
async def _plan_intent_with_llm(
message: str,
tools: List[Dict[str, Any]],
db: Session,
config_manager: ConfigManager,
) -> Optional[Dict[str, Any]]:
if not tools:
return None
llm_settings = normalize_llm_settings(config_manager.get_config().settings.llm)
planner_provider_token = llm_settings.get("assistant_planner_provider")
planner_model_override = llm_settings.get("assistant_planner_model")
llm_service = LLMProviderService(db)
providers = llm_service.get_all_providers()
provider_id = _resolve_provider_id(planner_provider_token, db)
provider = next((p for p in providers if p.id == provider_id), None)
if not provider:
return None
api_key = llm_service.get_decrypted_api_key(provider.id)
if not api_key:
return None
planner = LLMClient(
provider_type=LLMProviderType(provider.provider_type),
api_key=api_key,
base_url=provider.base_url,
default_model=planner_model_override or provider.default_model,
)
system_instruction = (
"You are a deterministic intent planner for backend tools.\n"
"Choose exactly one operation from available_tools or return clarify.\n"
"Output strict JSON object:\n"
"{"
'"domain": string, '
'"operation": string, '
'"entities": object, '
'"confidence": number, '
'"risk_level": "safe"|"guarded"|"dangerous", '
'"requires_confirmation": boolean'
"}\n"
"Rules:\n"
"- Use only operation names from available_tools.\n"
'- If input is ambiguous, operation must be "clarify" with low confidence.\n'
"- If dashboard is provided as name/slug (e.g., COVID), put it into entities.dashboard_ref.\n"
"- Keep entities minimal and factual.\n"
)
payload = {
"available_tools": tools,
"user_message": message,
"known_environments": [
{"id": e.id, "name": e.name} for e in config_manager.get_environments()
],
}
try:
response = await planner.get_json_completion(
[
{"role": "system", "content": system_instruction},
{"role": "user", "content": json.dumps(payload, ensure_ascii=False)},
]
)
except Exception as exc:
import traceback
logger.warning(
f"[assistant.planner][fallback] LLM planner unavailable: {exc}\n{traceback.format_exc()}"
)
return None
if not isinstance(response, dict):
return None
operation = response.get("operation")
valid_ops = {tool["operation"] for tool in tools}
if operation == "clarify":
return {
"domain": "unknown",
"operation": "clarify",
"entities": {},
"confidence": float(response.get("confidence", 0.3)),
"risk_level": "safe",
"requires_confirmation": False,
}
if operation not in valid_ops:
return None
by_operation = {tool["operation"]: tool for tool in tools}
selected = by_operation[operation]
intent = {
"domain": response.get("domain") or selected["domain"],
"operation": operation,
"entities": response.get("entities", {}),
"confidence": float(response.get("confidence", 0.75)),
"risk_level": response.get("risk_level") or selected["risk_level"],
"requires_confirmation": bool(
response.get("requires_confirmation", selected["requires_confirmation"])
),
}
intent = _coerce_intent_entities(intent)
defaults = selected.get("defaults") or {}
for key, value in defaults.items():
if value and not intent["entities"].get(key):
intent["entities"][key] = value
if operation in {"deploy_dashboard", "execute_migration"}:
env_token = intent["entities"].get("environment") or intent["entities"].get(
"target_env"
)
if _is_production_env(env_token, config_manager):
intent["risk_level"] = "dangerous"
intent["requires_confirmation"] = True
return intent
# [/DEF:_plan_intent_with_llm:Function]
# [DEF:_authorize_intent:Function]
# @COMPLEXITY: 2
# @PURPOSE: Validate user permissions for parsed intent before confirmation/dispatch.
# @PRE: intent.operation is present for known assistant command domains.
# @POST: Returns if authorized; raises HTTPException(403) when denied.
def _authorize_intent(intent: Dict[str, Any], current_user: User):
operation = intent.get("operation")
if operation in INTENT_PERMISSION_CHECKS:
_check_any_permission(current_user, INTENT_PERMISSION_CHECKS[operation])
# [/DEF:_authorize_intent:Function]
# [/DEF:AssistantLlmPlanner:Module]

173
backend/src/api/routes/assistant/_llm_planner_intent.py Normal file
View File

@@ -0,0 +1,173 @@
# [DEF:AssistantLlmPlannerIntent:Module]
# @COMPLEXITY: 3
# @PURPOSE: LLM-based intent planning and authorization for the assistant API — separated from tool catalog.
# @LAYER: API
# @RELATION: DEPENDS_ON -> [AssistantLlmPlanner]
# @RELATION: DEPENDS_ON -> [AssistantResolvers]
# @INVARIANT: Production deployments always require confirmation.
from __future__ import annotations
import json
from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session
from src.core.logger import belief_scope, logger
from src.core.config_manager import ConfigManager
from src.schemas.auth import User
from src.services.llm_provider import LLMProviderService
from src.services.llm_prompt_templates import (
normalize_llm_settings,
resolve_bound_provider_id,
)
from src.plugins.llm_analysis.service import LLMClient
from src.plugins.llm_analysis.models import LLMProviderType
from ._schemas import INTENT_PERMISSION_CHECKS
from ._resolvers import (
_is_production_env,
_resolve_provider_id,
)
from ._llm_planner import (
_check_any_permission,
_coerce_intent_entities,
)
# [DEF:_plan_intent_with_llm:Function]
# @COMPLEXITY: 2
# @PURPOSE: Use active LLM provider to select best tool/operation from dynamic catalog.
# @PRE: tools list contains allowed operations for current user.
# @POST: Returns normalized intent dict when planning succeeds; otherwise None.
async def _plan_intent_with_llm(
message: str,
tools: List[Dict[str, Any]],
db: Session,
config_manager: ConfigManager,
) -> Optional[Dict[str, Any]]:
if not tools:
return None
llm_settings = normalize_llm_settings(config_manager.get_config().settings.llm)
planner_provider_token = llm_settings.get("assistant_planner_provider")
planner_model_override = llm_settings.get("assistant_planner_model")
llm_service = LLMProviderService(db)
providers = llm_service.get_all_providers()
provider_id = _resolve_provider_id(planner_provider_token, db)
provider = next((p for p in providers if p.id == provider_id), None)
if not provider:
return None
api_key = llm_service.get_decrypted_api_key(provider.id)
if not api_key:
return None
planner = LLMClient(
provider_type=LLMProviderType(provider.provider_type),
api_key=api_key,
base_url=provider.base_url,
default_model=planner_model_override or provider.default_model,
)
system_instruction = (
"You are a deterministic intent planner for backend tools.\n"
"Choose exactly one operation from available_tools or return clarify.\n"
"Output strict JSON object:\n"
"{"
'"domain": string, '
'"operation": string, '
'"entities": object, '
'"confidence": number, '
'"risk_level": "safe"|"guarded"|"dangerous", '
'"requires_confirmation": boolean'
"}\n"
"Rules:\n"
"- Use only operation names from available_tools.\n"
'- If input is ambiguous, operation must be "clarify" with low confidence.\n'
"- If dashboard is provided as name/slug (e.g., COVID), put it into entities.dashboard_ref.\n"
"- Keep entities minimal and factual.\n"
)
payload = {
"available_tools": tools,
"user_message": message,
"known_environments": [
{"id": e.id, "name": e.name} for e in config_manager.get_environments()
],
}
try:
response = await planner.get_json_completion(
[
{"role": "system", "content": system_instruction},
{"role": "user", "content": json.dumps(payload, ensure_ascii=False)},
]
)
except Exception as exc:
import traceback
logger.warning(
f"[assistant.planner][fallback] LLM planner unavailable: {exc}\n{traceback.format_exc()}"
)
return None
if not isinstance(response, dict):
return None
operation = response.get("operation")
valid_ops = {tool["operation"] for tool in tools}
if operation == "clarify":
return {
"domain": "unknown",
"operation": "clarify",
"entities": {},
"confidence": float(response.get("confidence", 0.3)),
"risk_level": "safe",
"requires_confirmation": False,
}
if operation not in valid_ops:
return None
by_operation = {tool["operation"]: tool for tool in tools}
selected = by_operation[operation]
intent = {
"domain": response.get("domain") or selected["domain"],
"operation": operation,
"entities": response.get("entities", {}),
"confidence": float(response.get("confidence", 0.75)),
"risk_level": response.get("risk_level") or selected["risk_level"],
"requires_confirmation": bool(
response.get("requires_confirmation", selected["requires_confirmation"])
),
}
intent = _coerce_intent_entities(intent)
defaults = selected.get("defaults") or {}
for key, value in defaults.items():
if value and not intent["entities"].get(key):
intent["entities"][key] = value
if operation in {"deploy_dashboard", "execute_migration"}:
env_token = intent["entities"].get("environment") or intent["entities"].get(
"target_env"
)
if _is_production_env(env_token, config_manager):
intent["risk_level"] = "dangerous"
intent["requires_confirmation"] = True
return intent
# [/DEF:_plan_intent_with_llm:Function]
# [DEF:_authorize_intent:Function]
# @COMPLEXITY: 2
# @PURPOSE: Validate user permissions for parsed intent before confirmation/dispatch.
# @PRE: intent.operation is present for known assistant command domains.
# @POST: Returns if authorized; raises HTTPException(403) when denied.
def _authorize_intent(intent: Dict[str, Any], current_user: User):
operation = intent.get("operation")
if operation in INTENT_PERMISSION_CHECKS:
_check_any_permission(current_user, INTENT_PERMISSION_CHECKS[operation])
# [/DEF:_authorize_intent:Function]
# [/DEF:AssistantLlmPlannerIntent:Module]

10
backend/src/api/routes/assistant/_resolvers.py
View File

@@ -38,7 +38,6 @@ def _extract_id(text: str, patterns: List[str]) -> Optional[str]:
# [/DEF:_extract_id:Function]
# [DEF:_resolve_env_id:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve environment identifier/name token to canonical environment id.
@@ -60,7 +59,6 @@ def _resolve_env_id(
# [/DEF:_resolve_env_id:Function]
# [DEF:_is_production_env:Function]
# @COMPLEXITY: 2
# @PURPOSE: Determine whether environment token resolves to production-like target.
@@ -80,7 +78,6 @@ def _is_production_env(token: Optional[str], config_manager: ConfigManager) -> b
# [/DEF:_is_production_env:Function]
# [DEF:_resolve_provider_id:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve provider token to provider id with active/default fallback.
@@ -118,7 +115,6 @@ def _resolve_provider_id(
# [/DEF:_resolve_provider_id:Function]
# [DEF:_get_default_environment_id:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve default environment id from settings or first configured environment.
@@ -144,7 +140,6 @@ def _get_default_environment_id(config_manager: ConfigManager) -> Optional[str]:
# [/DEF:_get_default_environment_id:Function]
# [DEF:_resolve_dashboard_id_by_ref:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve dashboard id by title or slug reference in selected environment.
@@ -198,7 +193,6 @@ def _resolve_dashboard_id_by_ref(
# [/DEF:_resolve_dashboard_id_by_ref:Function]
# [DEF:_resolve_dashboard_id_entity:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve dashboard id from intent entities using numeric id or dashboard_ref fallback.
@@ -241,7 +235,6 @@ def _resolve_dashboard_id_entity(
# [/DEF:_resolve_dashboard_id_entity:Function]
# [DEF:_get_environment_name_by_id:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve human-readable environment name by id.
@@ -260,7 +253,6 @@ def _get_environment_name_by_id(
# [/DEF:_get_environment_name_by_id:Function]
# [DEF:_extract_result_deep_links:Function]
# @COMPLEXITY: 2
# @PURPOSE: Build deep-link actions to verify task result from assistant chat.
@@ -335,7 +327,6 @@ def _extract_result_deep_links(
# [/DEF:_extract_result_deep_links:Function]
# [DEF:_build_task_observability_summary:Function]
# @COMPLEXITY: 2
# @PURPOSE: Build compact textual summary for completed tasks to reduce "black box" effect.
@@ -403,5 +394,4 @@ def _build_task_observability_summary(task: Any, config_manager: ConfigManager)
# [/DEF:_build_task_observability_summary:Function]
# [/DEF:AssistantResolvers:Module]

269
backend/src/api/routes/assistant/_routes.py
View File

@@ -8,6 +8,7 @@
# @RELATION: DEPENDS_ON -> [AssistantLlmPlanner]
# @RELATION: DEPENDS_ON -> [AssistantDatasetReview]
# @RELATION: DEPENDS_ON -> [AssistantDispatch]
# @RELATION: DISPATCHES -> [AssistantAdminRoutes]
# @INVARIANT: Risky operations are never executed without valid confirmation token.
from __future__ import annotations
@@ -60,11 +61,8 @@ from ._history import (
_update_confirmation_state,
)
from ._command_parser import _parse_command
from ._llm_planner import (
_authorize_intent,
_build_tool_catalog,
_plan_intent_with_llm,
)
from ._llm_planner import _build_tool_catalog
from ._llm_planner_intent import _authorize_intent, _plan_intent_with_llm
from ._dataset_review import (
_load_dataset_review_context,
_plan_dataset_review_intent,
@@ -337,265 +335,4 @@ async def cancel_operation(
# [/DEF:cancel_operation:Function]
# [DEF:list_conversations:Function]
# @COMPLEXITY: 2
# @PURPOSE: Return paginated conversation list for current user with archived flag and last message preview.
# @PRE: Authenticated user context and valid pagination params.
# @POST: Conversations are grouped by conversation_id sorted by latest activity descending.
# @RETURN: Dict with items, paging metadata, and archive segmentation counts.
@router.get("/conversations")
async def list_conversations(
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
include_archived: bool = Query(False),
archived_only: bool = Query(False),
search: Optional[str] = Query(None),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
):
with belief_scope("assistant.conversations"):
user_id = current_user.id
include_archived = _coerce_query_bool(include_archived)
archived_only = _coerce_query_bool(archived_only)
_cleanup_history_ttl(db, user_id)
rows = (
db.query(AssistantMessageRecord)
.filter(AssistantMessageRecord.user_id == user_id)
.order_by(desc(AssistantMessageRecord.created_at))
.all()
)
summary: Dict[str, Dict[str, Any]] = {}
for row in rows:
conv_id = row.conversation_id
if not conv_id:
continue
created_at = row.created_at or datetime.utcnow()
if conv_id not in summary:
summary[conv_id] = {
"conversation_id": conv_id,
"title": "",
"updated_at": created_at,
"last_message": row.text,
"last_role": row.role,
"last_state": row.state,
"last_task_id": row.task_id,
"message_count": 0,
}
item = summary[conv_id]
item["message_count"] += 1
if row.role == "user" and row.text and not item["title"]:
item["title"] = row.text.strip()[:80]
items = []
search_term = search.lower().strip() if search else ""
archived_total = sum(
1
for c in summary.values()
if _is_conversation_archived(c.get("updated_at"))
)
active_total = len(summary) - archived_total
for conv in summary.values():
conv["archived"] = _is_conversation_archived(conv.get("updated_at"))
if not conv.get("title"):
conv["title"] = f"Conversation {conv['conversation_id'][:8]}"
if search_term:
haystack = (
f"{conv.get('title', '')} {conv.get('last_message', '')}".lower()
)
if search_term not in haystack:
continue
if archived_only and not conv["archived"]:
continue
if not archived_only and not include_archived and conv["archived"]:
continue
updated = conv.get("updated_at")
conv["updated_at"] = (
updated.isoformat() if isinstance(updated, datetime) else None
)
items.append(conv)
items.sort(key=lambda x: x.get("updated_at") or "", reverse=True)
total = len(items)
start = (page - 1) * page_size
page_items = items[start : start + page_size]
return {
"items": page_items,
"total": total,
"page": page,
"page_size": page_size,
"has_next": start + page_size < total,
"active_total": active_total,
"archived_total": archived_total,
}
# [/DEF:list_conversations:Function]
# [DEF:delete_conversation:Function]
# @COMPLEXITY: 2
# @PURPOSE: Soft-delete or hard-delete a conversation and clear its in-memory trace.
# @PRE: conversation_id belongs to current_user.
# @POST: Conversation records are removed from DB and CONVERSATIONS cache.
@router.delete("/conversations/{conversation_id}")
async def delete_conversation(
conversation_id: str,
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
):
with belief_scope("assistant.conversations.delete"):
user_id = current_user.id
# 1. Remove from in-memory cache
key = (user_id, conversation_id)
if key in CONVERSATIONS:
del CONVERSATIONS[key]
# 2. Delete from database
deleted_count = (
db.query(AssistantMessageRecord)
.filter(
AssistantMessageRecord.user_id == user_id,
AssistantMessageRecord.conversation_id == conversation_id,
)
.delete()
)
db.commit()
if deleted_count == 0:
raise HTTPException(
status_code=404, detail="Conversation not found or already deleted"
)
return {
"status": "success",
"deleted": deleted_count,
"conversation_id": conversation_id,
}
# [/DEF:delete_conversation:Function]
@router.get("/history")
# [DEF:get_history:Function]
# @PURPOSE: Retrieve paginated assistant conversation history for current user.
# @PRE: Authenticated user is available and page params are valid.
# @POST: Returns persistent messages and mirrored in-memory snapshot for diagnostics.
# @RETURN: Dict with items, paging metadata, and resolved conversation_id.
async def get_history(
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
conversation_id: Optional[str] = Query(None),
from_latest: bool = Query(False),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
):
with belief_scope("assistant.history"):
user_id = current_user.id
_cleanup_history_ttl(db, user_id)
conv_id = _resolve_or_create_conversation(user_id, conversation_id, db)
base_query = db.query(AssistantMessageRecord).filter(
AssistantMessageRecord.user_id == user_id,
AssistantMessageRecord.conversation_id == conv_id,
)
total = base_query.count()
start = (page - 1) * page_size
if from_latest:
rows = (
base_query.order_by(desc(AssistantMessageRecord.created_at))
.offset(start)
.limit(page_size)
.all()
)
rows = list(reversed(rows))
else:
rows = (
base_query.order_by(AssistantMessageRecord.created_at.asc())
.offset(start)
.limit(page_size)
.all()
)
persistent_items = [
{
"message_id": row.id,
"conversation_id": row.conversation_id,
"role": row.role,
"text": row.text,
"state": row.state,
"task_id": row.task_id,
"confirmation_id": row.confirmation_id,
"created_at": row.created_at.isoformat() if row.created_at else None,
"metadata": row.payload,
}
for row in rows
]
memory_items = CONVERSATIONS.get((user_id, conv_id), [])
return {
"items": persistent_items,
"memory_items": memory_items,
"total": total,
"page": page,
"page_size": page_size,
"has_next": start + page_size < total,
"from_latest": from_latest,
"conversation_id": conv_id,
}
# [/DEF:get_history:Function]
@router.get("/audit")
# [DEF:get_assistant_audit:Function]
# @PURPOSE: Return assistant audit decisions for current user from persistent and in-memory stores.
# @PRE: User has tasks:READ permission.
# @POST: Audit payload is returned in reverse chronological order from DB.
# @RETURN: Dict with persistent and memory audit slices.
async def get_assistant_audit(
limit: int = Query(50, ge=1, le=500),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
_=Depends(has_permission("tasks", "READ")),
):
with belief_scope("assistant.audit"):
memory_rows = ASSISTANT_AUDIT.get(current_user.id, [])
db_rows = (
db.query(AssistantAuditRecord)
.filter(AssistantAuditRecord.user_id == current_user.id)
.order_by(AssistantAuditRecord.created_at.desc())
.limit(limit)
.all()
)
persistent = [
{
"id": row.id,
"user_id": row.user_id,
"conversation_id": row.conversation_id,
"decision": row.decision,
"task_id": row.task_id,
"message": row.message,
"payload": row.payload,
"created_at": row.created_at.isoformat() if row.created_at else None,
}
for row in db_rows
]
return {
"items": persistent,
"memory_items": memory_rows[-limit:],
"total": len(persistent),
"memory_total": len(memory_rows),
}
# [/DEF:get_assistant_audit:Function]
# [/DEF:AssistantRoutes:Module]

1619
backend/src/api/routes/dashboards.py
View File

File diff suppressed because it is too large Load Diff

44
backend/src/api/routes/dashboards/__init__.py Normal file
View File

@@ -0,0 +1,44 @@
# [DEF:DashboardsApi:Module]
#
# @COMPLEXITY: 5
# @SEMANTICS: api, dashboards, resources, hub
# @PURPOSE: API endpoints for the Dashboard Hub - listing dashboards with Git and task status
# @LAYER: API
# @RELATION: DEPENDS_ON ->[AppDependencies]
# @RELATION: DEPENDS_ON ->[ResourceService]
# @RELATION: DEPENDS_ON ->[SupersetClient]
#
# @INVARIANT: All dashboard responses include git_status and last_task metadata
#
# @PRE: Valid environment configurations exist in ConfigManager.
# @POST: Dashboard responses are projected into DashboardsResponse DTO.
# @SIDE_EFFECT: Performs external calls to Superset API and potentially Git providers.
# @DATA_CONTRACT: Input(env_id, filters) -> Output(DashboardsResponse)
#
# @TEST_CONTRACT: DashboardsAPI -> {
# required_fields: {env_id: string, page: integer, page_size: integer},
# optional_fields: {search: string},
# invariants: ["Pagination must be valid", "Environment must exist"]
# }
#
# @TEST_FIXTURE: dashboard_list_happy -> {
# "env_id": "prod",
# "expected_count": 1,
# "dashboards": [{"id": 1, "title": "Main Revenue"}]
# }
#
# @TEST_EDGE: pagination_zero_page -> {"env_id": "prod", "page": 0, "status": 400}
# @TEST_EDGE: pagination_oversize -> {"env_id": "prod", "page_size": 101, "status": 400}
# @TEST_EDGE: missing_env -> {"env_id": "ghost", "status": 404}
# @TEST_EDGE: empty_dashboards -> {"env_id": "empty_env", "expected_total": 0}
# @TEST_EDGE: external_superset_failure -> {"env_id": "bad_conn", "status": 503}
#
# @TEST_INVARIANT: metadata_consistency -> verifies: [dashboard_list_happy, empty_dashboards]
from ._router import router
from ._schemas import *
from ._listing_routes import *
from ._detail_routes import *
from ._action_routes import *
# [/DEF:DashboardsApi:Module]

176
backend/src/api/routes/dashboards/_action_routes.py Normal file
View File

@@ -0,0 +1,176 @@
# [DEF:DashboardActionRoutes:Module]
# @COMPLEXITY: 2
# @SEMANTICS: api, dashboards, actions, routes
# @PURPOSE: Dashboard action route handlers — migrate, backup.
# @LAYER: API
# @RELATION: DEPENDS_ON -> [DashboardRouter]
# @RELATION: DEPENDS_ON -> [DashboardSchemas]
# [SECTION: IMPORTS]
from fastapi import Depends, HTTPException
from src.dependencies import (
get_config_manager,
get_task_manager,
has_permission,
)
from src.core.logger import logger, belief_scope
from ._schemas import (
MigrateRequest,
BackupRequest,
TaskResponse,
)
from ._router import router
# [/SECTION]
# [DEF:migrate_dashboards:Function]
# @COMPLEXITY: 2
# @PURPOSE: Trigger bulk migration of dashboards from source to target environment
# @PRE: User has permission plugin:migration:execute
# @PRE: source_env_id and target_env_id are valid environment IDs
# @PRE: dashboard_ids is a non-empty list
# @POST: Returns task_id for tracking migration progress
# @POST: Task is created and queued for execution
# @PARAM: request (MigrateRequest) - Migration request with source, target, and dashboard IDs
# @RETURN: TaskResponse - Task ID for tracking
# @RELATION: DISPATCHES ->[MigrationPlugin:execute]
# @RELATION: CALLS ->[TaskManager]
@router.post("/migrate", response_model=TaskResponse)
async def migrate_dashboards(
request: MigrateRequest,
config_manager=Depends(get_config_manager),
task_manager=Depends(get_task_manager),
_=Depends(has_permission("plugin:migration", "EXECUTE")),
):
with belief_scope(
"migrate_dashboards",
f"source={request.source_env_id}, target={request.target_env_id}, count={len(request.dashboard_ids)}",
):
if not request.dashboard_ids:
logger.error(
"[migrate_dashboards][Coherence:Failed] No dashboard IDs provided"
)
raise HTTPException(
status_code=400, detail="At least one dashboard ID must be provided"
)
environments = config_manager.get_environments()
source_env = next(
(e for e in environments if e.id == request.source_env_id), None
)
target_env = next(
(e for e in environments if e.id == request.target_env_id), None
)
if not source_env:
logger.error(
f"[migrate_dashboards][Coherence:Failed] Source environment not found: {request.source_env_id}"
)
raise HTTPException(status_code=404, detail="Source environment not found")
if not target_env:
logger.error(
f"[migrate_dashboards][Coherence:Failed] Target environment not found: {request.target_env_id}"
)
raise HTTPException(status_code=404, detail="Target environment not found")
try:
task_params = {
"source_env_id": request.source_env_id,
"target_env_id": request.target_env_id,
"selected_ids": request.dashboard_ids,
"replace_db_config": request.replace_db_config,
"db_mappings": request.db_mappings or {},
}
task_obj = await task_manager.create_task(
plugin_id="superset-migration", params=task_params
)
logger.info(
f"[migrate_dashboards][Coherence:OK] Migration task created: {task_obj.id} for {len(request.dashboard_ids)} dashboards"
)
return TaskResponse(task_id=str(task_obj.id))
except Exception as e:
logger.error(
f"[migrate_dashboards][Coherence:Failed] Failed to create migration task: {e}"
)
raise HTTPException(
status_code=503, detail=f"Failed to create migration task: {str(e)}"
)
# [/DEF:migrate_dashboards:Function]
# [DEF:backup_dashboards:Function]
# @COMPLEXITY: 2
# @PURPOSE: Trigger bulk backup of dashboards with optional cron schedule
# @PRE: User has permission plugin:backup:execute
# @PRE: env_id is a valid environment ID
# @PRE: dashboard_ids is a non-empty list
# @POST: Returns task_id for tracking backup progress
# @POST: Task is created and queued for execution
# @POST: If schedule is provided, a scheduled task is created
# @PARAM: request (BackupRequest) - Backup request with environment and dashboard IDs
# @RETURN: TaskResponse - Task ID for tracking
# @RELATION: DISPATCHES ->[BackupPlugin:execute]
# @RELATION: CALLS ->[TaskManager]
@router.post("/backup", response_model=TaskResponse)
async def backup_dashboards(
request: BackupRequest,
config_manager=Depends(get_config_manager),
task_manager=Depends(get_task_manager),
_=Depends(has_permission("plugin:backup", "EXECUTE")),
):
with belief_scope(
"backup_dashboards",
f"env={request.env_id}, count={len(request.dashboard_ids)}, schedule={request.schedule}",
):
if not request.dashboard_ids:
logger.error(
"[backup_dashboards][Coherence:Failed] No dashboard IDs provided"
)
raise HTTPException(
status_code=400, detail="At least one dashboard ID must be provided"
)
environments = config_manager.get_environments()
env = next((e for e in environments if e.id == request.env_id), None)
if not env:
logger.error(
f"[backup_dashboards][Coherence:Failed] Environment not found: {request.env_id}"
)
raise HTTPException(status_code=404, detail="Environment not found")
try:
task_params = {
"env": request.env_id,
"dashboards": request.dashboard_ids,
"schedule": request.schedule,
}
task_obj = await task_manager.create_task(
plugin_id="superset-backup", params=task_params
)
logger.info(
f"[backup_dashboards][Coherence:OK] Backup task created: {task_obj.id} for {len(request.dashboard_ids)} dashboards"
)
return TaskResponse(task_id=str(task_obj.id))
except Exception as e:
logger.error(
f"[backup_dashboards][Coherence:Failed] Failed to create backup task: {e}"
)
raise HTTPException(
status_code=503, detail=f"Failed to create backup task: {str(e)}"
)
# [/DEF:backup_dashboards:Function]
# [/DEF:DashboardActionRoutes:Module]

392
backend/src/api/routes/dashboards/_detail_routes.py Normal file
View File

@@ -0,0 +1,392 @@
# [DEF:DashboardDetailRoutes:Module]
# @COMPLEXITY: 3
# @SEMANTICS: api, dashboards, detail, routes
# @PURPOSE: Dashboard detail, db-mappings, task history, thumbnail route handlers.
# @LAYER: API
# @RELATION: DEPENDS_ON -> [DashboardRouter]
# @RELATION: DEPENDS_ON -> [DashboardSchemas]
# @RELATION: DEPENDS_ON -> [DashboardHelpers]
# @RELATION: DEPENDS_ON -> [DashboardProjection]
# [SECTION: IMPORTS]
from typing import Optional, List, Dict, Any
import re
from urllib.parse import urlparse
from fastapi import Depends, HTTPException, Query, Response
from fastapi.responses import JSONResponse
from src.dependencies import (
get_config_manager,
get_task_manager,
get_mapping_service,
has_permission,
)
from src.core.async_superset_client import AsyncSupersetClient
from src.core.superset_client import SupersetClient
from src.core.logger import logger, belief_scope
from src.core.utils.network import DashboardNotFoundError
from ._helpers import (
_resolve_dashboard_id_from_ref,
_resolve_dashboard_id_from_ref_async,
)
from ._projection import (
_task_matches_dashboard,
)
from ._schemas import (
DashboardDetailResponse,
DashboardTaskHistoryItem,
DashboardTaskHistoryResponse,
DatabaseMapping,
DatabaseMappingsResponse,
)
from ._router import router
# [/SECTION]
# [DEF:get_database_mappings:Function]
# @COMPLEXITY: 2
# @PURPOSE: Get database mapping suggestions between source and target environments
# @PRE: User has permission plugin:migration:read
# @PRE: source_env_id and target_env_id are valid environment IDs
# @POST: Returns list of suggested database mappings with confidence scores
# @PARAM: source_env_id (str) - Source environment ID
# @PARAM: target_env_id (str) - Target environment ID
# @RETURN: DatabaseMappingsResponse - List of suggested mappings
# @RELATION: CALLS ->[MappingService:get_suggestions]
@router.get("/db-mappings", response_model=DatabaseMappingsResponse)
async def get_database_mappings(
source_env_id: str,
target_env_id: str,
config_manager=Depends(get_config_manager),
mapping_service=Depends(get_mapping_service),
_=Depends(has_permission("plugin:migration", "READ")),
):
with belief_scope(
"get_database_mappings", f"source={source_env_id}, target={target_env_id}"
):
environments = config_manager.get_environments()
source_env = next((e for e in environments if e.id == source_env_id), None)
target_env = next((e for e in environments if e.id == target_env_id), None)
if not source_env:
logger.error(
f"[get_database_mappings][Coherence:Failed] Source environment not found: {source_env_id}"
)
raise HTTPException(status_code=404, detail="Source environment not found")
if not target_env:
logger.error(
f"[get_database_mappings][Coherence:Failed] Target environment not found: {target_env_id}"
)
raise HTTPException(status_code=404, detail="Target environment not found")
try:
suggestions = await mapping_service.get_suggestions(
source_env_id, target_env_id
)
mappings = [
DatabaseMapping(
source_db=s.get("source_db", ""),
target_db=s.get("target_db", ""),
source_db_uuid=s.get("source_db_uuid"),
target_db_uuid=s.get("target_db_uuid"),
confidence=s.get("confidence", 0.0),
)
for s in suggestions
]
logger.info(
f"[get_database_mappings][Coherence:OK] Returning {len(mappings)} database mapping suggestions"
)
return DatabaseMappingsResponse(mappings=mappings)
except Exception as e:
logger.error(
f"[get_database_mappings][Coherence:Failed] Failed to get database mappings: {e}"
)
raise HTTPException(
status_code=503, detail=f"Failed to get database mappings: {str(e)}"
)
# [/DEF:get_database_mappings:Function]
# [DEF:get_dashboard_detail:Function]
# @COMPLEXITY: 2
# @PURPOSE: Fetch detailed dashboard info with related charts and datasets
# @PRE: env_id must be valid and dashboard ref (slug or id) must exist
# @POST: Returns dashboard detail payload for overview page
# @RELATION: CALLS ->[AsyncSupersetClient]
@router.get("/{dashboard_ref}", response_model=DashboardDetailResponse)
async def get_dashboard_detail(
dashboard_ref: str,
env_id: str,
config_manager=Depends(get_config_manager),
_=Depends(has_permission("plugin:migration", "READ")),
):
with belief_scope(
"get_dashboard_detail", f"dashboard_ref={dashboard_ref}, env_id={env_id}"
):
environments = config_manager.get_environments()
env = next((e for e in environments if e.id == env_id), None)
if not env:
logger.error(
f"[get_dashboard_detail][Coherence:Failed] Environment not found: {env_id}"
)
raise HTTPException(status_code=404, detail="Environment not found")
try:
sync_client = SupersetClient(env)
dashboard_id = _resolve_dashboard_id_from_ref(dashboard_ref, sync_client)
detail = sync_client.get_dashboard_detail(dashboard_id)
logger.info(
f"[get_dashboard_detail][Coherence:OK] Dashboard ref={dashboard_ref} resolved_id={dashboard_id}: {detail.get('chart_count', 0)} charts, {detail.get('dataset_count', 0)} datasets"
)
return DashboardDetailResponse(**detail)
except HTTPException:
raise
except Exception as e:
logger.error(
f"[get_dashboard_detail][Coherence:Failed] Failed to fetch dashboard detail: {e}"
)
raise HTTPException(
status_code=503, detail=f"Failed to fetch dashboard detail: {str(e)}"
)
# [/DEF:get_dashboard_detail:Function]
# [DEF:get_dashboard_tasks_history:Function]
# @COMPLEXITY: 2
# @PURPOSE: Returns history of backup and LLM validation tasks for a dashboard.
# @PRE: dashboard ref (slug or id) is valid.
# @POST: Response contains sorted task history (newest first).
@router.get("/{dashboard_ref}/tasks", response_model=DashboardTaskHistoryResponse)
async def get_dashboard_tasks_history(
dashboard_ref: str,
env_id: Optional[str] = None,
limit: int = Query(20, ge=1, le=100),
config_manager=Depends(get_config_manager),
task_manager=Depends(get_task_manager),
_=Depends(has_permission("tasks", "READ")),
):
with belief_scope(
"get_dashboard_tasks_history",
f"dashboard_ref={dashboard_ref}, env_id={env_id}, limit={limit}",
):
dashboard_id: Optional[int] = None
client: Optional[AsyncSupersetClient] = None
try:
if dashboard_ref.isdigit():
dashboard_id = int(dashboard_ref)
elif env_id:
environments = config_manager.get_environments()
env = next((e for e in environments if e.id == env_id), None)
if not env:
logger.error(
f"[get_dashboard_tasks_history][Coherence:Failed] Environment not found: {env_id}"
)
raise HTTPException(status_code=404, detail="Environment not found")
client = AsyncSupersetClient(env)
dashboard_id = await _resolve_dashboard_id_from_ref_async(
dashboard_ref, client
)
else:
logger.error(
"[get_dashboard_tasks_history][Coherence:Failed] Non-numeric dashboard ref requires env_id"
)
raise HTTPException(
status_code=400,
detail="env_id is required when dashboard reference is a slug",
)
matching_tasks = []
for task in task_manager.get_all_tasks():
if _task_matches_dashboard(task, dashboard_id, env_id):
matching_tasks.append(task)
def _sort_key(task_obj: Any) -> str:
return str(getattr(task_obj, "started_at", "") or "") or str(
getattr(task_obj, "finished_at", "") or ""
)
matching_tasks.sort(key=_sort_key, reverse=True)
selected = matching_tasks[:limit]
items = []
for task in selected:
result = getattr(task, "result", None)
summary = None
validation_status = None
if isinstance(result, dict):
raw_validation_status = result.get("status")
if raw_validation_status is not None:
validation_status = str(raw_validation_status)
summary = (
result.get("summary")
or result.get("status")
or result.get("message")
)
params = getattr(task, "params", {}) or {}
items.append(
DashboardTaskHistoryItem(
id=str(getattr(task, "id", "")),
plugin_id=str(getattr(task, "plugin_id", "")),
status=str(getattr(task, "status", "")),
validation_status=validation_status,
started_at=getattr(task, "started_at", None).isoformat()
if getattr(task, "started_at", None)
else None,
finished_at=getattr(task, "finished_at", None).isoformat()
if getattr(task, "finished_at", None)
else None,
env_id=str(params.get("environment_id") or params.get("env"))
if (params.get("environment_id") or params.get("env"))
else None,
summary=summary,
)
)
logger.info(
f"[get_dashboard_tasks_history][Coherence:OK] Found {len(items)} tasks for dashboard_ref={dashboard_ref}, dashboard_id={dashboard_id}"
)
return DashboardTaskHistoryResponse(dashboard_id=dashboard_id, items=items)
finally:
if client is not None:
await client.aclose()
# [/DEF:get_dashboard_tasks_history:Function]
# [DEF:get_dashboard_thumbnail:Function]
# @COMPLEXITY: 3
# @PURPOSE: Proxies Superset dashboard thumbnail with cache support.
# @RELATION: CALLS ->[AsyncSupersetClient]
# @PRE: env_id must exist.
# @POST: Returns image bytes or 202 when thumbnail is being prepared by Superset.
@router.get("/{dashboard_ref}/thumbnail")
async def get_dashboard_thumbnail(
dashboard_ref: str,
env_id: str,
force: bool = Query(False),
config_manager=Depends(get_config_manager),
_=Depends(has_permission("plugin:migration", "READ")),
):
with belief_scope(
"get_dashboard_thumbnail",
f"dashboard_ref={dashboard_ref}, env_id={env_id}, force={force}",
):
environments = config_manager.get_environments()
env = next((e for e in environments if e.id == env_id), None)
if not env:
logger.error(
f"[get_dashboard_thumbnail][Coherence:Failed] Environment not found: {env_id}"
)
raise HTTPException(status_code=404, detail="Environment not found")
try:
client = SupersetClient(env)
dashboard_id = _resolve_dashboard_id_from_ref(dashboard_ref, client)
digest = None
thumb_endpoint = None
try:
screenshot_payload = client.network.request(
method="POST",
endpoint=f"/dashboard/{dashboard_id}/cache_dashboard_screenshot/",
json={"force": force},
)
payload = (
screenshot_payload.get("result", screenshot_payload)
if isinstance(screenshot_payload, dict)
else {}
)
image_url = (
payload.get("image_url", "") if isinstance(payload, dict) else ""
)
if isinstance(image_url, str) and image_url:
matched = re.search(
r"/dashboard/\d+/(?:thumbnail|screenshot)/([^/]+)/?$",
image_url,
)
if matched:
digest = matched.group(1)
except DashboardNotFoundError:
logger.warning(
"[get_dashboard_thumbnail][Fallback] cache_dashboard_screenshot endpoint unavailable, fallback to dashboard.thumbnail_url"
)
if not digest:
dashboard_payload = client.network.request(
method="GET",
endpoint=f"/dashboard/{dashboard_id}",
)
dashboard_data = (
dashboard_payload.get("result", dashboard_payload)
if isinstance(dashboard_payload, dict)
else {}
)
thumbnail_url = (
dashboard_data.get("thumbnail_url", "")
if isinstance(dashboard_data, dict)
else ""
)
if isinstance(thumbnail_url, str) and thumbnail_url:
parsed = urlparse(thumbnail_url)
parsed_path = parsed.path or thumbnail_url
if parsed_path.startswith("/api/v1/"):
parsed_path = parsed_path[len("/api/v1") :]
thumb_endpoint = parsed_path
matched = re.search(
r"/dashboard/\d+/(?:thumbnail|screenshot)/([^/]+)/?$",
parsed_path,
)
if matched:
digest = matched.group(1)
if not thumb_endpoint:
thumb_endpoint = (
f"/dashboard/{dashboard_id}/thumbnail/{digest or 'latest'}/"
)
thumb_response = client.network.request(
method="GET",
endpoint=thumb_endpoint,
raw_response=True,
allow_redirects=True,
)
if thumb_response.status_code == 202:
payload_202: Dict[str, Any] = {}
try:
payload_202 = thumb_response.json()
except Exception:
payload_202 = {"message": "Thumbnail is being generated"}
return JSONResponse(status_code=202, content=payload_202)
content_type = thumb_response.headers.get("Content-Type", "image/png")
return Response(content=thumb_response.content, media_type=content_type)
except DashboardNotFoundError as e:
logger.error(
f"[get_dashboard_thumbnail][Coherence:Failed] Dashboard not found for thumbnail: {e}"
)
raise HTTPException(status_code=404, detail="Dashboard thumbnail not found")
except HTTPException:
raise
except Exception as e:
logger.error(
f"[get_dashboard_thumbnail][Coherence:Failed] Failed to fetch dashboard thumbnail: {e}"
)
raise HTTPException(
status_code=503,
detail=f"Failed to fetch dashboard thumbnail: {str(e)}",
)
# [/DEF:get_dashboard_thumbnail:Function]
# [/DEF:DashboardDetailRoutes:Module]

187
backend/src/api/routes/dashboards/_helpers.py Normal file
View File

@@ -0,0 +1,187 @@
# [DEF:DashboardHelpers:Module]
# @COMPLEXITY: 2
# @SEMANTICS: api, dashboards, helpers, resolution
# @PURPOSE: Basic helper functions for dashboard route handlers — slug resolution, filter normalization.
# @LAYER: Infra
# @RELATION: DEPENDS_ON -> [SupersetClient]
# @RELATION: DEPENDS_ON -> [AsyncSupersetClient]
from typing import List, Optional, Dict, Any
from fastapi import HTTPException
from src.core.superset_client import SupersetClient
from src.core.async_superset_client import AsyncSupersetClient
from src.core.logger import logger
# [DEF:_find_dashboard_id_by_slug:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve dashboard numeric ID by slug using Superset list endpoint.
# @PRE: `dashboard_slug` is non-empty.
# @POST: Returns dashboard ID when found, otherwise None.
def _find_dashboard_id_by_slug(
client: SupersetClient,
dashboard_slug: str,
) -> Optional[int]:
query_variants = [
{
"filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}],
"page": 0,
"page_size": 1,
},
{
"filters": [{"col": "slug", "op": "eq", "value": dashboard_slug}],
"page": 0,
"page_size": 1,
},
]
for query in query_variants:
try:
_count, dashboards = client.get_dashboards_page(query=query)
if dashboards:
resolved_id = dashboards[0].get("id")
if resolved_id is not None:
return int(resolved_id)
except Exception:
continue
return None
# [/DEF:_find_dashboard_id_by_slug:Function]
# [DEF:_resolve_dashboard_id_from_ref:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve dashboard ID from slug-first reference with numeric fallback.
# @PRE: `dashboard_ref` is provided in route path.
# @POST: Returns a valid dashboard ID or raises HTTPException(404).
def _resolve_dashboard_id_from_ref(
dashboard_ref: str,
client: SupersetClient,
) -> int:
normalized_ref = str(dashboard_ref or "").strip()
if not normalized_ref:
raise HTTPException(status_code=404, detail="Dashboard not found")
# Slug-first: even if ref looks numeric, try slug first.
slug_match_id = _find_dashboard_id_by_slug(client, normalized_ref)
if slug_match_id is not None:
return slug_match_id
if normalized_ref.isdigit():
return int(normalized_ref)
raise HTTPException(status_code=404, detail="Dashboard not found")
# [/DEF:_resolve_dashboard_id_from_ref:Function]
# [DEF:_find_dashboard_id_by_slug_async:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve dashboard numeric ID by slug using async Superset list endpoint.
# @PRE: dashboard_slug is non-empty.
# @POST: Returns dashboard ID when found, otherwise None.
async def _find_dashboard_id_by_slug_async(
client: AsyncSupersetClient,
dashboard_slug: str,
) -> Optional[int]:
query_variants = [
{
"filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}],
"page": 0,
"page_size": 1,
},
{
"filters": [{"col": "slug", "op": "eq", "value": dashboard_slug}],
"page": 0,
"page_size": 1,
},
]
for query in query_variants:
try:
_count, dashboards = await client.get_dashboards_page_async(query=query)
if dashboards:
resolved_id = dashboards[0].get("id")
if resolved_id is not None:
return int(resolved_id)
except Exception:
continue
return None
# [/DEF:_find_dashboard_id_by_slug_async:Function]
# [DEF:_resolve_dashboard_id_from_ref_async:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve dashboard ID from slug-first reference using async Superset client.
# @PRE: dashboard_ref is provided in route path.
# @POST: Returns valid dashboard ID or raises HTTPException(404).
async def _resolve_dashboard_id_from_ref_async(
dashboard_ref: str,
client: AsyncSupersetClient,
) -> int:
normalized_ref = str(dashboard_ref or "").strip()
if not normalized_ref:
raise HTTPException(status_code=404, detail="Dashboard not found")
slug_match_id = await _find_dashboard_id_by_slug_async(client, normalized_ref)
if slug_match_id is not None:
return slug_match_id
if normalized_ref.isdigit():
return int(normalized_ref)
raise HTTPException(status_code=404, detail="Dashboard not found")
# [/DEF:_resolve_dashboard_id_from_ref_async:Function]
# [DEF:_normalize_filter_values:Function]
# @COMPLEXITY: 2
# @PURPOSE: Normalize query filter values to lower-cased non-empty tokens.
# @PRE: values may be None or list of strings.
# @POST: Returns trimmed normalized list preserving input order.
def _normalize_filter_values(values: Optional[List[str]]) -> List[str]:
if not values:
return []
normalized: List[str] = []
for value in values:
token = str(value or "").strip().lower()
if token:
normalized.append(token)
return normalized
# [/DEF:_normalize_filter_values:Function]
# [DEF:_dashboard_git_filter_value:Function]
# @COMPLEXITY: 2
# @PURPOSE: Build comparable git status token for dashboards filtering.
# @PRE: dashboard payload may contain git_status or None.
# @POST: Returns one of ok|diff|no_repo|error|pending.
def _dashboard_git_filter_value(dashboard: Dict[str, Any]) -> str:
git_status = dashboard.get("git_status") or {}
sync_status = str(git_status.get("sync_status") or "").strip().upper()
has_repo = git_status.get("has_repo")
if has_repo is False or sync_status == "NO_REPO":
return "no_repo"
if sync_status == "DIFF":
return "diff"
if sync_status == "OK":
return "ok"
if sync_status == "ERROR":
return "error"
return "pending"
# [/DEF:_dashboard_git_filter_value:Function]
# [/DEF:DashboardHelpers:Module]

398
backend/src/api/routes/dashboards/_listing_routes.py Normal file
View File

@@ -0,0 +1,398 @@
# [DEF:DashboardListingRoutes:Module]
# @COMPLEXITY: 4
# @SEMANTICS: api, dashboards, listing, routes
# @PURPOSE: Dashboard listing route handler for Dashboard Hub.
# @LAYER: API
# @RELATION: DEPENDS_ON -> [DashboardRouter]
# @RELATION: DEPENDS_ON -> [DashboardSchemas]
# @RELATION: DEPENDS_ON -> [DashboardHelpers]
# @RELATION: DEPENDS_ON -> [DashboardProjection]
# [SECTION: IMPORTS]
import os
from typing import List, Optional, Dict, Any, Literal
from fastapi import Depends, HTTPException, Query
from sqlalchemy.orm import Session
from src.dependencies import (
get_config_manager, get_task_manager, get_resource_service,
get_current_user, has_permission,
)
from src.core.database import get_db
from src.core.logger import logger, belief_scope
from src.models.auth import User
from src.services.profile_service import ProfileService
from ._helpers import _normalize_filter_values, _dashboard_git_filter_value
from ._projection import (
_project_dashboard_response_items, _get_profile_filter_binding,
_resolve_profile_actor_aliases, _matches_dashboard_actor_aliases,
)
from ._schemas import EffectiveProfileFilter, DashboardsResponse
from ._router import router
# [/SECTION]
# [DEF:get_dashboards:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetch list of dashboards from a specific environment with Git status and last task status
# @PRE: env_id must be a valid environment ID
# @PRE: page must be >= 1 if provided
# @PRE: page_size must be between 1 and 100 if provided
# @POST: Returns a list of dashboards with enhanced metadata and pagination info
# @POST: Response includes pagination metadata (page, page_size, total, total_pages)
# @POST: Response includes effective profile filter metadata for main dashboards page context
# @PARAM: env_id (str) - The environment ID to fetch dashboards from
# @PARAM: search (Optional[str]) - Filter by title/slug
# @PARAM: page (Optional[int]) - Page number (default: 1)
# @PARAM: page_size (Optional[int]) - Items per page (default: 10, max: 100)
# @RETURN: DashboardsResponse - List of dashboards with status metadata
# @RELATION: CALLS ->[get_dashboards_with_status]
@router.get("", response_model=DashboardsResponse)
async def get_dashboards(
env_id: str,
search: Optional[str] = None,
page: int = 1,
page_size: int = 10,
page_context: Literal["dashboards_main", "other"] = Query("dashboards_main"),
apply_profile_default: bool = Query(default=True),
override_show_all: bool = Query(default=False),
filter_title: Optional[List[str]] = Query(default=None),
filter_git_status: Optional[List[str]] = Query(default=None),
filter_llm_status: Optional[List[str]] = Query(default=None),
filter_changed_on: Optional[List[str]] = Query(default=None),
filter_actor: Optional[List[str]] = Query(default=None),
config_manager=Depends(get_config_manager),
task_manager=Depends(get_task_manager),
resource_service=Depends(get_resource_service),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
_=Depends(has_permission("plugin:migration", "READ")),
):
with belief_scope(
"get_dashboards",
f"env_id={env_id}, search={search}, page={page}, page_size={page_size}, "
f"page_context={page_context}, applyp={apply_profile_default}, overhide={override_show_all}",
):
if page < 1:
logger.error(f"[get_dashboards][Coherence:Failed] Invalid page: {page}")
raise HTTPException(status_code=400, detail="Page must be >= 1")
if page_size < 1 or page_size > 100:
logger.error(f"[get_dashboards][Coherence:Failed] Invalid page_size: {page_size}")
raise HTTPException(status_code=400, detail="Page size must be between 1 and 100")
environments = config_manager.get_environments()
env = next((e for e in environments if e.id == env_id), None)
if not env:
logger.error(f"[get_dashboards][Coherence:Failed] Environment not found: {env_id}")
raise HTTPException(status_code=404, detail="Environment not found")
bound_username: Optional[str] = None
can_apply_profile_filter = False
can_apply_slug_filter = False
effective_profile_filter = EffectiveProfileFilter(
applied=False,
source_page=page_context,
override_show_all=bool(override_show_all),
username=None,
match_logic=None,
)
profile_service: Optional[ProfileService] = None
try:
profile_service_module = getattr(ProfileService, "__module__", "")
is_mock_db = db.__class__.__module__.startswith("unittest.mock")
use_profile_service = (not is_mock_db) or profile_service_module.startswith(
"unittest.mock"
)
if use_profile_service:
profile_service = ProfileService(db=db, config_manager=config_manager)
profile_preference = _get_profile_filter_binding(
profile_service, current_user
)
normalized_username = (
str(profile_preference.get("superset_username_normalized") or "")
.strip()
.lower()
)
raw_username = (
str(profile_preference.get("superset_username") or "")
.strip()
.lower()
)
bound_username = normalized_username or raw_username or None
can_apply_profile_filter = (
page_context == "dashboards_main"
and bool(apply_profile_default)
and not bool(override_show_all)
and bool(profile_preference.get("show_only_my_dashboards", False))
and bool(bound_username)
)
can_apply_slug_filter = (
page_context == "dashboards_main"
and bool(apply_profile_default)
and not bool(override_show_all)
and bool(profile_preference.get("show_only_slug_dashboards", True))
)
profile_match_logic = None
if can_apply_profile_filter and can_apply_slug_filter:
profile_match_logic = "owners_or_modified_by+slug_only"
elif can_apply_profile_filter:
profile_match_logic = "owners_or_modified_by"
elif can_apply_slug_filter:
profile_match_logic = "slug_only"
effective_profile_filter = EffectiveProfileFilter(
applied=bool(can_apply_profile_filter or can_apply_slug_filter),
source_page=page_context,
override_show_all=bool(override_show_all),
username=bound_username if can_apply_profile_filter else None,
match_logic=profile_match_logic,
)
except Exception as profile_error:
logger.explore(
f"[EXPLORE] Profile preference unavailable; continuing without profile-default filter: {profile_error}"
)
try:
all_tasks = task_manager.get_all_tasks()
title_filters = _normalize_filter_values(filter_title)
git_filters = _normalize_filter_values(filter_git_status)
llm_filters = _normalize_filter_values(filter_llm_status)
changed_on_filters = _normalize_filter_values(filter_changed_on)
actor_filters = _normalize_filter_values(filter_actor)
has_column_filters = any(
(
title_filters,
git_filters,
llm_filters,
changed_on_filters,
actor_filters,
)
)
needs_full_scan = (
has_column_filters
or bool(can_apply_profile_filter)
or bool(can_apply_slug_filter)
)
if not needs_full_scan:
try:
page_payload = (
await resource_service.get_dashboards_page_with_status(
env,
all_tasks,
page=page,
page_size=page_size,
search=search,
include_git_status=False,
require_slug=bool(can_apply_slug_filter),
)
)
paginated_dashboards = page_payload["dashboards"]
total = page_payload["total"]
total_pages = page_payload["total_pages"]
except Exception as page_error:
logger.warning(
"[get_dashboards][Action] Page-based fetch failed; using compatibility fallback: %s",
page_error,
)
if can_apply_slug_filter:
dashboards = await resource_service.get_dashboards_with_status(
env,
all_tasks,
include_git_status=False,
require_slug=True,
)
else:
dashboards = await resource_service.get_dashboards_with_status(
env,
all_tasks,
include_git_status=False,
)
if search:
search_lower = search.lower()
dashboards = [
d
for d in dashboards
if search_lower in d.get("title", "").lower()
or search_lower in d.get("slug", "").lower()
]
total = len(dashboards)
total_pages = (
(total + page_size - 1) // page_size if total > 0 else 1
)
start_idx = (page - 1) * page_size
end_idx = start_idx + page_size
paginated_dashboards = dashboards[start_idx:end_idx]
else:
if can_apply_slug_filter:
dashboards = await resource_service.get_dashboards_with_status(
env,
all_tasks,
include_git_status=bool(git_filters),
require_slug=True,
)
else:
dashboards = await resource_service.get_dashboards_with_status(
env,
all_tasks,
include_git_status=bool(git_filters),
)
if (
can_apply_profile_filter
and bound_username
and profile_service is not None
):
actor_aliases = _resolve_profile_actor_aliases(env, bound_username)
if not actor_aliases:
actor_aliases = [bound_username]
logger.reason(
"[REASON] Applying profile actor filter "
f"(env={env_id}, bound_username={bound_username}, actor_aliases={actor_aliases!r}, "
f"dashboards_before={len(dashboards)})"
)
filtered_dashboards: List[Dict[str, Any]] = []
max_actor_samples = 15
for index, dashboard in enumerate(dashboards):
owners_value = dashboard.get("owners")
created_by_value = dashboard.get("created_by")
modified_by_value = dashboard.get("modified_by")
matches_actor = _matches_dashboard_actor_aliases(
profile_service=profile_service,
actor_aliases=actor_aliases,
owners=owners_value,
modified_by=modified_by_value,
)
if index < max_actor_samples:
logger.reflect(
"[REFLECT] Profile actor filter sample "
f"(env={env_id}, dashboard_id={dashboard.get('id')}, "
f"bound_username={bound_username!r}, actor_aliases={actor_aliases!r}, "
f"owners={owners_value!r}, created_by={created_by_value!r}, "
f"modified_by={modified_by_value!r}, matches={matches_actor})"
)
if matches_actor:
filtered_dashboards.append(dashboard)
logger.reflect(
"[REFLECT] Profile actor filter summary "
f"(env={env_id}, bound_username={bound_username!r}, "
f"dashboards_before={len(dashboards)}, dashboards_after={len(filtered_dashboards)})"
)
dashboards = filtered_dashboards
if can_apply_slug_filter:
dashboards = [
dashboard
for dashboard in dashboards
if str(dashboard.get("slug") or "").strip()
]
if search:
search_lower = search.lower()
dashboards = [
d
for d in dashboards
if search_lower in d.get("title", "").lower()
or search_lower in d.get("slug", "").lower()
]
def _matches_dashboard_filters(dashboard: Dict[str, Any]) -> bool:
title_value = str(dashboard.get("title") or "").strip().lower()
if title_filters and title_value not in title_filters:
return False
if git_filters:
git_value = _dashboard_git_filter_value(dashboard)
if git_value not in git_filters:
return False
llm_value = (
str(
(
(dashboard.get("last_task") or {}).get(
"validation_status"
)
)
or "UNKNOWN"
)
.strip()
.lower()
)
if llm_filters and llm_value not in llm_filters:
return False
changed_on_raw = (
str(dashboard.get("last_modified") or "").strip().lower()
)
changed_on_prefix = (
changed_on_raw[:10]
if len(changed_on_raw) >= 10
else changed_on_raw
)
if (
changed_on_filters
and changed_on_raw not in changed_on_filters
and changed_on_prefix not in changed_on_filters
):
return False
owners = dashboard.get("owners") or []
if isinstance(owners, list):
actor_value = ", ".join(
str(item).strip() for item in owners if str(item).strip()
).lower()
else:
actor_value = str(owners).strip().lower()
if not actor_value:
actor_value = "-"
if actor_filters and actor_value not in actor_filters:
return False
return True
if has_column_filters:
dashboards = [
d for d in dashboards if _matches_dashboard_filters(d)
]
total = len(dashboards)
total_pages = (total + page_size - 1) // page_size if total > 0 else 1
start_idx = (page - 1) * page_size
end_idx = start_idx + page_size
paginated_dashboards = dashboards[start_idx:end_idx]
logger.info(
f"[get_dashboards][Coherence:OK] Returning {len(paginated_dashboards)} dashboards "
f"(page {page}/{total_pages}, total: {total}, profile_filter_applied={effective_profile_filter.applied})"
)
response_dashboards = _project_dashboard_response_items(
paginated_dashboards
)
return DashboardsResponse(
dashboards=response_dashboards,
total=total,
page=page,
page_size=page_size,
total_pages=total_pages,
effective_profile_filter=effective_profile_filter,
)
except HTTPException:
raise
except Exception as e:
logger.error(
f"[get_dashboards][Coherence:Failed] Failed to fetch dashboards: {e}"
)
raise HTTPException(
status_code=503, detail=f"Failed to fetch dashboards: {str(e)}"
)
# [/DEF:get_dashboards:Function]
# [/DEF:DashboardListingRoutes:Module]

260
backend/src/api/routes/dashboards/_projection.py Normal file
View File

@@ -0,0 +1,260 @@
# [DEF:DashboardProjection:Module]
# @COMPLEXITY: 2
# @SEMANTICS: api, dashboards, projection, profile, filtering
# @PURPOSE: Dashboard response projection and profile-filter helpers for Dashboard Hub routes.
# @LAYER: Infra
# @RELATION: DEPENDS_ON -> [SupersetClient]
# @RELATION: DEPENDS_ON -> [ProfileService]
from typing import Any, Dict, List, Optional
from src.core.logger import logger, belief_scope
from src.core.superset_client import SupersetClient
from src.core.superset_profile_lookup import SupersetAccountLookupAdapter
from src.models.auth import User
from src.services.profile_service import ProfileService
# [DEF:_normalize_actor_alias_token:Function]
# @COMPLEXITY: 2
# @PURPOSE: Normalize actor alias token to comparable trim+lower text.
def _normalize_actor_alias_token(value: Any) -> Optional[str]:
if value is None:
return None
normalized = str(value).strip().lower()
return normalized if normalized else None
# [/DEF:_normalize_actor_alias_token:Function]
# [DEF:_normalize_owner_display_token:Function]
# @COMPLEXITY: 2
# @PURPOSE: Project owner payload value into stable display string for API response contracts.
def _normalize_owner_display_token(owner: Any) -> Optional[str]:
if owner is None:
return None
if isinstance(owner, dict):
for key in ("username", "full_name", "first_name", "email"):
candidate = owner.get(key)
if isinstance(candidate, str) and candidate.strip():
return candidate.strip()
return None
if isinstance(owner, str):
return owner.strip() or None
return None
# [/DEF:_normalize_owner_display_token:Function]
# [DEF:_normalize_dashboard_owner_values:Function]
# @COMPLEXITY: 2
# @PURPOSE: Normalize dashboard owners payload to optional list of display strings.
def _normalize_dashboard_owner_values(owners: Any) -> Optional[List[str]]:
if owners is None:
return None
raw_items: List[Any]
if isinstance(owners, list):
raw_items = owners
else:
raw_items = [owners]
normalized: List[str] = []
for owner in raw_items:
token = _normalize_owner_display_token(owner)
if token and token not in normalized:
normalized.append(token)
return normalized
# [/DEF:_normalize_dashboard_owner_values:Function]
# [DEF:_project_dashboard_response_items:Function]
# @COMPLEXITY: 2
# @PURPOSE: Project dashboard payloads to response-contract-safe shape.
def _project_dashboard_response_items(
dashboards: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
projected: List[Dict[str, Any]] = []
for dashboard in dashboards:
projected_dashboard = dict(dashboard)
projected_dashboard["owners"] = _normalize_dashboard_owner_values(
projected_dashboard.get("owners")
)
projected.append(projected_dashboard)
return projected
# [/DEF:_project_dashboard_response_items:Function]
# [DEF:_get_profile_filter_binding:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve dashboard profile-filter binding through current or legacy profile service contracts.
def _get_profile_filter_binding(
profile_service: Any, current_user: User
) -> Dict[str, Any]:
def _read_optional_string(value: Any) -> Optional[str]:
return value if isinstance(value, str) else None
def _read_bool(value: Any, default: bool) -> bool:
return value if isinstance(value, bool) else default
if hasattr(profile_service, "get_dashboard_filter_binding"):
binding = profile_service.get_dashboard_filter_binding(current_user)
if isinstance(binding, dict):
return {
"superset_username": _read_optional_string(
binding.get("superset_username")
),
"superset_username_normalized": _read_optional_string(
binding.get("superset_username_normalized")
),
"show_only_my_dashboards": _read_bool(
binding.get("show_only_my_dashboards"), False
),
"show_only_slug_dashboards": _read_bool(
binding.get("show_only_slug_dashboards"), False
),
}
if hasattr(profile_service, "get_my_preference"):
response = profile_service.get_my_preference(current_user)
preference = getattr(response, "preference", None)
return {
"superset_username": _read_optional_string(
getattr(preference, "superset_username", None)
),
"superset_username_normalized": _read_optional_string(
getattr(preference, "superset_username_normalized", None)
),
"show_only_my_dashboards": _read_bool(
getattr(preference, "show_only_my_dashboards", False), False
),
"show_only_slug_dashboards": _read_bool(
getattr(preference, "show_only_slug_dashboards", False), False
),
}
return {
"superset_username": None,
"superset_username_normalized": None,
"show_only_my_dashboards": False,
"show_only_slug_dashboards": False,
}
# [/DEF:_get_profile_filter_binding:Function]
# [DEF:_resolve_profile_actor_aliases:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve stable actor aliases for profile filtering without per-dashboard detail fan-out.
# @SIDE_EFFECT: Performs at most one Superset users-lookup request.
def _resolve_profile_actor_aliases(env: Any, bound_username: str) -> List[str]:
normalized_bound = _normalize_actor_alias_token(bound_username)
if not normalized_bound:
return []
aliases: List[str] = [normalized_bound]
try:
client = SupersetClient(env)
adapter = SupersetAccountLookupAdapter(
network_client=client.network,
environment_id=str(getattr(env, "id", "")),
)
lookup_payload = adapter.get_users_page(
search=normalized_bound,
page_index=0,
page_size=20,
sort_column="username",
sort_order="asc",
)
lookup_items = (
lookup_payload.get("items", []) if isinstance(lookup_payload, dict) else []
)
matched_item: Optional[Dict[str, Any]] = None
for item in lookup_items:
if not isinstance(item, dict):
continue
if _normalize_actor_alias_token(item.get("username")) == normalized_bound:
matched_item = item
break
if matched_item is None:
for item in lookup_items:
if isinstance(item, dict):
matched_item = item
break
display_alias = _normalize_actor_alias_token(
(matched_item or {}).get("display_name")
)
if display_alias and display_alias not in aliases:
aliases.append(display_alias)
logger.reflect(
"[REFLECT] Resolved profile actor aliases "
f"(env={getattr(env, 'id', None)}, bound_username={normalized_bound!r}, "
f"lookup_items={len(lookup_items)}, aliases={aliases!r})"
)
except Exception as alias_error:
logger.explore(
"[EXPLORE] Failed to resolve profile actor aliases via Superset users lookup "
f"(env={getattr(env, 'id', None)}, bound_username={normalized_bound!r}): {alias_error}"
)
return aliases
# [/DEF:_resolve_profile_actor_aliases:Function]
# [DEF:_matches_dashboard_actor_aliases:Function]
# @COMPLEXITY: 2
# @PURPOSE: Apply profile actor matching against multiple aliases (username + optional display name).
def _matches_dashboard_actor_aliases(
profile_service: ProfileService,
actor_aliases: List[str],
owners: Optional[Any],
modified_by: Optional[str],
) -> bool:
for actor_alias in actor_aliases:
if profile_service.matches_dashboard_actor(
bound_username=actor_alias,
owners=owners,
modified_by=modified_by,
):
return True
return False
# [/DEF:_matches_dashboard_actor_aliases:Function]
# [DEF:_task_matches_dashboard:Function]
# @COMPLEXITY: 2
# @PURPOSE: Checks whether task params are tied to a specific dashboard and environment.
def _task_matches_dashboard(
task: Any, dashboard_id: int, env_id: Optional[str]
) -> bool:
plugin_id = getattr(task, "plugin_id", None)
if plugin_id not in {"superset-backup", "llm_dashboard_validation"}:
return False
params = getattr(task, "params", {}) or {}
dashboard_id_str = str(dashboard_id)
if plugin_id == "llm_dashboard_validation":
task_dashboard_id = params.get("dashboard_id")
if str(task_dashboard_id) != dashboard_id_str:
return False
if env_id:
task_env = params.get("environment_id")
return str(task_env) == str(env_id)
return True
dashboard_ids = params.get("dashboard_ids") or params.get("dashboards") or []
normalized_ids = {str(item) for item in dashboard_ids}
if dashboard_id_str not in normalized_ids:
return False
if env_id:
task_env = params.get("environment_id") or params.get("env")
return str(task_env) == str(env_id)
return True
# [/DEF:_task_matches_dashboard:Function]
# [/DEF:DashboardProjection:Module]

8
backend/src/api/routes/dashboards/_router.py Normal file
View File

@@ -0,0 +1,8 @@
# [DEF:DashboardsRouter:Block]
# @COMPLEXITY: 1
# @PURPOSE: Single APIRouter instance for all dashboard endpoints.
# @RELATION: BINDS_TO -> [DashboardsApi]
from fastapi import APIRouter
router = APIRouter(prefix="/api/dashboards", tags=["Dashboards"])
# [/DEF:DashboardsRouter:Block]

232
backend/src/api/routes/dashboards/_schemas.py Normal file
View File

@@ -0,0 +1,232 @@
# [DEF:DashboardSchemas:Module]
# @COMPLEXITY: 1
# @SEMANTICS: api, dashboards, schemas, dto
# @PURPOSE: DTO classes for the Dashboard Hub API.
# @LAYER: Infra
# @RELATION: DEPENDS_ON -> [Pydantic]
from typing import List, Optional, Dict, Any, Literal
from pydantic import BaseModel, Field
# [DEF:GitStatus:DataClass]
# @COMPLEXITY: 1
# @PURPOSE: DTO for dashboard Git synchronization status.
class GitStatus(BaseModel):
branch: Optional[str] = None
sync_status: Optional[str] = Field(None, pattern="^OK|DIFF|NO_REPO|ERROR$")
has_repo: Optional[bool] = None
has_changes_for_commit: Optional[bool] = None
# [/DEF:GitStatus:DataClass]
# [DEF:LastTask:DataClass]
# @COMPLEXITY: 2
# @PURPOSE: DTO for the most recent background task associated with a dashboard.
class LastTask(BaseModel):
task_id: Optional[str] = None
status: Optional[str] = Field(
None,
pattern="^PENDING|RUNNING|SUCCESS|FAILED|ERROR|AWAITING_INPUT|WAITING_INPUT|AWAITING_MAPPING$",
)
validation_status: Optional[str] = Field(None, pattern="^PASS|FAIL|WARN|UNKNOWN$")
# [/DEF:LastTask:DataClass]
# [DEF:DashboardItem:DataClass]
# @COMPLEXITY: 1
# @PURPOSE: DTO representing a single dashboard with projected metadata.
class DashboardItem(BaseModel):
id: int
title: str
slug: Optional[str] = None
url: Optional[str] = None
last_modified: Optional[str] = None
created_by: Optional[str] = None
modified_by: Optional[str] = None
owners: Optional[List[str]] = None
git_status: Optional[GitStatus] = None
last_task: Optional[LastTask] = None
# [/DEF:DashboardItem:DataClass]
# [DEF:EffectiveProfileFilter:DataClass]
# @COMPLEXITY: 1
# @PURPOSE: Metadata about applied profile filters for UI context.
class EffectiveProfileFilter(BaseModel):
applied: bool
source_page: Literal["dashboards_main", "other"] = "dashboards_main"
override_show_all: bool = False
username: Optional[str] = None
match_logic: Optional[
Literal["owners_or_modified_by", "slug_only", "owners_or_modified_by+slug_only"]
] = None
# [/DEF:EffectiveProfileFilter:DataClass]
# [DEF:DashboardsResponse:DataClass]
# @COMPLEXITY: 1
# @PURPOSE: Envelope DTO for paginated dashboards list.
class DashboardsResponse(BaseModel):
dashboards: List[DashboardItem]
total: int
page: int
page_size: int
total_pages: int
effective_profile_filter: Optional[EffectiveProfileFilter] = None
# [/DEF:DashboardsResponse:DataClass]
# [DEF:DashboardChartItem:DataClass]
# @COMPLEXITY: 1
# @PURPOSE: DTO for a chart linked to a dashboard.
class DashboardChartItem(BaseModel):
id: int
title: str
viz_type: Optional[str] = None
dataset_id: Optional[int] = None
last_modified: Optional[str] = None
overview: Optional[str] = None
# [/DEF:DashboardChartItem:DataClass]
# [DEF:DashboardDatasetItem:DataClass]
# @COMPLEXITY: 1
# @PURPOSE: DTO for a dataset associated with a dashboard.
class DashboardDatasetItem(BaseModel):
id: int
table_name: str
schema: Optional[str] = None
database: str
last_modified: Optional[str] = None
overview: Optional[str] = None
# [/DEF:DashboardDatasetItem:DataClass]
# [DEF:DashboardDetailResponse:DataClass]
# @COMPLEXITY: 1
# @PURPOSE: Detailed dashboard metadata including children.
class DashboardDetailResponse(BaseModel):
id: int
title: str
slug: Optional[str] = None
url: Optional[str] = None
description: Optional[str] = None
last_modified: Optional[str] = None
published: Optional[bool] = None
charts: List[DashboardChartItem]
datasets: List[DashboardDatasetItem]
chart_count: int
dataset_count: int
# [/DEF:DashboardDetailResponse:DataClass]
# [DEF:DashboardTaskHistoryItem:DataClass]
# @COMPLEXITY: 1
# @PURPOSE: Individual history record entry.
class DashboardTaskHistoryItem(BaseModel):
id: str
plugin_id: str
status: str
validation_status: Optional[str] = None
started_at: Optional[str] = None
finished_at: Optional[str] = None
env_id: Optional[str] = None
summary: Optional[str] = None
# [/DEF:DashboardTaskHistoryItem:DataClass]
# [DEF:DashboardTaskHistoryResponse:DataClass]
# @COMPLEXITY: 1
# @PURPOSE: Collection DTO for task history.
class DashboardTaskHistoryResponse(BaseModel):
dashboard_id: int
items: List[DashboardTaskHistoryItem]
# [/DEF:DashboardTaskHistoryResponse:DataClass]
# [DEF:DatabaseMapping:DataClass]
# @COMPLEXITY: 2
# @PURPOSE: DTO for cross-environment database ID mapping.
class DatabaseMapping(BaseModel):
source_db: str
target_db: str
source_db_uuid: Optional[str] = None
target_db_uuid: Optional[str] = None
confidence: float
# [/DEF:DatabaseMapping:DataClass]
# [DEF:DatabaseMappingsResponse:DataClass]
# @COMPLEXITY: 1
# @PURPOSE: Wrapper for database mappings.
class DatabaseMappingsResponse(BaseModel):
mappings: List[DatabaseMapping]
# [/DEF:DatabaseMappingsResponse:DataClass]
# [DEF:MigrateRequest:DataClass]
# @COMPLEXITY: 1
# @PURPOSE: DTO for dashboard migration requests.
class MigrateRequest(BaseModel):
source_env_id: str = Field(..., description="Source environment ID")
target_env_id: str = Field(..., description="Target environment ID")
dashboard_ids: List[int] = Field(
..., description="List of dashboard IDs to migrate"
)
db_mappings: Optional[Dict[str, str]] = Field(
None, description="Database mappings for migration"
)
replace_db_config: bool = Field(False, description="Replace database configuration")
# [/DEF:MigrateRequest:DataClass]
# [DEF:TaskResponse:DataClass]
# @COMPLEXITY: 1
# @PURPOSE: DTO for async task ID return.
class TaskResponse(BaseModel):
task_id: str
# [/DEF:TaskResponse:DataClass]
# [DEF:BackupRequest:DataClass]
# @COMPLEXITY: 1
# @PURPOSE: DTO for dashboard backup requests.
class BackupRequest(BaseModel):
env_id: str = Field(..., description="Environment ID")
dashboard_ids: List[int] = Field(..., description="List of dashboard IDs to backup")
schedule: Optional[str] = Field(
None, description="Cron schedule for recurring backups (e.g., '0 0 * * *')"
)
# [/DEF:BackupRequest:DataClass]
# [/DEF:DashboardSchemas:Module]

1757
backend/src/api/routes/git.py
View File

File diff suppressed because it is too large Load Diff

44
backend/src/api/routes/git/__init__.py Normal file
View File

@@ -0,0 +1,44 @@
# [DEF:GitPackage:Module]
# @COMPLEXITY: 3
# @PURPOSE: Package root for decomposed git routes. Re-exports all public symbols from submodules.
# @LAYER: API
# @SEMANTICS: git, package, re-exports
# @RELATION: USES -> [GitRouter, GitDeps, GitHelpers, GitConfigRoutes, GitGiteaRoutes,
# GitRepoRoutes, GitRepoOperationsRoutes, GitRepoLifecycleRoutes,
# GitMergeRoutes, GitEnvironmentRoutes]
# @INVARIANT: git_service and os are module-level attributes for test monkeypatch compatibility.
# All route functions are re-exported for direct access via `from src.api.routes import git`.
import os
from src.services.git_service import GitService
from ._router import router
from ._deps import MAX_REPOSITORY_STATUS_BATCH
# -- git_service singleton (canonical instance; tests may monkeypatch git_routes.git_service) --
git_service = GitService()
# -- Config routes --
from ._config_routes import get_git_configs, create_git_config, update_git_config, delete_git_config, test_git_config # noqa: E501, F401
# -- Gitea routes --
from ._gitea_routes import list_gitea_repositories, create_gitea_repository, delete_gitea_repository, create_remote_repository # noqa: E501, F401
# -- Repo routes (core) --
from ._repo_routes import init_repository, get_repository_binding, delete_repository, get_branches, create_branch, checkout_branch # noqa: E501, F401
from ._helpers import _resolve_dashboard_id_from_ref, _resolve_dashboard_id_from_ref_async, _resolve_repo_key_from_ref # noqa: E501, F401 — re-exported for test monkeypatch compatibility
# -- Repo operations routes (commit, push, pull, status, diff, history, generate-message) --
from ._repo_operations_routes import commit_changes, push_changes, pull_changes, get_repository_status, get_repository_status_batch, get_repository_diff, get_history, generate_commit_message # noqa: E501, F401
# -- Repo lifecycle routes (sync, promote, deploy) --
from ._repo_lifecycle_routes import sync_dashboard, promote_dashboard, deploy_dashboard # noqa: E501, F401
# -- Merge routes --
from ._merge_routes import get_merge_status, get_merge_conflicts, resolve_merge_conflicts, abort_merge, continue_merge # noqa: E501, F401
# -- Environment routes --
from ._environment_routes import get_environments # noqa: E501, F401
# [/DEF:GitPackage:Module]

159
backend/src/api/routes/git/_config_routes.py Normal file
View File

@@ -0,0 +1,159 @@
# [DEF:GitConfigRoutes:Module]
# @COMPLEXITY: 2
# @PURPOSE: FastAPI endpoints for Git server configuration CRUD and connection testing.
# @LAYER: API
# @SEMANTICS: git, config, routes, crud
from typing import List
from fastapi import Depends, HTTPException
from sqlalchemy.orm import Session
from src.core.database import get_db
from src.core.logger import belief_scope, logger
from src.dependencies import has_permission
from src.models.git import GitServerConfig
from src.api.routes.git_schemas import (
GitServerConfigCreate,
GitServerConfigSchema,
GitServerConfigUpdate,
)
from ._router import router
from ._helpers import _get_git_config_or_404
from ._deps import get_git_service
# [DEF:get_git_configs:Function]
# @COMPLEXITY: 2
# @PURPOSE: List all configured Git servers.
@router.get("/config", response_model=List[GitServerConfigSchema])
async def get_git_configs(
db: Session = Depends(get_db),
_=Depends(has_permission("git_config", "READ")),
):
with belief_scope("get_git_configs"):
configs = db.query(GitServerConfig).all()
result = []
for config in configs:
schema = GitServerConfigSchema.from_orm(config)
schema.pat = "********"
result.append(schema)
return result
# [/DEF:get_git_configs:Function]
# [DEF:create_git_config:Function]
# @COMPLEXITY: 2
# @PURPOSE: Register a new Git server configuration.
@router.post("/config", response_model=GitServerConfigSchema)
async def create_git_config(
config: GitServerConfigCreate,
db: Session = Depends(get_db),
_=Depends(has_permission("admin:settings", "WRITE")),
):
with belief_scope("create_git_config"):
config_dict = config.dict(exclude={"config_id"})
db_config = GitServerConfig(**config_dict)
db.add(db_config)
db.commit()
db.refresh(db_config)
return db_config
# [/DEF:create_git_config:Function]
# [DEF:update_git_config:Function]
# @COMPLEXITY: 2
# @PURPOSE: Update an existing Git server configuration.
@router.put("/config/{config_id}", response_model=GitServerConfigSchema)
async def update_git_config(
config_id: str,
config_update: GitServerConfigUpdate,
db: Session = Depends(get_db),
_=Depends(has_permission("admin:settings", "WRITE")),
):
with belief_scope("update_git_config"):
db_config = (
db.query(GitServerConfig).filter(GitServerConfig.id == config_id).first()
)
if not db_config:
raise HTTPException(status_code=404, detail="Configuration not found")
update_data = config_update.dict(exclude_unset=True)
if update_data.get("pat") == "********":
update_data.pop("pat")
for key, value in update_data.items():
setattr(db_config, key, value)
db.commit()
db.refresh(db_config)
result_schema = GitServerConfigSchema.from_orm(db_config)
result_schema.pat = "********"
return result_schema
# [/DEF:update_git_config:Function]
# [DEF:delete_git_config:Function]
# @COMPLEXITY: 2
# @PURPOSE: Remove a Git server configuration.
@router.delete("/config/{config_id}")
async def delete_git_config(
config_id: str,
db: Session = Depends(get_db),
_=Depends(has_permission("admin:settings", "WRITE")),
):
with belief_scope("delete_git_config"):
db_config = (
db.query(GitServerConfig).filter(GitServerConfig.id == config_id).first()
)
if not db_config:
raise HTTPException(status_code=404, detail="Configuration not found")
db.delete(db_config)
db.commit()
return {"status": "success", "message": "Configuration deleted"}
# [/DEF:delete_git_config:Function]
# [DEF:test_git_config:Function]
# @COMPLEXITY: 2
# @PURPOSE: Validate connection to a Git server using provided credentials.
@router.post("/config/test")
async def test_git_config(
config: GitServerConfigCreate,
db: Session = Depends(get_db),
_=Depends(has_permission("git_config", "READ")),
):
with belief_scope("test_git_config"):
_git_service = get_git_service()
pat_to_use = config.pat
if pat_to_use == "********":
if config.config_id:
db_config = (
db.query(GitServerConfig)
.filter(GitServerConfig.id == config.config_id)
.first()
)
if db_config:
pat_to_use = db_config.pat
else:
db_config = (
db.query(GitServerConfig)
.filter(GitServerConfig.url == config.url)
.first()
)
if db_config:
pat_to_use = db_config.pat
success = await _git_service.test_connection(
config.provider, config.url, pat_to_use
)
if success:
return {"status": "success", "message": "Connection successful"}
else:
raise HTTPException(status_code=400, detail="Connection failed")
# [/DEF:test_git_config:Function]
# [/DEF:GitConfigRoutes:Module]

24
backend/src/api/routes/git/_deps.py Normal file
View File

@@ -0,0 +1,24 @@
# [DEF:GitDeps:Module]
# @COMPLEXITY: 1
# @PURPOSE: Shared dependency wiring for monkeypatch-safe git_service access and constants.
# @LAYER: API
# @SEMANTICS: git, deps, service-locator
# @INVARIANT: get_git_service() resolves from sys.modules at call time so test monkeypatching
# of git_routes.git_service (the __init__ attribute) takes effect across all submodules.
import sys
# Batch limit for repository status queries
MAX_REPOSITORY_STATUS_BATCH: int = 50
def get_git_service():
"""Resolve git_service from the git package root at call time.
This indirection ensures that test monkeypatching via
``monkeypatch.setattr(git_routes, 'git_service', mock)``
is visible to every submodule function, because each call
re-resolves ``sys.modules['src.api.routes.git'].git_service``.
"""
return sys.modules["src.api.routes.git"].git_service
# [/DEF:GitDeps:Module]

36
backend/src/api/routes/git/_environment_routes.py Normal file
View File

@@ -0,0 +1,36 @@
# [DEF:GitEnvironmentRoutes:Module]
# @COMPLEXITY: 2
# @PURPOSE: FastAPI endpoint for listing deployment environments.
# @LAYER: API
# @SEMANTICS: git, environments, routes
from typing import List
from fastapi import Depends
from src.core.logger import belief_scope
from src.dependencies import get_config_manager, has_permission
from src.api.routes.git_schemas import DeploymentEnvironmentSchema
from ._router import router
# [DEF:get_environments:Function]
# @COMPLEXITY: 2
# @PURPOSE: List all deployment environments.
@router.get("/environments", response_model=List[DeploymentEnvironmentSchema])
async def get_environments(
config_manager=Depends(get_config_manager),
_=Depends(has_permission("environments", "READ")),
):
with belief_scope("get_environments"):
envs = config_manager.get_environments()
return [
DeploymentEnvironmentSchema(
id=e.id, name=e.name, superset_url=e.url, is_active=True
)
for e in envs
]
# [/DEF:get_environments:Function]
# [/DEF:GitEnvironmentRoutes:Module]

188
backend/src/api/routes/git/_gitea_routes.py Normal file
View File

@@ -0,0 +1,188 @@
# [DEF:GitGiteaRoutes:Module]
# @COMPLEXITY: 2
# @PURPOSE: FastAPI endpoints for Gitea-specific repository operations.
# @LAYER: API
# @SEMANTICS: git, gitea, routes, repositories
from typing import List
from fastapi import Depends, HTTPException
from sqlalchemy.orm import Session
from src.core.database import get_db
from src.core.logger import belief_scope
from src.dependencies import has_permission
from src.models.git import GitProvider
from src.api.routes.git_schemas import (
GiteaRepoCreateRequest,
GiteaRepoSchema,
RemoteRepoCreateRequest,
RemoteRepoSchema,
)
from ._router import router
from ._helpers import _get_git_config_or_404
from ._deps import get_git_service
# [DEF:list_gitea_repositories:Function]
# @COMPLEXITY: 2
# @PURPOSE: List repositories in Gitea for a saved Gitea config.
@router.get("/config/{config_id}/gitea/repos", response_model=List[GiteaRepoSchema])
async def list_gitea_repositories(
config_id: str,
db: Session = Depends(get_db),
_=Depends(has_permission("git_config", "READ")),
):
_gs = get_git_service()
with belief_scope("list_gitea_repositories"):
config = _get_git_config_or_404(db, config_id)
if config.provider != GitProvider.GITEA:
raise HTTPException(
status_code=400, detail="This endpoint supports GITEA provider only"
)
repos = await _gs.list_gitea_repositories(config.url, config.pat)
return [
GiteaRepoSchema(
name=repo.get("name", ""),
full_name=repo.get("full_name", ""),
private=bool(repo.get("private", False)),
clone_url=repo.get("clone_url"),
html_url=repo.get("html_url"),
ssh_url=repo.get("ssh_url"),
default_branch=repo.get("default_branch"),
)
for repo in repos
]
# [/DEF:list_gitea_repositories:Function]
# [DEF:create_gitea_repository:Function]
# @COMPLEXITY: 2
# @PURPOSE: Create a repository in Gitea for a saved Gitea config.
@router.post("/config/{config_id}/gitea/repos", response_model=GiteaRepoSchema)
async def create_gitea_repository(
config_id: str,
request: GiteaRepoCreateRequest,
db: Session = Depends(get_db),
_=Depends(has_permission("admin:settings", "WRITE")),
):
_gs = get_git_service()
with belief_scope("create_gitea_repository"):
config = _get_git_config_or_404(db, config_id)
if config.provider != GitProvider.GITEA:
raise HTTPException(
status_code=400, detail="This endpoint supports GITEA provider only"
)
repo = await _gs.create_gitea_repository(
server_url=config.url,
pat=config.pat,
name=request.name,
private=request.private,
description=request.description,
auto_init=request.auto_init,
default_branch=request.default_branch,
)
return GiteaRepoSchema(
name=repo.get("name", ""),
full_name=repo.get("full_name", ""),
private=bool(repo.get("private", False)),
clone_url=repo.get("clone_url"),
html_url=repo.get("html_url"),
ssh_url=repo.get("ssh_url"),
default_branch=repo.get("default_branch"),
)
# [/DEF:create_gitea_repository:Function]
# [DEF:delete_gitea_repository:Function]
# @COMPLEXITY: 2
# @PURPOSE: Delete repository in Gitea for a saved Gitea config.
@router.delete("/config/{config_id}/gitea/repos/{owner}/{repo_name}")
async def delete_gitea_repository(
config_id: str,
owner: str,
repo_name: str,
db: Session = Depends(get_db),
_=Depends(has_permission("admin:settings", "WRITE")),
):
_gs = get_git_service()
with belief_scope("delete_gitea_repository"):
config = _get_git_config_or_404(db, config_id)
if config.provider != GitProvider.GITEA:
raise HTTPException(
status_code=400, detail="This endpoint supports GITEA provider only"
)
await _gs.delete_gitea_repository(
server_url=config.url,
pat=config.pat,
owner=owner,
repo_name=repo_name,
)
return {"status": "success", "message": "Repository deleted"}
# [/DEF:delete_gitea_repository:Function]
# [DEF:create_remote_repository:Function]
# @COMPLEXITY: 2
# @PURPOSE: Create repository on remote Git server using selected provider config.
@router.post("/config/{config_id}/repositories", response_model=RemoteRepoSchema)
async def create_remote_repository(
config_id: str,
request: RemoteRepoCreateRequest,
db: Session = Depends(get_db),
_=Depends(has_permission("admin:settings", "WRITE")),
):
_gs = get_git_service()
with belief_scope("create_remote_repository"):
config = _get_git_config_or_404(db, config_id)
if config.provider == GitProvider.GITEA:
repo = await _gs.create_gitea_repository(
server_url=config.url,
pat=config.pat,
name=request.name,
private=request.private,
description=request.description,
auto_init=request.auto_init,
default_branch=request.default_branch,
)
elif config.provider == GitProvider.GITHUB:
repo = await _gs.create_github_repository(
server_url=config.url,
pat=config.pat,
name=request.name,
private=request.private,
description=request.description,
auto_init=request.auto_init,
default_branch=request.default_branch,
)
elif config.provider == GitProvider.GITLAB:
repo = await _gs.create_gitlab_repository(
server_url=config.url,
pat=config.pat,
name=request.name,
private=request.private,
description=request.description,
auto_init=request.auto_init,
default_branch=request.default_branch,
)
else:
raise HTTPException(
status_code=501,
detail=f"Provider {config.provider} is not supported",
)
return RemoteRepoSchema(
provider=config.provider,
name=repo.get("name", ""),
full_name=repo.get("full_name", repo.get("name", "")),
private=bool(repo.get("private", False)),
clone_url=repo.get("clone_url"),
html_url=repo.get("html_url"),
ssh_url=repo.get("ssh_url"),
default_branch=repo.get("default_branch"),
)
# [/DEF:create_remote_repository:Function]
# [/DEF:GitGiteaRoutes:Module]

345
backend/src/api/routes/git/_helpers.py Normal file
View File

@@ -0,0 +1,345 @@
# [DEF:GitHelpers:Module]
# @COMPLEXITY: 3
# @PURPOSE: Shared helper functions for Git route modules.
# @LAYER: API
# @SEMANTICS: git, helpers, resolution, identity
# @RELATION USES -> [GitDeps]
# @RELATION USES -> [SupersetClient]
# @RELATION USES -> [UserDashboardPreference]
import os
from typing import Optional
from fastapi import HTTPException
from sqlalchemy.orm import Session
from src.core.logger import logger
from src.core.superset_client import SupersetClient
from src.dependencies import get_config_manager
from src.models.auth import User
from src.models.git import GitServerConfig
from src.models.profile import UserDashboardPreference
from ._deps import get_git_service, MAX_REPOSITORY_STATUS_BATCH
# [DEF:_build_no_repo_status_payload:Function]
# @COMPLEXITY: 1
# @PURPOSE: Build a consistent status payload for dashboards without initialized repositories.
# @POST: Returns a stable payload compatible with frontend repository status parsing.
def _build_no_repo_status_payload() -> dict:
return {
"is_dirty": False,
"untracked_files": [],
"modified_files": [],
"staged_files": [],
"current_branch": None,
"upstream_branch": None,
"has_upstream": False,
"ahead_count": 0,
"behind_count": 0,
"is_diverged": False,
"sync_state": "NO_REPO",
"sync_status": "NO_REPO",
"has_repo": False,
}
# [/DEF:_build_no_repo_status_payload:Function]
# [DEF:_handle_unexpected_git_route_error:Function]
# @COMPLEXITY: 1
# @PURPOSE: Convert unexpected route-level exceptions to stable 500 API responses.
# @PRE: `error` is a non-HTTPException instance.
# @POST: Raises HTTPException(500) with route-specific context.
def _handle_unexpected_git_route_error(route_name: str, error: Exception) -> None:
logger.error(f"[{route_name}][Coherence:Failed] {error}")
raise HTTPException(status_code=500, detail=f"{route_name} failed: {str(error)}")
# [/DEF:_handle_unexpected_git_route_error:Function]
# [DEF:_resolve_repository_status:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve repository status for one dashboard with graceful NO_REPO semantics.
# @PRE: `dashboard_id` is a valid integer.
# @POST: Returns standard status payload or `NO_REPO` payload when repository path is absent.
def _resolve_repository_status(dashboard_id: int) -> dict:
git_service = get_git_service()
repo_path = git_service._get_repo_path(dashboard_id)
if not os.path.exists(repo_path):
logger.debug(
f"[get_repository_status][Action] Repository is not initialized for dashboard {dashboard_id}"
)
return _build_no_repo_status_payload()
try:
return git_service.get_status(dashboard_id)
except HTTPException as e:
if e.status_code == 404:
logger.debug(
f"[get_repository_status][Action] Repository is not initialized for dashboard {dashboard_id}"
)
return _build_no_repo_status_payload()
raise
# [/DEF:_resolve_repository_status:Function]
# [DEF:_get_git_config_or_404:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve GitServerConfig by id or raise 404.
def _get_git_config_or_404(db: Session, config_id: str) -> GitServerConfig:
config = db.query(GitServerConfig).filter(GitServerConfig.id == config_id).first()
if not config:
raise HTTPException(status_code=404, detail="Git configuration not found")
return config
# [/DEF:_get_git_config_or_404:Function]
# [DEF:_find_dashboard_id_by_slug:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve dashboard numeric ID by slug in a specific environment.
def _find_dashboard_id_by_slug(
client: SupersetClient,
dashboard_slug: str,
) -> Optional[int]:
query_variants = [
{
"filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}],
"page": 0,
"page_size": 1,
},
{
"filters": [{"col": "slug", "op": "eq", "value": dashboard_slug}],
"page": 0,
"page_size": 1,
},
]
for query in query_variants:
try:
_count, dashboards = client.get_dashboards_page(query=query)
if dashboards:
resolved_id = dashboards[0].get("id")
if resolved_id is not None:
return int(resolved_id)
except Exception:
continue
return None
# [/DEF:_find_dashboard_id_by_slug:Function]
# [DEF:_resolve_dashboard_id_from_ref:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve dashboard ID from slug-or-id reference for Git routes.
def _resolve_dashboard_id_from_ref(
dashboard_ref: str,
config_manager,
env_id: Optional[str] = None,
) -> int:
normalized_ref = str(dashboard_ref or "").strip()
if not normalized_ref:
raise HTTPException(status_code=400, detail="dashboard_ref is required")
if normalized_ref.isdigit():
return int(normalized_ref)
if not env_id:
raise HTTPException(
status_code=400,
detail="env_id is required for slug-based Git operations",
)
environments = config_manager.get_environments()
env = next((e for e in environments if e.id == env_id), None)
if not env:
raise HTTPException(status_code=404, detail="Environment not found")
dashboard_id = _find_dashboard_id_by_slug(SupersetClient(env), normalized_ref)
if dashboard_id is None:
raise HTTPException(
status_code=404, detail=f"Dashboard slug '{normalized_ref}' not found"
)
return dashboard_id
# [/DEF:_resolve_dashboard_id_from_ref:Function]
# [DEF:_find_dashboard_id_by_slug_async:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve dashboard numeric ID by slug asynchronously for hot-path Git routes.
async def _find_dashboard_id_by_slug_async(
client: "AsyncSupersetClient",
dashboard_slug: str,
) -> Optional[int]:
query_variants = [
{
"filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}],
"page": 0,
"page_size": 1,
},
{
"filters": [{"col": "slug", "op": "eq", "value": dashboard_slug}],
"page": 0,
"page_size": 1,
},
]
for query in query_variants:
try:
_count, dashboards = await client.get_dashboards_page_async(query=query)
if dashboards:
resolved_id = dashboards[0].get("id")
if resolved_id is not None:
return int(resolved_id)
except Exception:
continue
return None
# [/DEF:_find_dashboard_id_by_slug_async:Function]
# [DEF:_resolve_dashboard_id_from_ref_async:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve dashboard ID asynchronously from slug-or-id reference for hot Git routes.
async def _resolve_dashboard_id_from_ref_async(
dashboard_ref: str,
config_manager,
env_id: Optional[str] = None,
) -> int:
normalized_ref = str(dashboard_ref or "").strip()
if not normalized_ref:
raise HTTPException(status_code=400, detail="dashboard_ref is required")
if normalized_ref.isdigit():
return int(normalized_ref)
if not env_id:
raise HTTPException(
status_code=400,
detail="env_id is required for slug-based Git operations",
)
environments = config_manager.get_environments()
env = next((e for e in environments if e.id == env_id), None)
if not env:
raise HTTPException(status_code=404, detail="Environment not found")
from src.core.async_superset_client import AsyncSupersetClient
client = AsyncSupersetClient(env)
try:
dashboard_id = await _find_dashboard_id_by_slug_async(client, normalized_ref)
if dashboard_id is None:
raise HTTPException(
status_code=404, detail=f"Dashboard slug '{normalized_ref}' not found"
)
return dashboard_id
finally:
await client.aclose()
# [/DEF:_resolve_dashboard_id_from_ref_async:Function]
# [DEF:_resolve_repo_key_from_ref:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve repository folder key with slug-first strategy and deterministic fallback.
def _resolve_repo_key_from_ref(
dashboard_ref: str,
dashboard_id: int,
config_manager,
env_id: Optional[str] = None,
) -> str:
normalized_ref = str(dashboard_ref or "").strip()
if normalized_ref and not normalized_ref.isdigit():
return normalized_ref
if env_id:
try:
environments = config_manager.get_environments()
env = next((e for e in environments if e.id == env_id), None)
if env:
payload = SupersetClient(env).get_dashboard(dashboard_id)
dashboard_data = (
payload.get("result", payload) if isinstance(payload, dict) else {}
)
dashboard_slug = dashboard_data.get("slug")
if dashboard_slug:
return str(dashboard_slug)
except Exception:
pass
return f"dashboard-{dashboard_id}"
# [/DEF:_resolve_repo_key_from_ref:Function]
# [DEF:_sanitize_optional_identity_value:Function]
# @COMPLEXITY: 1
# @PURPOSE: Normalize optional identity value into trimmed string or None.
def _sanitize_optional_identity_value(value: Optional[str]) -> Optional[str]:
normalized = str(value or "").strip()
if not normalized:
return None
return normalized
# [/DEF:_sanitize_optional_identity_value:Function]
# [DEF:_resolve_current_user_git_identity:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve configured Git username/email from current user's profile preferences.
def _resolve_current_user_git_identity(
db: Session,
current_user: Optional[User],
) -> Optional[tuple[str, str]]:
if db is None or not hasattr(db, "query"):
return None
user_id = _sanitize_optional_identity_value(getattr(current_user, "id", None))
if not user_id:
return None
try:
preference = (
db.query(UserDashboardPreference)
.filter(UserDashboardPreference.user_id == user_id)
.first()
)
except Exception as resolve_error:
logger.warning(
"[_resolve_current_user_git_identity][Action] Failed to load profile preference for user %s: %s",
user_id,
resolve_error,
)
return None
if not preference:
return None
git_username = _sanitize_optional_identity_value(
getattr(preference, "git_username", None)
)
git_email = _sanitize_optional_identity_value(
getattr(preference, "git_email", None)
)
if not git_username or not git_email:
return None
return git_username, git_email
# [/DEF:_resolve_current_user_git_identity:Function]
# [DEF:_apply_git_identity_from_profile:Function]
# @COMPLEXITY: 2
# @PURPOSE: Apply user-scoped Git identity to repository-local config before write/pull operations.
def _apply_git_identity_from_profile(
dashboard_id: int,
db: Session,
current_user: Optional[User],
) -> None:
identity = _resolve_current_user_git_identity(db, current_user)
if not identity:
return
git_service = get_git_service()
configure_identity = getattr(git_service, "configure_identity", None)
if not callable(configure_identity):
return
git_username, git_email = identity
configure_identity(dashboard_id, git_username, git_email)
# [/DEF:_apply_git_identity_from_profile:Function]
# [/DEF:GitHelpers:Module]

169
backend/src/api/routes/git/_merge_routes.py Normal file
View File

@@ -0,0 +1,169 @@
# [DEF:GitMergeRoutes:Module]
# @COMPLEXITY: 3
# @PURPOSE: FastAPI endpoints for merge operations (status, conflicts, resolve, abort, continue).
# @LAYER: API
# @SEMANTICS: git, merge, routes, conflicts
from typing import List, Optional
from fastapi import Depends, HTTPException
from src.core.logger import belief_scope
from src.dependencies import get_config_manager, has_permission
from src.api.routes.git_schemas import (
MergeConflictFileSchema,
MergeContinueRequest,
MergeResolveRequest,
MergeStatusSchema,
)
from ._router import router
from ._helpers import (
_handle_unexpected_git_route_error,
)
from ._deps import get_git_service
# [DEF:get_merge_status:Function]
# @COMPLEXITY: 2
# @PURPOSE: Return unfinished-merge status for repository (web-only recovery support).
@router.get(
"/repositories/{dashboard_ref}/merge/status", response_model=MergeStatusSchema
)
async def get_merge_status(
dashboard_ref: str,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("get_merge_status"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
return _gs.get_merge_status(dashboard_id)
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("get_merge_status", e)
# [/DEF:get_merge_status:Function]
# [DEF:get_merge_conflicts:Function]
# @COMPLEXITY: 2
# @PURPOSE: Return conflicted files with mine/theirs previews for web conflict resolver.
@router.get(
"/repositories/{dashboard_ref}/merge/conflicts",
response_model=List[MergeConflictFileSchema],
)
async def get_merge_conflicts(
dashboard_ref: str,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("get_merge_conflicts"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
return _gs.get_merge_conflicts(dashboard_id)
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("get_merge_conflicts", e)
# [/DEF:get_merge_conflicts:Function]
# [DEF:resolve_merge_conflicts:Function]
# @COMPLEXITY: 3
# @PURPOSE: Apply mine/theirs/manual conflict resolutions from WebUI and stage files.
# @RELATION: CALLS -> [GitService]
@router.post("/repositories/{dashboard_ref}/merge/resolve")
async def resolve_merge_conflicts(
dashboard_ref: str,
resolve_data: MergeResolveRequest,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("resolve_merge_conflicts"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
resolved_files = _gs.resolve_merge_conflicts(
dashboard_id,
[item.dict() for item in resolve_data.resolutions],
)
return {"status": "success", "resolved_files": resolved_files}
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("resolve_merge_conflicts", e)
# [/DEF:resolve_merge_conflicts:Function]
# [DEF:abort_merge:Function]
# @COMPLEXITY: 2
# @PURPOSE: Abort unfinished merge from WebUI flow.
@router.post("/repositories/{dashboard_ref}/merge/abort")
async def abort_merge(
dashboard_ref: str,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("abort_merge"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
return _gs.abort_merge(dashboard_id)
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("abort_merge", e)
# [/DEF:abort_merge:Function]
# [DEF:continue_merge:Function]
# @COMPLEXITY: 3
# @PURPOSE: Finalize unfinished merge from WebUI flow.
# @RELATION: CALLS -> [GitService]
@router.post("/repositories/{dashboard_ref}/merge/continue")
async def continue_merge(
dashboard_ref: str,
continue_data: MergeContinueRequest,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("continue_merge"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
return _gs.continue_merge(dashboard_id, continue_data.message)
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("continue_merge", e)
# [/DEF:continue_merge:Function]
# [/DEF:GitMergeRoutes:Module]

228
backend/src/api/routes/git/_repo_lifecycle_routes.py Normal file
View File

@@ -0,0 +1,228 @@
# [DEF:GitRepoLifecycleRoutes:Module]
# @COMPLEXITY: 3
# @PURPOSE: FastAPI endpoints for Git lifecycle operations (sync, promote, deploy).
# @LAYER: API
# @SEMANTICS: git, lifecycle, sync, promote, deploy
import typing
from typing import Optional
from fastapi import Depends, HTTPException
from sqlalchemy.orm import Session
from src.core.database import get_db
from src.core.logger import belief_scope, logger
from src.dependencies import get_config_manager, get_current_user, has_permission
from src.models.auth import User
from src.models.git import GitProvider, GitRepository
from src.api.routes.git_schemas import (
DeployRequest,
PromoteRequest,
PromoteResponse,
)
from ._router import router
from ._helpers import (
_apply_git_identity_from_profile,
_get_git_config_or_404,
_handle_unexpected_git_route_error,
)
from ._deps import get_git_service
# [DEF:sync_dashboard:Function]
# @COMPLEXITY: 3
# @PURPOSE: Sync dashboard state from Superset to Git using the GitPlugin.
# @RELATION: CALLS -> [GitPlugin]
@router.post("/repositories/{dashboard_ref}/sync")
async def sync_dashboard(
dashboard_ref: str,
env_id: Optional[str] = None,
source_env_id: typing.Optional[str] = None,
config_manager=Depends(get_config_manager),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
with belief_scope("sync_dashboard"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
from src.plugins.git_plugin import GitPlugin
plugin = GitPlugin()
return await plugin.execute(
{
"operation": "sync",
"dashboard_id": dashboard_id,
"source_env_id": source_env_id,
}
)
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("sync_dashboard", e)
# [/DEF:sync_dashboard:Function]
# [DEF:promote_dashboard:Function]
# @COMPLEXITY: 3
# @PURPOSE: Promote changes between branches via MR or direct merge.
# @RELATION: CALLS -> [GitPlugin]
@router.post("/repositories/{dashboard_ref}/promote", response_model=PromoteResponse)
async def promote_dashboard(
dashboard_ref: str,
payload: PromoteRequest,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("promote_dashboard"):
from . import _resolve_dashboard_id_from_ref
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
db_repo = (
db.query(GitRepository)
.filter(GitRepository.dashboard_id == dashboard_id)
.first()
)
if not db_repo:
raise HTTPException(
status_code=404,
detail=f"Repository for dashboard {dashboard_ref} is not initialized",
)
config = _get_git_config_or_404(db, db_repo.config_id)
from_branch = payload.from_branch.strip()
to_branch = payload.to_branch.strip()
if not from_branch or not to_branch:
raise HTTPException(
status_code=400, detail="from_branch and to_branch are required"
)
if from_branch == to_branch:
raise HTTPException(
status_code=400, detail="from_branch and to_branch must be different"
)
mode = (payload.mode or "mr").strip().lower()
if mode == "direct":
reason = (payload.reason or "").strip()
if not reason:
raise HTTPException(
status_code=400, detail="Direct promote requires non-empty reason"
)
logger.warning(
"[promote_dashboard][PolicyViolation] Direct promote without MR by actor=unknown dashboard_ref=%s from=%s to=%s reason=%s",
dashboard_ref,
from_branch,
to_branch,
reason,
)
_apply_git_identity_from_profile(dashboard_id, db, current_user)
result = _gs.promote_direct_merge(
dashboard_id=dashboard_id,
from_branch=from_branch,
to_branch=to_branch,
)
return PromoteResponse(
mode="direct",
from_branch=from_branch,
to_branch=to_branch,
status=result.get("status", "merged"),
policy_violation=True,
)
title = (payload.title or "").strip() or f"Promote {from_branch} -> {to_branch}"
description = payload.description
if config.provider == GitProvider.GITEA:
pr = await _gs.create_gitea_pull_request(
server_url=config.url,
pat=config.pat,
remote_url=db_repo.remote_url,
from_branch=from_branch,
to_branch=to_branch,
title=title,
description=description,
)
elif config.provider == GitProvider.GITHUB:
pr = await _gs.create_github_pull_request(
server_url=config.url,
pat=config.pat,
remote_url=db_repo.remote_url,
from_branch=from_branch,
to_branch=to_branch,
title=title,
description=description,
draft=payload.draft,
)
elif config.provider == GitProvider.GITLAB:
pr = await _gs.create_gitlab_merge_request(
server_url=config.url,
pat=config.pat,
remote_url=db_repo.remote_url,
from_branch=from_branch,
to_branch=to_branch,
title=title,
description=description,
remove_source_branch=payload.remove_source_branch,
)
else:
raise HTTPException(
status_code=501,
detail=f"Provider {config.provider} does not support promotion API",
)
return PromoteResponse(
mode="mr",
from_branch=from_branch,
to_branch=to_branch,
status=pr.get("status", "opened"),
url=pr.get("url"),
reference_id=str(pr.get("id")) if pr.get("id") is not None else None,
policy_violation=False,
)
# [/DEF:promote_dashboard:Function]
# [DEF:deploy_dashboard:Function]
# @COMPLEXITY: 3
# @PURPOSE: Deploy dashboard from Git to a target environment.
# @RELATION: CALLS -> [GitPlugin]
@router.post("/repositories/{dashboard_ref}/deploy")
async def deploy_dashboard(
dashboard_ref: str,
deploy_data: DeployRequest,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
with belief_scope("deploy_dashboard"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
from src.plugins.git_plugin import GitPlugin
plugin = GitPlugin()
return await plugin.execute(
{
"operation": "deploy",
"dashboard_id": dashboard_id,
"environment_id": deploy_data.environment_id,
}
)
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("deploy_dashboard", e)
# [/DEF:deploy_dashboard:Function]
# [/DEF:GitRepoLifecycleRoutes:Module]

365
backend/src/api/routes/git/_repo_operations_routes.py Normal file
View File

@@ -0,0 +1,365 @@
# [DEF:GitRepoOperationsRoutes:Module]
# @COMPLEXITY: 3
# @PURPOSE: FastAPI endpoints for Git repository operations (commit, push, pull, status, diff, history).
# @LAYER: API
# @SEMANTICS: git, repository, operations, commit, push, pull, status
from typing import List, Optional
from fastapi import Depends, HTTPException
from sqlalchemy.orm import Session
from src.core.database import get_db
from src.core.logger import belief_scope, logger
from src.dependencies import get_config_manager, get_current_user, has_permission
from src.models.auth import User
from src.models.git import GitRepository, GitServerConfig
from src.api.routes.git_schemas import (
CommitCreate,
CommitSchema,
RepoStatusBatchRequest,
RepoStatusBatchResponse,
)
from ._router import router
from ._helpers import (
_apply_git_identity_from_profile,
_build_no_repo_status_payload,
_handle_unexpected_git_route_error,
_resolve_repository_status,
)
from ._deps import get_git_service, MAX_REPOSITORY_STATUS_BATCH
# [DEF:commit_changes:Function]
# @COMPLEXITY: 2
# @PURPOSE: Stage and commit changes in the dashboard's repository.
@router.post("/repositories/{dashboard_ref}/commit")
async def commit_changes(
dashboard_ref: str,
commit_data: CommitCreate,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("commit_changes"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
_apply_git_identity_from_profile(dashboard_id, db, current_user)
_gs.commit_changes(
dashboard_id, commit_data.message, commit_data.files
)
return {"status": "success"}
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("commit_changes", e)
# [/DEF:commit_changes:Function]
# [DEF:push_changes:Function]
# @COMPLEXITY: 2
# @PURPOSE: Push local commits to the remote repository.
@router.post("/repositories/{dashboard_ref}/push")
async def push_changes(
dashboard_ref: str,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("push_changes"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
_gs.push_changes(dashboard_id)
return {"status": "success"}
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("push_changes", e)
# [/DEF:push_changes:Function]
# [DEF:pull_changes:Function]
# @COMPLEXITY: 3
# @PURPOSE: Pull changes from the remote repository.
# @RELATION: CALLS -> [GitService]
@router.post("/repositories/{dashboard_ref}/pull")
async def pull_changes(
dashboard_ref: str,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("pull_changes"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
db_repo = None
config_url = None
config_provider = None
try:
db_repo_candidate = (
db.query(GitRepository)
.filter(GitRepository.dashboard_id == dashboard_id)
.first()
)
if getattr(db_repo_candidate, "config_id", None):
db_repo = db_repo_candidate
config_row = (
db.query(GitServerConfig)
.filter(GitServerConfig.id == db_repo.config_id)
.first()
)
if config_row:
config_url = config_row.url
config_provider = config_row.provider
except Exception as diagnostics_error:
logger.warning(
"[pull_changes][Action] Failed to load repository binding diagnostics for dashboard %s: %s",
dashboard_id,
diagnostics_error,
)
logger.info(
"[pull_changes][Action] Route diagnostics dashboard_ref=%s env_id=%s resolved_dashboard_id=%s "
"binding_exists=%s binding_local_path=%s binding_remote_url=%s binding_config_id=%s config_provider=%s config_url=%s",
dashboard_ref,
env_id,
dashboard_id,
bool(db_repo),
(db_repo.local_path if db_repo else None),
(db_repo.remote_url if db_repo else None),
(db_repo.config_id if db_repo else None),
config_provider,
config_url,
)
_apply_git_identity_from_profile(dashboard_id, db, current_user)
_gs.pull_changes(dashboard_id)
return {"status": "success"}
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("pull_changes", e)
# [/DEF:pull_changes:Function]
# [DEF:get_repository_status:Function]
# @COMPLEXITY: 2
# @PURPOSE: Get current Git status for a dashboard repository.
@router.get("/repositories/{dashboard_ref}/status")
async def get_repository_status(
dashboard_ref: str,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
with belief_scope("get_repository_status"):
from . import _resolve_dashboard_id_from_ref_async
try:
dashboard_id = await _resolve_dashboard_id_from_ref_async(
dashboard_ref, config_manager, env_id
)
return _resolve_repository_status(dashboard_id)
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("get_repository_status", e)
# [/DEF:get_repository_status:Function]
# [DEF:get_repository_status_batch:Function]
# @COMPLEXITY: 2
# @PURPOSE: Get Git statuses for multiple dashboard repositories in one request.
@router.post("/repositories/status/batch", response_model=RepoStatusBatchResponse)
async def get_repository_status_batch(
request: RepoStatusBatchRequest,
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
with belief_scope("get_repository_status_batch"):
dashboard_ids = list(dict.fromkeys(request.dashboard_ids))
if len(dashboard_ids) > MAX_REPOSITORY_STATUS_BATCH:
logger.warning(
"[get_repository_status_batch][Action] Batch size %s exceeds limit %s. Truncating request.",
len(dashboard_ids),
MAX_REPOSITORY_STATUS_BATCH,
)
dashboard_ids = dashboard_ids[:MAX_REPOSITORY_STATUS_BATCH]
statuses = {}
for dashboard_id in dashboard_ids:
try:
statuses[str(dashboard_id)] = _resolve_repository_status(dashboard_id)
except HTTPException:
statuses[str(dashboard_id)] = {
**_build_no_repo_status_payload(),
"sync_state": "ERROR",
"sync_status": "ERROR",
}
except Exception as e:
logger.error(
f"[get_repository_status_batch][Coherence:Failed] Failed for dashboard {dashboard_id}: {e}"
)
statuses[str(dashboard_id)] = {
**_build_no_repo_status_payload(),
"sync_state": "ERROR",
"sync_status": "ERROR",
}
return RepoStatusBatchResponse(statuses=statuses)
# [/DEF:get_repository_status_batch:Function]
# [DEF:get_repository_diff:Function]
# @COMPLEXITY: 2
# @PURPOSE: Get Git diff for a dashboard repository.
@router.get("/repositories/{dashboard_ref}/diff")
async def get_repository_diff(
dashboard_ref: str,
file_path: Optional[str] = None,
staged: bool = False,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("get_repository_diff"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
return _gs.get_diff(dashboard_id, file_path, staged)
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("get_repository_diff", e)
# [/DEF:get_repository_diff:Function]
# [DEF:get_history:Function]
# @COMPLEXITY: 2
# @PURPOSE: View commit history for a dashboard's repository.
@router.get("/repositories/{dashboard_ref}/history", response_model=List[CommitSchema])
async def get_history(
dashboard_ref: str,
limit: int = 50,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("get_history"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
return _gs.get_commit_history(dashboard_id, limit)
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("get_history", e)
# [/DEF:get_history:Function]
# [DEF:generate_commit_message:Function]
# @COMPLEXITY: 3
# @PURPOSE: Generate a suggested commit message using LLM.
# @RELATION: CALLS -> [GitService]
@router.post("/repositories/{dashboard_ref}/generate-message")
async def generate_commit_message(
dashboard_ref: str,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
db: Session = Depends(get_db),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("generate_commit_message"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
diff = _gs.get_diff(dashboard_id, staged=True)
if not diff:
diff = _gs.get_diff(dashboard_id, staged=False)
if not diff:
return {"message": "No changes detected"}
history_objs = _gs.get_commit_history(dashboard_id, limit=5)
history = [h.message for h in history_objs if hasattr(h, "message")]
from ...services.llm_provider import LLMProviderService
from ...plugins.llm_analysis.service import LLMClient
from ...plugins.llm_analysis.models import LLMProviderType
from ...services.llm_prompt_templates import (
DEFAULT_LLM_PROMPTS,
normalize_llm_settings,
resolve_bound_provider_id,
)
llm_service = LLMProviderService(db)
providers = llm_service.get_all_providers()
llm_settings = normalize_llm_settings(
config_manager.get_config().settings.llm
)
bound_provider_id = resolve_bound_provider_id(llm_settings, "git_commit")
provider = next((p for p in providers if p.id == bound_provider_id), None)
if not provider:
provider = next((p for p in providers if p.is_active), None)
if not provider:
raise HTTPException(
status_code=400, detail="No active LLM provider found"
)
api_key = llm_service.get_decrypted_api_key(provider.id)
client = LLMClient(
provider_type=LLMProviderType(provider.provider_type),
api_key=api_key,
base_url=provider.base_url,
default_model=provider.default_model,
)
from ...plugins.git.llm_extension import GitLLMExtension
extension = GitLLMExtension(client)
git_prompt = llm_settings["prompts"].get(
"git_commit_prompt",
DEFAULT_LLM_PROMPTS["git_commit_prompt"],
)
message = await extension.suggest_commit_message(
diff,
history,
prompt_template=git_prompt,
)
return {"message": message}
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("generate_commit_message", e)
# [/DEF:generate_commit_message:Function]
# [/DEF:GitRepoOperationsRoutes:Module]

269
backend/src/api/routes/git/_repo_routes.py Normal file
View File

@@ -0,0 +1,269 @@
# [DEF:GitRepoRoutes:Module]
# @COMPLEXITY: 3
# @PURPOSE: FastAPI endpoints for core Git repository operations (init, binding, branches, checkout).
# @LAYER: API
# @SEMANTICS: git, repository, routes, init, branches
from typing import List, Optional
from fastapi import Depends, HTTPException
from sqlalchemy.orm import Session
from src.core.database import get_db
from src.core.logger import belief_scope, logger
from src.dependencies import get_config_manager, get_current_user, has_permission
from src.models.auth import User
from src.models.git import GitRepository, GitServerConfig
from src.api.routes.git_schemas import (
BranchCheckout,
BranchCreate,
BranchSchema,
RepoInitRequest,
RepositoryBindingSchema,
)
from ._router import router
from ._helpers import (
_apply_git_identity_from_profile,
_get_git_config_or_404,
_handle_unexpected_git_route_error,
)
from ._deps import get_git_service
# [DEF:init_repository:Function]
# @COMPLEXITY: 3
# @PURPOSE: Link a dashboard to a Git repository and perform initial clone/init.
# @RELATION: CALLS -> [GitService]
@router.post("/repositories/{dashboard_ref}/init")
async def init_repository(
dashboard_ref: str,
init_data: RepoInitRequest,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
db: Session = Depends(get_db),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("init_repository"):
from . import _resolve_dashboard_id_from_ref, _resolve_repo_key_from_ref
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
repo_key = _resolve_repo_key_from_ref(
dashboard_ref, dashboard_id, config_manager, env_id
)
config = (
db.query(GitServerConfig)
.filter(GitServerConfig.id == init_data.config_id)
.first()
)
if not config:
raise HTTPException(status_code=404, detail="Git configuration not found")
try:
logger.info(
f"[init_repository][Action] Initializing repo for dashboard {dashboard_id}"
)
_gs.init_repo(
dashboard_id,
init_data.remote_url,
config.pat,
repo_key=repo_key,
default_branch=config.default_branch,
)
repo_path = _gs._get_repo_path(dashboard_id, repo_key=repo_key)
db_repo = (
db.query(GitRepository)
.filter(GitRepository.dashboard_id == dashboard_id)
.first()
)
if not db_repo:
db_repo = GitRepository(
dashboard_id=dashboard_id,
config_id=config.id,
remote_url=init_data.remote_url,
local_path=repo_path,
current_branch="dev",
)
db.add(db_repo)
else:
db_repo.config_id = config.id
db_repo.remote_url = init_data.remote_url
db_repo.local_path = repo_path
db_repo.current_branch = "dev"
db.commit()
logger.info(
f"[init_repository][Coherence:OK] Repository initialized for dashboard {dashboard_id}"
)
return {"status": "success", "message": "Repository initialized"}
except Exception as e:
db.rollback()
logger.error(
f"[init_repository][Coherence:Failed] Failed to init repository: {e}"
)
if isinstance(e, HTTPException):
raise
_handle_unexpected_git_route_error("init_repository", e)
# [/DEF:init_repository:Function]
# [DEF:get_repository_binding:Function]
# @COMPLEXITY: 2
# @PURPOSE: Return repository binding with provider metadata for selected dashboard.
@router.get("/repositories/{dashboard_ref}", response_model=RepositoryBindingSchema)
async def get_repository_binding(
dashboard_ref: str,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
db: Session = Depends(get_db),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
with belief_scope("get_repository_binding"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
db_repo = (
db.query(GitRepository)
.filter(GitRepository.dashboard_id == dashboard_id)
.first()
)
if not db_repo:
raise HTTPException(
status_code=404, detail="Repository not initialized"
)
config = _get_git_config_or_404(db, db_repo.config_id)
return RepositoryBindingSchema(
dashboard_id=db_repo.dashboard_id,
config_id=db_repo.config_id,
provider=config.provider,
remote_url=db_repo.remote_url,
local_path=db_repo.local_path,
)
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("get_repository_binding", e)
# [/DEF:get_repository_binding:Function]
# [DEF:delete_repository:Function]
# @COMPLEXITY: 2
# @PURPOSE: Delete local repository workspace and DB binding for selected dashboard.
@router.delete("/repositories/{dashboard_ref}")
async def delete_repository(
dashboard_ref: str,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("delete_repository"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
_gs.delete_repo(dashboard_id)
return {"status": "success"}
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("delete_repository", e)
# [/DEF:delete_repository:Function]
# [DEF:get_branches:Function]
# @COMPLEXITY: 2
# @PURPOSE: List all branches for a dashboard's repository.
@router.get("/repositories/{dashboard_ref}/branches", response_model=List[BranchSchema])
async def get_branches(
dashboard_ref: str,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("get_branches"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
return _gs.list_branches(dashboard_id)
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("get_branches", e)
# [/DEF:get_branches:Function]
# [DEF:create_branch:Function]
# @COMPLEXITY: 2
# @PURPOSE: Create a new branch in the dashboard's repository.
@router.post("/repositories/{dashboard_ref}/branches")
async def create_branch(
dashboard_ref: str,
branch_data: BranchCreate,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("create_branch"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
_apply_git_identity_from_profile(dashboard_id, db, current_user)
_gs.create_branch(
dashboard_id, branch_data.name, branch_data.from_branch
)
return {"status": "success"}
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("create_branch", e)
# [/DEF:create_branch:Function]
# [DEF:checkout_branch:Function]
# @COMPLEXITY: 2
# @PURPOSE: Switch the dashboard's repository to a specific branch.
@router.post("/repositories/{dashboard_ref}/checkout")
async def checkout_branch(
dashboard_ref: str,
checkout_data: BranchCheckout,
env_id: Optional[str] = None,
config_manager=Depends(get_config_manager),
_=Depends(has_permission("plugin:git", "EXECUTE")),
):
_gs = get_git_service()
with belief_scope("checkout_branch"):
from . import _resolve_dashboard_id_from_ref
try:
dashboard_id = _resolve_dashboard_id_from_ref(
dashboard_ref, config_manager, env_id
)
_gs.checkout_branch(dashboard_id, checkout_data.name)
return {"status": "success"}
except HTTPException:
raise
except Exception as e:
_handle_unexpected_git_route_error("checkout_branch", e)
# [/DEF:checkout_branch:Function]
# [/DEF:GitRepoRoutes:Module]

10
backend/src/api/routes/git/_router.py Normal file
View File

@@ -0,0 +1,10 @@
# [DEF:GitRouter:Module]
# @COMPLEXITY: 1
# @PURPOSE: Shared APIRouter for all Git route modules.
# @LAYER: API
# @SEMANTICS: git, routes, fastapi, router
from fastapi import APIRouter
router = APIRouter(tags=["git"])
# [/DEF:GitRouter:Module]

37
backend/src/api/routes/translate/__init__.py Normal file
View File

@@ -0,0 +1,37 @@
# [DEF:TranslateRoutes:Module]
# @COMPLEXITY: 4
# @SEMANTICS: api, routes, translate
# @PURPOSE: API routes for LLM-based SQL/dashboard translation management including terminology dictionary CRUD and import.
# @LAYER: UI (API)
# @RELATION: DEPENDS_ON -> [TranslateJobResponse]
# @RELATION: DEPENDS_ON -> [DictionaryManager:Class]
# @RELATION: DEPENDS_ON -> [get_current_user]
# @RELATION: DEPENDS_ON -> [get_db]
# @RELATION: DEPENDS_ON -> [has_permission]
# @RELATION: DEPENDS_ON -> [ConfigManager]
# @RELATION: DEPENDS_ON -> [SupersetClient]
# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity.
"""
Translate routes package.
Re-exports the shared router and all route handler modules.
All sub-modules register their handlers on the router at import time.
"""
from ._router import router
from . import _job_routes # noqa: F401 — registers job handlers
from . import _run_routes # noqa: F401 — registers run handlers
from . import _run_list_routes # noqa: F401 — registers run list/detail/csv handlers
from . import _preview_routes # noqa: F401 — registers preview handlers
from . import _dictionary_routes # noqa: F401 — registers dictionary handlers
from . import _schedule_routes # noqa: F401 — registers schedule handlers
from . import _metrics_routes # noqa: F401 — registers metrics handlers
from . import _correction_routes # noqa: F401 — registers correction handlers
__all__ = [
"router",
]
# [/DEF:TranslateRoutes:Module]

97
backend/src/api/routes/translate/_correction_routes.py Normal file
View File

@@ -0,0 +1,97 @@
# [DEF:TranslateCorrectionRoutesModule:Module]
# @COMPLEXITY: 2
# @PURPOSE: Term correction submission endpoints.
# @LAYER: API
# @SEMANTICS: api, routes, translate, corrections
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session
from ....core.database import get_db
from ....core.logger import logger, belief_scope
from ....schemas.auth import User
from ....dependencies import get_current_user, has_permission
from ....plugins.translate.dictionary import DictionaryManager
from ....schemas.translate import (
TermCorrectionSubmit,
TermCorrectionBulkSubmit,
CorrectionSubmitResponse,
)
from ._router import router
# ============================================================
# Corrections
# ============================================================
# [DEF:submit_correction:Function]
# @PURPOSE: Submit a term correction from a run result review.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Correction applied with conflict detection.
@router.post("/corrections")
async def submit_correction(
payload: TermCorrectionSubmit,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.dictionary", "EDIT")),
db: Session = Depends(get_db),
):
"""Submit a term correction from a run result review."""
logger.info(f"[translate_routes][submit_correction] User: {current_user.username}")
if not payload.dictionary_id:
raise HTTPException(status_code=422, detail="dictionary_id is required")
try:
result = DictionaryManager.submit_correction(
db,
dict_id=payload.dictionary_id,
source_term=payload.source_term,
incorrect_target_term=payload.incorrect_target_term,
corrected_target_term=payload.corrected_target_term,
origin_run_id=payload.origin_run_id,
origin_row_key=payload.origin_row_key,
origin_user_id=current_user.username,
)
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:submit_correction:Function]
# [DEF:submit_bulk_corrections:Function]
# @PURPOSE: Submit multiple term corrections atomically.
# @PRE: User has translate.dictionary.edit permission.
# @POST: All corrections applied or none with conflict list.
@router.post("/corrections/bulk")
async def submit_bulk_corrections(
payload: TermCorrectionBulkSubmit,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.dictionary", "EDIT")),
db: Session = Depends(get_db),
):
"""Submit multiple term corrections atomically."""
logger.info(f"[translate_routes][submit_bulk_corrections] User: {current_user.username}")
try:
corr_list = []
for c in payload.corrections:
corr_list.append({
"source_term": c.source_term,
"incorrect_target_term": c.incorrect_target_term,
"corrected_target_term": c.corrected_target_term,
"origin_run_id": c.origin_run_id,
"origin_row_key": c.origin_row_key,
})
result = DictionaryManager.submit_bulk_corrections(
db,
dict_id=payload.dictionary_id,
corrections=corr_list,
origin_user_id=current_user.username,
)
if result.get("status") == "conflicts":
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=result)
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:submit_bulk_corrections:Function]
# [/DEF:TranslateCorrectionRoutesModule:Module]

356
backend/src/api/routes/translate/_dictionary_routes.py Normal file
View File

@@ -0,0 +1,356 @@
# [DEF:TranslateDictionaryRoutesModule:Module]
# @COMPLEXITY: 3
# @PURPOSE: Terminology Dictionary CRUD, entries management, and import routes.
# @LAYER: API
# @SEMANTICS: api, routes, translate, dictionaries
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session
from ....core.database import get_db
from ....core.logger import logger, belief_scope
from ....schemas.auth import User
from ....dependencies import get_current_user, has_permission
from ....plugins.translate.dictionary import DictionaryManager
from ....schemas.translate import (
DictionaryCreate,
DictionaryEntryCreate,
DictionaryEntryResponse,
DictionaryImport,
DictionaryResponse,
)
from ._router import router
from ._helpers import _dict_to_response, _get_dictionary_entry_counts
# ============================================================
# Terminology Dictionaries
# ============================================================
# [DEF:list_dictionaries:Function]
# @PURPOSE: List all terminology dictionaries.
# @PRE: User has translate.dictionary.view permission.
# @POST: Returns list of dictionaries with total count.
@router.get("/dictionaries")
async def list_dictionaries(
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.dictionary", "VIEW")),
db: Session = Depends(get_db),
):
"""List all terminology dictionaries."""
with belief_scope("list_dictionaries"):
logger.reason(f"User: {current_user.username}")
dicts, total = DictionaryManager.list_dictionaries(db, page=page, page_size=page_size)
dict_ids = [d.id for d in dicts]
counts = _get_dictionary_entry_counts(db, dict_ids) if dict_ids else {}
items = [_dict_to_response(d, counts.get(d.id, 0)) for d in dicts]
logger.reflect("Dictionaries listed", {"total": total, "returned": len(items)})
return {"items": items, "total": total, "page": page, "page_size": page_size}
# [/DEF:list_dictionaries:Function]
# [DEF:get_dictionary:Function]
# @PURPOSE: Get a single terminology dictionary by ID.
# @PRE: User has translate.dictionary.view permission.
# @POST: Returns the dictionary with entry_count.
@router.get("/dictionaries/{dictionary_id}")
async def get_dictionary(
dictionary_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.dictionary", "VIEW")),
db: Session = Depends(get_db),
):
"""Get a terminology dictionary by ID."""
with belief_scope("get_dictionary"):
logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}")
try:
d = DictionaryManager.get_dictionary(db, dictionary_id)
from ...models.translate import DictionaryEntry
entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count()
logger.reflect("Dictionary found", {"id": dictionary_id})
return _dict_to_response(d, entry_count)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_dictionary:Function]
# [DEF:create_dictionary:Function]
# @PURPOSE: Create a new terminology dictionary.
# @PRE: User has translate.dictionary.create permission.
# @POST: Returns the created dictionary.
@router.post("/dictionaries", status_code=status.HTTP_201_CREATED)
async def create_dictionary(
payload: DictionaryCreate,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.dictionary", "CREATE")),
db: Session = Depends(get_db),
):
"""Create a new terminology dictionary."""
with belief_scope("create_dictionary"):
logger.reason(f"User: {current_user.username}")
d = DictionaryManager.create_dictionary(
db,
name=payload.name,
source_dialect=payload.source_dialect,
target_dialect=payload.target_dialect,
created_by=current_user.username,
description=payload.description,
is_active=payload.is_active,
)
logger.reflect("Dictionary created", {"id": d.id})
return _dict_to_response(d, 0)
# [/DEF:create_dictionary:Function]
# [DEF:update_dictionary:Function]
# @PURPOSE: Update an existing terminology dictionary.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Returns the updated dictionary.
@router.put("/dictionaries/{dictionary_id}")
async def update_dictionary(
dictionary_id: str,
payload: DictionaryCreate,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.dictionary", "EDIT")),
db: Session = Depends(get_db),
):
"""Update a terminology dictionary."""
with belief_scope("update_dictionary"):
logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}")
try:
d = DictionaryManager.update_dictionary(
db,
dict_id=dictionary_id,
name=payload.name,
description=payload.description,
source_dialect=payload.source_dialect,
target_dialect=payload.target_dialect,
is_active=payload.is_active,
)
from ...models.translate import DictionaryEntry
entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count()
logger.reflect("Dictionary updated", {"id": dictionary_id})
return _dict_to_response(d, entry_count)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:update_dictionary:Function]
# [DEF:delete_dictionary:Function]
# @PURPOSE: Delete a terminology dictionary, blocked if attached to active/scheduled jobs.
# @PRE: User has translate.dictionary.delete permission.
# @POST: Dictionary is deleted.
@router.delete("/dictionaries/{dictionary_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_dictionary(
dictionary_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.dictionary", "DELETE")),
db: Session = Depends(get_db),
):
"""Delete a terminology dictionary."""
with belief_scope("delete_dictionary"):
logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}")
try:
DictionaryManager.delete_dictionary(db, dictionary_id)
logger.reflect("Dictionary deleted", {"id": dictionary_id})
return None
except ValueError as e:
if "active/scheduled" in str(e):
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:delete_dictionary:Function]
# ============================================================
# Dictionary Entries CRUD
# ============================================================
# [DEF:list_dictionary_entries:Function]
# @PURPOSE: List entries for a dictionary.
# @PRE: User has translate.dictionary.view permission.
# @POST: Returns paginated list of entries.
@router.get("/dictionaries/{dictionary_id}/entries")
async def list_dictionary_entries(
dictionary_id: str,
page: int = Query(1, ge=1),
page_size: int = Query(50, ge=1, le=500),
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.dictionary", "VIEW")),
db: Session = Depends(get_db),
):
"""List entries for a dictionary."""
with belief_scope("list_dictionary_entries"):
logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}")
try:
entries, total = DictionaryManager.list_entries(db, dictionary_id, page=page, page_size=page_size)
items = [
{
"id": e.id,
"dictionary_id": e.dictionary_id,
"source_term": e.source_term,
"source_term_normalized": e.source_term_normalized,
"target_term": e.target_term,
"context_notes": e.context_notes,
"created_at": e.created_at,
"updated_at": e.updated_at,
}
for e in entries
]
logger.reflect("Entries listed", {"dict_id": dictionary_id, "total": total})
return {"items": items, "total": total, "page": page, "page_size": page_size}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:list_dictionary_entries:Function]
# [DEF:add_dictionary_entry:Function]
# @PURPOSE: Add a new entry to a dictionary.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Entry is created.
@router.post("/dictionaries/{dictionary_id}/entries", status_code=status.HTTP_201_CREATED)
async def add_dictionary_entry(
dictionary_id: str,
payload: DictionaryEntryCreate,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.dictionary", "EDIT")),
db: Session = Depends(get_db),
):
"""Add a new entry to a dictionary."""
with belief_scope("add_dictionary_entry"):
logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}")
try:
entry = DictionaryManager.add_entry(
db, dictionary_id,
source_term=payload.source_term,
target_term=payload.target_term,
context_notes=payload.context_notes,
)
logger.reflect("Entry added", {"entry_id": entry.id})
return {
"id": entry.id,
"dictionary_id": entry.dictionary_id,
"source_term": entry.source_term,
"source_term_normalized": entry.source_term_normalized,
"target_term": entry.target_term,
"context_notes": entry.context_notes,
"created_at": entry.created_at,
"updated_at": entry.updated_at,
}
except ValueError as e:
if "already exists" in str(e):
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:add_dictionary_entry:Function]
# [DEF:edit_dictionary_entry:Function]
# @PURPOSE: Update an existing dictionary entry.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Entry is updated.
@router.put("/dictionaries/{dictionary_id}/entries/{entry_id}")
async def edit_dictionary_entry(
dictionary_id: str,
entry_id: str,
payload: DictionaryEntryCreate,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.dictionary", "EDIT")),
db: Session = Depends(get_db),
):
"""Update an existing dictionary entry."""
with belief_scope("edit_dictionary_entry"):
logger.reason(f"Entry: {entry_id}, User: {current_user.username}")
try:
entry = DictionaryManager.edit_entry(
db, entry_id,
source_term=payload.source_term,
target_term=payload.target_term,
context_notes=payload.context_notes,
)
logger.reflect("Entry updated", {"entry_id": entry_id})
return {
"id": entry.id,
"dictionary_id": entry.dictionary_id,
"source_term": entry.source_term,
"source_term_normalized": entry.source_term_normalized,
"target_term": entry.target_term,
"context_notes": entry.context_notes,
"created_at": entry.created_at,
"updated_at": entry.updated_at,
}
except ValueError as e:
if "already exists" in str(e):
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:edit_dictionary_entry:Function]
# [DEF:delete_dictionary_entry:Function]
# @PURPOSE: Delete a dictionary entry.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Entry is deleted.
@router.delete("/dictionaries/{dictionary_id}/entries/{entry_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_dictionary_entry(
dictionary_id: str,
entry_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.dictionary", "EDIT")),
db: Session = Depends(get_db),
):
"""Delete a dictionary entry."""
with belief_scope("delete_dictionary_entry"):
logger.reason(f"Entry: {entry_id}, User: {current_user.username}")
try:
DictionaryManager.delete_entry(db, entry_id)
logger.reflect("Entry deleted", {"entry_id": entry_id})
return None
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:delete_dictionary_entry:Function]
# [DEF:import_dictionary_entries:Function]
# @PURPOSE: Import entries into a terminology dictionary from CSV/TSV content.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Entries are imported per conflict mode.
@router.post("/dictionaries/{dictionary_id}/import")
async def import_dictionary_entries(
dictionary_id: str,
payload: DictionaryImport,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.dictionary", "EDIT")),
db: Session = Depends(get_db),
):
"""Import entries into a terminology dictionary from CSV/TSV content."""
with belief_scope("import_dictionary_entries"):
logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}")
if payload.on_conflict not in ("overwrite", "keep_existing", "cancel"):
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail="on_conflict must be 'overwrite', 'keep_existing', or 'cancel'",
)
try:
result = DictionaryManager.import_entries(
db,
dict_id=dictionary_id,
content=payload.content,
delimiter=payload.delimiter,
on_conflict=payload.on_conflict,
preview_only=payload.preview_only,
)
logger.reflect("Import completed", {
"dict_id": dictionary_id,
"total": result["total"],
"errors": len(result["errors"]),
})
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:import_dictionary_entries:Function]
# [/DEF:TranslateDictionaryRoutesModule:Module]

75
backend/src/api/routes/translate/_helpers.py Normal file
View File

@@ -0,0 +1,75 @@
# [DEF:TranslateHelpersModule:Module]
# @COMPLEXITY: 2
# @PURPOSE: Shared helper functions for translate route handlers.
# @LAYER: API
# @SEMANTICS: api, routes, translate, helpers
# @RELATION: DEPENDS_ON -> [models.translate]
from typing import Any, Dict, List
from sqlalchemy.orm import Session
# [DEF:_run_to_response:Function]
# @COMPLEXITY: 2
# @PURPOSE: Convert TranslationRun ORM to response dict.
def _run_to_response(run: Any) -> dict:
return {
"id": run.id,
"job_id": run.job_id,
"status": run.status,
"trigger_type": run.trigger_type,
"started_at": run.started_at.isoformat() if run.started_at else None,
"completed_at": run.completed_at.isoformat() if run.completed_at else None,
"error_message": run.error_message,
"total_records": run.total_records or 0,
"successful_records": run.successful_records or 0,
"failed_records": run.failed_records or 0,
"skipped_records": run.skipped_records or 0,
"insert_status": run.insert_status,
"superset_execution_id": run.superset_execution_id,
"config_snapshot": run.config_snapshot,
"key_hash": run.key_hash,
"config_hash": run.config_hash,
"dict_snapshot_hash": run.dict_snapshot_hash,
"created_by": run.created_by,
"created_at": run.created_at.isoformat() if run.created_at else None,
}
# [/DEF:_run_to_response:Function]
# [DEF:_dict_to_response:Function]
# @COMPLEXITY: 2
# @PURPOSE: Convert a TerminologyDictionary ORM model to a response dict with computed entry_count.
@staticmethod
def _dict_to_response(d: Any, entry_count: int = 0) -> dict:
return {
"id": d.id,
"name": d.name,
"description": d.description,
"source_dialect": d.source_dialect,
"target_dialect": d.target_dialect,
"is_active": d.is_active,
"created_by": d.created_by,
"created_at": d.created_at,
"updated_at": d.updated_at,
"entry_count": entry_count,
}
# [/DEF:_dict_to_response:Function]
# [DEF:_get_dictionary_entry_counts:Function]
# @COMPLEXITY: 2
# @PURPOSE: Get entry counts for a list of dictionary IDs.
def _get_dictionary_entry_counts(db: Session, dict_ids: List[str]) -> Dict[str, int]:
from sqlalchemy import func
from ...models.translate import DictionaryEntry
counts = (
db.query(DictionaryEntry.dictionary_id, func.count(DictionaryEntry.id))
.filter(DictionaryEntry.dictionary_id.in_(dict_ids))
.group_by(DictionaryEntry.dictionary_id)
.all()
)
return {row[0]: row[1] for row in counts}
# [/DEF:_get_dictionary_entry_counts:Function]
# [/DEF:TranslateHelpersModule:Module]

250
backend/src/api/routes/translate/_job_routes.py Normal file
View File

@@ -0,0 +1,250 @@
# [DEF:TranslateJobRoutesModule:Module]
# @COMPLEXITY: 3
# @PURPOSE: Translation Job CRUD and datasource column routes.
# @LAYER: API
# @SEMANTICS: api, routes, translate, jobs
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session
from ....core.database import get_db
from ....core.logger import logger, belief_scope
from ....schemas.auth import User
from ....dependencies import get_current_user, has_permission, get_config_manager
from ....core.config_manager import ConfigManager
from ....plugins.translate.service import (
TranslateJobService,
job_to_response,
get_datasource_columns as fetch_datasource_columns_service,
)
from ....schemas.translate import (
TranslateJobCreate,
TranslateJobUpdate,
TranslateJobResponse,
DatasourceColumnsResponse,
DuplicateJobResponse,
)
from ._router import router
# ============================================================
# Translation Job CRUD
# ============================================================
# [DEF:list_jobs:Function]
# @PURPOSE: List all translation jobs.
# @PRE: User has translate.job.view permission.
# @POST: Returns list of translation jobs.
@router.get("/jobs", response_model=List[TranslateJobResponse])
async def list_jobs(
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
status: Optional[str] = Query(None),
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "VIEW")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""List all translation jobs."""
logger.info(f"[translate_routes][list_jobs] User: {current_user.username}")
try:
service = TranslateJobService(db, config_manager, current_user.username)
total, jobs = service.list_jobs(
page=page,
page_size=page_size,
status_filter=status,
)
results = []
for job in jobs:
dict_ids = service.get_job_dictionary_ids(job.id)
results.append(job_to_response(job, dict_ids))
return results
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:list_jobs:Function]
# [DEF:get_job:Function]
# @PURPOSE: Get a single translation job by ID.
# @PRE: User has translate.job.view permission.
# @POST: Returns the translation job.
@router.get("/jobs/{job_id}", response_model=TranslateJobResponse)
async def get_job(
job_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "VIEW")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Get a translation job by ID."""
logger.info(f"[translate_routes][get_job] Job: {job_id}, User: {current_user.username}")
try:
service = TranslateJobService(db, config_manager, current_user.username)
job = service.get_job(job_id)
dict_ids = service.get_job_dictionary_ids(job.id)
return job_to_response(job, dict_ids)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_job:Function]
# [DEF:create_job:Function]
# @PURPOSE: Create a new translation job.
# @PRE: User has translate.job.create permission.
# @POST: Returns the created translation job.
# @SIDE_EFFECT: Validates columns via SupersetClient; caches database_dialect.
@router.post("/jobs", response_model=TranslateJobResponse, status_code=status.HTTP_201_CREATED)
async def create_job(
payload: TranslateJobCreate,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "CREATE")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Create a new translation job."""
logger.info(f"[translate_routes][create_job] User: {current_user.username}")
try:
service = TranslateJobService(db, config_manager, current_user.username)
job = service.create_job(payload)
dict_ids = service.get_job_dictionary_ids(job.id)
return job_to_response(job, dict_ids)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))
# [/DEF:create_job:Function]
# [DEF:update_job:Function]
# @PURPOSE: Update an existing translation job.
# @PRE: User has translate.job.edit permission.
# @POST: Returns the updated translation job.
# @SIDE_EFFECT: Re-detects database_dialect if datasource changed.
@router.put("/jobs/{job_id}", response_model=TranslateJobResponse)
async def update_job(
job_id: str,
payload: TranslateJobUpdate,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EDIT")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Update a translation job."""
logger.info(f"[translate_routes][update_job] Job: {job_id}, User: {current_user.username}")
try:
service = TranslateJobService(db, config_manager, current_user.username)
job = service.update_job(job_id, payload)
dict_ids = service.get_job_dictionary_ids(job.id)
return job_to_response(job, dict_ids)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:update_job:Function]
# [DEF:delete_job:Function]
# @PURPOSE: Delete a translation job.
# @PRE: User has translate.job.delete permission.
# @POST: Job is deleted.
@router.delete("/jobs/{job_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_job(
job_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "DELETE")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Delete a translation job."""
logger.info(f"[translate_routes][delete_job] Job: {job_id}, User: {current_user.username}")
try:
service = TranslateJobService(db, config_manager, current_user.username)
service.delete_job(job_id)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:delete_job:Function]
# [DEF:duplicate_job:Function]
# @PURPOSE: Duplicate a translation job.
# @PRE: User has translate.job.create permission.
# @POST: Returns the duplicated job with status DRAFT.
@router.post("/jobs/{job_id}/duplicate", response_model=DuplicateJobResponse, status_code=status.HTTP_201_CREATED)
async def duplicate_job(
job_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "CREATE")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Duplicate a translation job."""
logger.info(f"[translate_routes][duplicate_job] Job: {job_id}, User: {current_user.username}")
try:
service = TranslateJobService(db, config_manager, current_user.username)
new_job = service.duplicate_job(job_id)
return DuplicateJobResponse(
id=new_job.id,
name=new_job.name,
message="Job duplicated successfully",
)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:duplicate_job:Function]
# [DEF:get_datasource_columns:Function]
# @PURPOSE: Get column metadata and database dialect for a Superset datasource.
# @PRE: User has translate.job.view permission.
# @POST: Returns column list with metadata and database_dialect.
# @SIDE_EFFECT: Queries Superset API for dataset detail and database info.
# @RATIONALE: Dialect detection from Superset connection cached on TranslationJob at save time.
# @REJECTED: Hardcoding dialect list per database — would drift from actual Superset config.
@router.get("/datasources")
async def list_datasources(
env_id: str = Query(..., description="Superset environment ID"),
search: Optional[str] = Query(None, description="Search by table name"),
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "VIEW")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""List all Superset datasets available for translation jobs."""
with belief_scope("list_datasources"):
try:
service = TranslateJobService(db, config_manager)
datasets = service.fetch_available_datasources(env_id, search)
return datasets
except Exception as e:
logger.error(f"[translate_routes][list_datasources] Error: {e}")
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(e))
@router.get("/datasources/{datasource_id}/columns", response_model=DatasourceColumnsResponse)
async def get_datasource_columns(
datasource_id: int,
env_id: str = Query(..., description="Superset environment ID"),
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "VIEW")),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Get column metadata and database dialect for a Superset datasource."""
logger.info(
f"[translate_routes][get_datasource_columns] "
f"Datasource: {datasource_id}, Env: {env_id}, User: {current_user.username}"
)
try:
result = fetch_datasource_columns_service(
datasource_id=datasource_id,
env_id=env_id,
config_manager=config_manager,
)
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
except Exception as e:
logger.error(f"[translate_routes][get_datasource_columns] Error: {e}")
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail=f"Failed to fetch datasource columns from Superset: {e}",
)
# [/DEF:get_datasource_columns:Function]
# [/DEF:TranslateJobRoutesModule:Module]

67
backend/src/api/routes/translate/_metrics_routes.py Normal file
View File

@@ -0,0 +1,67 @@
# [DEF:TranslateMetricsRoutesModule:Module]
# @COMPLEXITY: 2
# @PURPOSE: Translation Metrics endpoints.
# @LAYER: API
# @SEMANTICS: api, routes, translate, metrics
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session
from ....core.database import get_db
from ....core.logger import logger, belief_scope
from ....schemas.auth import User
from ....dependencies import get_current_user, has_permission
from ....plugins.translate.metrics import TranslationMetrics
from ._router import router
# ============================================================
# Metrics
# ============================================================
# [DEF:get_metrics:Function]
# @PURPOSE: Get translation metrics, optionally filtered by job.
# @PRE: User has translate.metrics.view permission.
# @POST: Returns metrics data.
@router.get("/metrics")
async def get_metrics(
job_id: Optional[str] = Query(None),
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.metrics", "VIEW")),
db: Session = Depends(get_db),
):
"""Get translation metrics."""
logger.info(f"[translate_routes][get_metrics] User: {current_user.username}")
try:
metrics_svc = TranslationMetrics(db)
if job_id:
return metrics_svc.get_job_metrics(job_id)
return metrics_svc.get_all_metrics()
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_metrics:Function]
# [DEF:get_job_metrics:Function]
# @PURPOSE: Get aggregated metrics for a specific job.
# @PRE: User has translate.metrics.view permission.
# @POST: Returns metrics dict.
@router.get("/jobs/{job_id}/metrics")
async def get_job_metrics(
job_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.metrics", "VIEW")),
db: Session = Depends(get_db),
):
"""Get aggregated metrics for a specific job."""
logger.info(f"[translate_routes][get_job_metrics] Job: {job_id}, User: {current_user.username}")
try:
metrics_svc = TranslationMetrics(db)
return metrics_svc.get_job_metrics(job_id)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_job_metrics:Function]
# [/DEF:TranslateMetricsRoutesModule:Module]

194
backend/src/api/routes/translate/_preview_routes.py Normal file
View File

@@ -0,0 +1,194 @@
# [DEF:TranslatePreviewRoutesModule:Module]
# @COMPLEXITY: 3
# @PURPOSE: Translation Preview session management routes.
# @LAYER: API
# @SEMANTICS: api, routes, translate, preview
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session
from ....core.database import get_db
from ....core.logger import logger, belief_scope
from ....schemas.auth import User
from ....dependencies import get_current_user, has_permission, get_config_manager
from ....core.config_manager import ConfigManager
from ....plugins.translate.preview import TranslationPreview
from ....schemas.translate import (
PreviewRequest,
PreviewRowUpdate,
PreviewAcceptResponse,
PreviewRow,
)
from ._router import router
# ============================================================
# Preview
# ============================================================
# [DEF:preview_translation:Function]
# @PURPOSE: Preview a translation before applying it.
# @PRE: User has translate.job.execute permission.
# @POST: Returns a preview session with records and cost estimation.
# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows.
@router.post("/jobs/{job_id}/preview", status_code=status.HTTP_201_CREATED)
async def preview_translation(
job_id: str,
payload: PreviewRequest = None,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EXECUTE")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Preview a translation before applying it."""
logger.info(f"[translate_routes][preview_translation] Job: {job_id}, User: {current_user.username}")
if payload is None:
payload = PreviewRequest()
try:
preview_service = TranslationPreview(db, config_manager, current_user.username)
result = preview_service.preview_rows(
job_id=job_id,
sample_size=payload.sample_size,
prompt_template=payload.prompt_template,
)
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
except Exception as e:
logger.error(f"[translate_routes][preview_translation] Error: {e}")
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Preview failed: {e}")
# [/DEF:preview_translation:Function]
# [DEF:update_preview_row:Function]
# @PURPOSE: Approve, edit, or reject a preview row.
# @PRE: User has translate.job.execute permission.
# @POST: Preview row status is updated.
@router.put("/jobs/{job_id}/preview/rows/{row_key}")
async def update_preview_row(
job_id: str,
row_key: str,
payload: PreviewRowUpdate,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EXECUTE")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Approve, edit, or reject a preview row."""
logger.info(f"[translate_routes][update_preview_row] Job: {job_id}, Row: {row_key}, Action: {payload.action}")
try:
preview_service = TranslationPreview(db, config_manager, current_user.username)
result = preview_service.update_preview_row(
job_id=job_id,
row_id=row_key,
action=payload.action,
translation=payload.translation,
feedback=payload.feedback,
)
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:update_preview_row:Function]
# [DEF:accept_preview_session:Function]
# @PURPOSE: Accept a preview session, marking it as the quality gate for full execution.
# @PRE: User has translate.job.execute permission. Job has an ACTIVE preview session.
# @POST: Preview session is marked as APPLIED; full execution can proceed.
@router.post("/jobs/{job_id}/preview/accept")
async def accept_preview_session(
job_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EXECUTE")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Accept a preview session, enabling full translation execution."""
logger.info(f"[translate_routes][accept_preview_session] Job: {job_id}, User: {current_user.username}")
try:
preview_service = TranslationPreview(db, config_manager, current_user.username)
result = preview_service.accept_preview_session(job_id=job_id)
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:accept_preview_session:Function]
# [DEF:apply_preview:Function]
# @PURPOSE: Apply a preview session (alias for accept when accepting at session level).
# @PRE: User has translate.job.execute permission.
# @POST: Preview is applied.
@router.post("/preview/{session_id}/apply")
async def apply_preview(
session_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EXECUTE")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Apply a preview session by session ID."""
logger.info(f"[translate_routes][apply_preview] Session: {session_id}, User: {current_user.username}")
# Find job_id from session
from ...models.translate import TranslationPreviewSession
session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first()
if not session:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Preview session '{session_id}' not found")
try:
preview_service = TranslationPreview(db, config_manager, current_user.username)
result = preview_service.accept_preview_session(job_id=session.job_id)
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:apply_preview:Function]
# ============================================================
# Preview Records
# ============================================================
# [DEF:get_preview_records:Function]
# @PURPOSE: Get records for a preview session.
# @PRE: User has translate.job.view permission.
# @POST: Returns list of preview records.
@router.get("/preview/{session_id}/records", response_model=List[PreviewRow])
async def get_preview_records(
session_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "VIEW")),
db: Session = Depends(get_db),
):
"""Get records for a preview session."""
logger.info(f"[translate_routes][get_preview_records] Session: {session_id}, User: {current_user.username}")
try:
from ...models.translate import TranslationPreviewSession, TranslationPreviewRecord
session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first()
if not session:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Preview session '{session_id}' not found")
records = (
db.query(TranslationPreviewRecord)
.filter(TranslationPreviewRecord.session_id == session_id)
.all()
)
return [
PreviewRow(
id=r.id,
source_sql=r.source_sql,
target_sql=r.target_sql,
source_object_type=r.source_object_type,
source_object_id=r.source_object_id,
source_object_name=r.source_object_name,
status=r.status,
feedback=r.feedback,
)
for r in records
]
except HTTPException:
raise
except Exception as e:
logger.error(f"[translate_routes][get_preview_records] Error: {e}")
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:get_preview_records:Function]
# [/DEF:TranslatePreviewRoutesModule:Module]

14
backend/src/api/routes/translate/_router.py Normal file
View File

@@ -0,0 +1,14 @@
# [DEF:TranslateRouterModule:Module]
# @COMPLEXITY: 1
# @PURPOSE: APIRouter instance for translate routes.
# @LAYER: API
# @SEMANTICS: api, routes, translate, router
from fastapi import APIRouter
# [DEF:translate_router:Global]
# @PURPOSE: APIRouter instance for all translate sub-routes.
router = APIRouter(prefix="/api/translate", tags=["translate"])
# [/DEF:translate_router:Global]
# [/DEF:TranslateRouterModule:Module]

206
backend/src/api/routes/translate/_run_list_routes.py Normal file
View File

@@ -0,0 +1,206 @@
# [DEF:TranslateRunListRoutesModule:Module]
# @COMPLEXITY: 3
# @PURPOSE: Translation Run listing, detail, and CSV download routes (cross-job).
# @LAYER: API
# @SEMANTICS: api, routes, translate, runs, list, detail
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session
from ....core.database import get_db
from ....core.logger import logger, belief_scope
from ....schemas.auth import User
from ....dependencies import get_current_user, has_permission, get_config_manager
from ....core.config_manager import ConfigManager
from ....plugins.translate.orchestrator import TranslationOrchestrator
from ....plugins.translate.events import TranslationEventLog
from ._router import router
# ============================================================
# List Runs (cross-job)
# ============================================================
# [DEF:list_runs:Function]
# @PURPOSE: List all runs with cross-job filtering and pagination.
# @PRE: User has translate.history.view permission.
# @POST: Returns paginated list of runs.
@router.get("/runs")
async def list_runs(
job_id: Optional[str] = Query(None),
run_status: Optional[str] = Query(None),
trigger_type: Optional[str] = Query(None),
created_by: Optional[str] = Query(None),
date_from: Optional[str] = Query(None),
date_to: Optional[str] = Query(None),
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.history", "VIEW")),
db: Session = Depends(get_db),
):
"""List all runs with cross-job filtering and pagination."""
logger.info(f"[translate_routes][list_runs] User: {current_user.username}")
try:
from ...models.translate import TranslationRun
query = db.query(TranslationRun)
if job_id:
query = query.filter(TranslationRun.job_id == job_id)
if run_status:
query = query.filter(TranslationRun.status == run_status)
if trigger_type:
query = query.filter(TranslationRun.trigger_type == trigger_type)
if created_by:
query = query.filter(TranslationRun.created_by == created_by)
if date_from:
try:
from datetime import datetime
dt_from = datetime.fromisoformat(date_from)
query = query.filter(TranslationRun.created_at >= dt_from)
except ValueError:
pass
if date_to:
try:
from datetime import datetime
dt_to = datetime.fromisoformat(date_to)
query = query.filter(TranslationRun.created_at <= dt_to)
except ValueError:
pass
total = query.count()
runs = (
query.order_by(TranslationRun.created_at.desc())
.offset((page - 1) * page_size)
.limit(page_size)
.all()
)
items = []
for r in runs:
items.append({
"id": r.id,
"job_id": r.job_id,
"status": r.status,
"trigger_type": r.trigger_type,
"started_at": r.started_at.isoformat() if r.started_at else None,
"completed_at": r.completed_at.isoformat() if r.completed_at else None,
"error_message": r.error_message,
"total_records": r.total_records or 0,
"successful_records": r.successful_records or 0,
"failed_records": r.failed_records or 0,
"skipped_records": r.skipped_records or 0,
"insert_status": r.insert_status,
"superset_execution_id": r.superset_execution_id,
"config_hash": r.config_hash,
"dict_snapshot_hash": r.dict_snapshot_hash,
"created_by": r.created_by,
"created_at": r.created_at.isoformat() if r.created_at else None,
})
return {"items": items, "total": total, "page": page, "page_size": page_size}
except Exception as e:
logger.error(f"[translate_routes][list_runs] Error: {e}")
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:list_runs:Function]
# ============================================================
# Run Detail
# ============================================================
# [DEF:get_run_detail:Function]
# @PURPOSE: Get detailed run info with config_snapshot, records, events.
# @PRE: User has translate.history.view permission.
# @POST: Returns run detail with records and events.
@router.get("/runs/{run_id}/detail")
async def get_run_detail(
run_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.history", "VIEW")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Get detailed run info with config_snapshot, records, events."""
logger.info(f"[translate_routes][get_run_detail] Run: {run_id}, User: {current_user.username}")
try:
orch = TranslationOrchestrator(db, config_manager, current_user.username)
status_info = orch.get_run_status(run_id)
records = orch.get_run_records(run_id, page=1, page_size=50)
events = TranslationEventLog(db).query_events(run_id=run_id)
event_summary = TranslationEventLog(db).get_run_event_summary(run_id)
# Get config snapshot from run
from ...models.translate import TranslationRun
run = db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
config_snapshot = run.config_snapshot if run else None
return {
**status_info,
"config_snapshot": config_snapshot,
"config_hash": run.config_hash if run else None,
"records": records,
"events": events,
"event_invariants": event_summary,
}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_run_detail:Function]
# ============================================================
# Download Skipped CSV
# ============================================================
# [DEF:download_skipped_csv:Function]
# @PURPOSE: Download a CSV of skipped translation records from a run.
# @PRE: User has translate.job.view permission.
# @POST: Returns CSV file of skipped records.
@router.get("/runs/{run_id}/skipped.csv")
async def download_skipped_csv(
run_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "VIEW")),
db: Session = Depends(get_db),
):
"""Download skipped translation records as CSV."""
logger.info(f"[translate_routes][download_skipped_csv] Run: {run_id}, User: {current_user.username}")
try:
from ...models.translate import TranslationRecord
from fastapi.responses import StreamingResponse
import csv
import io
records = (
db.query(TranslationRecord)
.filter(
TranslationRecord.run_id == run_id,
TranslationRecord.status == "SKIPPED",
)
.all()
)
output = io.StringIO()
writer = csv.writer(output)
writer.writerow(["id", "source_sql", "target_sql", "source_object_type",
"source_object_id", "source_object_name", "error_message"])
for rec in records:
writer.writerow([
rec.id, rec.source_sql, rec.target_sql, rec.source_object_type,
rec.source_object_id, rec.source_object_name, rec.error_message,
])
output.seek(0)
return StreamingResponse(
iter([output.getvalue()]),
media_type="text/csv",
headers={"Content-Disposition": f"attachment; filename=skipped_{run_id}.csv"},
)
except Exception as e:
logger.error(f"[translate_routes][download_skipped_csv] Error: {e}")
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:download_skipped_csv:Function]
# [/DEF:TranslateRunListRoutesModule:Module]

252
backend/src/api/routes/translate/_run_routes.py Normal file
View File

@@ -0,0 +1,252 @@
# [DEF:TranslateRunRoutesModule:Module]
# @COMPLEXITY: 3
# @PURPOSE: Translation Run execution, history, status, records and batches routes.
# @LAYER: API
# @SEMANTICS: api, routes, translate, runs
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session
from ....core.database import get_db
from ....core.logger import logger, belief_scope
from ....schemas.auth import User
from ....dependencies import get_current_user, has_permission, get_config_manager
from ....core.config_manager import ConfigManager
from ....plugins.translate.orchestrator import TranslationOrchestrator
from ....plugins.translate.events import TranslationEventLog
from ._router import router
from ._helpers import _run_to_response
# ============================================================
# Translation Run / Execute
# ============================================================
# [DEF:run_translation:Function]
# @PURPOSE: Execute a translation job (trigger a run).
# @PRE: User has translate.job.execute permission.
# @POST: Returns the created translation run.
@router.post("/jobs/{job_id}/run", status_code=status.HTTP_201_CREATED)
async def run_translation(
job_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EXECUTE")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Execute a translation job (trigger a run)."""
logger.info(f"[translate_routes][run_translation] Job: {job_id}, User: {current_user.username}")
try:
orch = TranslationOrchestrator(db, config_manager, current_user.username)
run = orch.start_run(job_id=job_id, is_scheduled=False)
# Execute asynchronously in background
import threading
threading.Thread(
target=orch.execute_run,
args=(run,),
daemon=True,
).start()
return _run_to_response(run)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
except Exception as e:
logger.error(f"[translate_routes][run_translation] Error: {e}")
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Run failed: {e}")
# [/DEF:run_translation:Function]
# [DEF:retry_run:Function]
# @PURPOSE: Retry failed batches in a translation run.
# @PRE: User has translate.job.execute permission.
# @POST: Returns the updated translation run.
@router.post("/runs/{run_id}/retry")
async def retry_run(
run_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EXECUTE")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Retry failed batches in a translation run."""
logger.info(f"[translate_routes][retry_run] Run: {run_id}, User: {current_user.username}")
try:
orch = TranslationOrchestrator(db, config_manager, current_user.username)
run = orch.retry_failed_batches(run_id)
return _run_to_response(run)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
except Exception as e:
logger.error(f"[translate_routes][retry_run] Error: {e}")
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry failed: {e}")
# [/DEF:retry_run:Function]
# [DEF:retry_insert:Function]
# @PURPOSE: Retry the SQL insert phase for a completed run.
# @PRE: User has translate.job.execute permission.
# @POST: Returns the updated run.
@router.post("/runs/{run_id}/retry-insert")
async def retry_insert(
run_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EXECUTE")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Retry the SQL insert phase for a completed run."""
logger.info(f"[translate_routes][retry_insert] Run: {run_id}, User: {current_user.username}")
try:
orch = TranslationOrchestrator(db, config_manager, current_user.username)
run = orch.retry_insert(run_id)
return _run_to_response(run)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
except Exception as e:
logger.error(f"[translate_routes][retry_insert] Error: {e}")
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry insert failed: {e}")
# [/DEF:retry_insert:Function]
# [DEF:cancel_run:Function]
# @PURPOSE: Cancel a running translation.
# @PRE: User has translate.job.execute permission.
# @POST: Run is cancelled.
@router.post("/runs/{run_id}/cancel")
async def cancel_run(
run_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EXECUTE")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Cancel a running translation."""
logger.info(f"[translate_routes][cancel_run] Run: {run_id}, User: {current_user.username}")
try:
orch = TranslationOrchestrator(db, config_manager, current_user.username)
run = orch.cancel_run(run_id)
return _run_to_response(run)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:cancel_run:Function]
# [DEF:get_run_history:Function]
# @PURPOSE: Get run history for a translation job.
# @PRE: User has translate.history.view permission.
# @POST: Returns list of runs.
@router.get("/jobs/{job_id}/runs")
async def get_run_history(
job_id: str,
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.history", "VIEW")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Get run history for a translation job."""
logger.info(f"[translate_routes][get_run_history] Job: {job_id}, User: {current_user.username}")
try:
orch = TranslationOrchestrator(db, config_manager, current_user.username)
total, runs = orch.get_run_history(job_id, page=page, page_size=page_size)
return {"items": runs, "total": total, "page": page, "page_size": page_size}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_run_history:Function]
# [DEF:get_run_status:Function]
# @PURPOSE: Get status and statistics for a translation run.
# @PRE: User has translate.history.view permission.
# @POST: Returns run details with statistics.
@router.get("/runs/{run_id}")
async def get_run_status(
run_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.history", "VIEW")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Get status and statistics for a translation run."""
logger.info(f"[translate_routes][get_run_status] Run: {run_id}, User: {current_user.username}")
try:
orch = TranslationOrchestrator(db, config_manager, current_user.username)
return orch.get_run_status(run_id)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_run_status:Function]
# [DEF:get_run_records:Function]
# @PURPOSE: Get paginated records for a translation run.
# @PRE: User has translate.history.view permission.
# @POST: Returns paginated records.
@router.get("/runs/{run_id}/records")
async def get_run_records(
run_id: str,
page: int = Query(1, ge=1),
page_size: int = Query(50, ge=1, le=500),
status: Optional[str] = Query(None),
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.history", "VIEW")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Get paginated records for a translation run."""
logger.info(f"[translate_routes][get_run_records] Run: {run_id}, User: {current_user.username}")
try:
orch = TranslationOrchestrator(db, config_manager, current_user.username)
return orch.get_run_records(run_id, page=page, page_size=page_size, status_filter=status)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_run_records:Function]
# ============================================================
# Batches
# ============================================================
# [DEF:get_batches:Function]
# @PURPOSE: Get batches for a translation run.
# @PRE: User has translate.job.view permission.
# @POST: Returns list of batches.
@router.get("/runs/{run_id}/batches")
async def get_batches(
run_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "VIEW")),
db: Session = Depends(get_db),
):
"""Get batches for a translation run."""
logger.info(f"[translate_routes][get_batches] Run: {run_id}, User: {current_user.username}")
try:
from ...models.translate import TranslationBatch
batches = (
db.query(TranslationBatch)
.filter(TranslationBatch.run_id == run_id)
.order_by(TranslationBatch.batch_index.asc())
.all()
)
return [
{
"id": b.id,
"run_id": b.run_id,
"batch_index": b.batch_index,
"status": b.status,
"total_records": b.total_records or 0,
"successful_records": b.successful_records or 0,
"failed_records": b.failed_records or 0,
"started_at": b.started_at.isoformat() if b.started_at else None,
"completed_at": b.completed_at.isoformat() if b.completed_at else None,
"created_at": b.created_at.isoformat() if b.created_at else None,
}
for b in batches
]
except Exception as e:
logger.error(f"[translate_routes][get_batches] Error: {e}")
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:get_batches:Function]
# [/DEF:TranslateRunRoutesModule:Module]

235
backend/src/api/routes/translate/_schedule_routes.py Normal file
View File

@@ -0,0 +1,235 @@
# [DEF:TranslateScheduleRoutesModule:Module]
# @COMPLEXITY: 3
# @PURPOSE: Translation Schedule management routes.
# @LAYER: API
# @SEMANTICS: api, routes, translate, schedule
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session
from ....core.database import get_db
from ....core.logger import logger, belief_scope
from ....schemas.auth import User
from ....dependencies import get_current_user, has_permission, get_config_manager
from ....core.config_manager import ConfigManager
from ....plugins.translate.scheduler import TranslationScheduler
from ....schemas.translate import ScheduleConfig, ScheduleResponse
from ._router import router
# ============================================================
# Schedule
# ============================================================
# [DEF:get_schedule:Function]
# @PURPOSE: Get the schedule for a translation job.
# @PRE: User has translate.schedule.view permission.
# @POST: Returns the schedule configuration.
@router.get("/jobs/{job_id}/schedule")
async def get_schedule(
job_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.schedule", "VIEW")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Get schedule for a translation job."""
logger.info(f"[translate_routes][get_schedule] Job: {job_id}, User: {current_user.username}")
try:
scheduler = TranslationScheduler(db, config_manager, current_user.username)
schedule = scheduler.get_schedule(job_id)
return {
"id": schedule.id,
"job_id": schedule.job_id,
"cron_expression": schedule.cron_expression,
"timezone": schedule.timezone,
"is_active": schedule.is_active,
"last_run_at": schedule.last_run_at.isoformat() if schedule.last_run_at else None,
"next_run_at": schedule.next_run_at.isoformat() if schedule.next_run_at else None,
"created_by": schedule.created_by,
"created_at": schedule.created_at.isoformat() if schedule.created_at else None,
"updated_at": schedule.updated_at.isoformat() if schedule.updated_at else None,
}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_schedule:Function]
# [DEF:set_schedule:Function]
# @PURPOSE: Set or update the schedule for a translation job.
# @PRE: User has translate.schedule.manage permission.
# @POST: Schedule is created or updated.
@router.put("/jobs/{job_id}/schedule")
async def set_schedule(
job_id: str,
payload: ScheduleConfig,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.schedule", "MANAGE")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Set or update schedule for a translation job."""
logger.info(f"[translate_routes][set_schedule] Job: {job_id}, User: {current_user.username}")
try:
scheduler = TranslationScheduler(db, config_manager, current_user.username)
# Check if schedule already exists
from ...models.translate import TranslationSchedule
existing = db.query(TranslationSchedule).filter(
TranslationSchedule.job_id == job_id
).first()
if existing:
schedule = scheduler.update_schedule(
job_id,
cron_expression=payload.cron_expression,
timezone=payload.timezone,
is_active=payload.is_active,
)
else:
schedule = scheduler.create_schedule(
job_id,
cron_expression=payload.cron_expression,
timezone=payload.timezone,
is_active=payload.is_active,
)
# Register with APScheduler via SchedulerService
from ...dependencies import get_scheduler_service
sched_svc = get_scheduler_service()
sched_svc.add_translation_job(
schedule_id=schedule.id,
job_id=job_id,
cron_expression=schedule.cron_expression,
timezone=schedule.timezone,
)
return {
"id": schedule.id,
"job_id": schedule.job_id,
"cron_expression": schedule.cron_expression,
"timezone": schedule.timezone,
"is_active": schedule.is_active,
"last_run_at": schedule.last_run_at.isoformat() if schedule.last_run_at else None,
"created_by": schedule.created_by,
"created_at": schedule.created_at.isoformat() if schedule.created_at else None,
"updated_at": schedule.updated_at.isoformat() if schedule.updated_at else None,
}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:set_schedule:Function]
# [DEF:enable_schedule:Function]
# @PURPOSE: Enable a schedule for a translation job.
# @PRE: User has translate.schedule.manage permission.
# @POST: Schedule is enabled.
@router.post("/jobs/{job_id}/schedule/enable")
async def enable_schedule(
job_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.schedule", "MANAGE")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Enable a schedule for a translation job."""
logger.info(f"[translate_routes][enable_schedule] Job: {job_id}, User: {current_user.username}")
try:
scheduler = TranslationScheduler(db, config_manager, current_user.username)
schedule = scheduler.set_schedule_active(job_id, is_active=True)
from ...dependencies import get_scheduler_service
sched_svc = get_scheduler_service()
sched_svc.add_translation_job(
schedule_id=schedule.id,
job_id=job_id,
cron_expression=schedule.cron_expression,
timezone=schedule.timezone,
)
return {"status": "enabled", "job_id": job_id}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:enable_schedule:Function]
# [DEF:disable_schedule:Function]
# @PURPOSE: Disable a schedule for a translation job.
# @PRE: User has translate.schedule.manage permission.
# @POST: Schedule is disabled.
@router.post("/jobs/{job_id}/schedule/disable")
async def disable_schedule(
job_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.schedule", "MANAGE")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Disable a schedule for a translation job."""
logger.info(f"[translate_routes][disable_schedule] Job: {job_id}, User: {current_user.username}")
try:
scheduler = TranslationScheduler(db, config_manager, current_user.username)
scheduler.set_schedule_active(job_id, is_active=False)
from ...dependencies import get_scheduler_service
sched_svc = get_scheduler_service()
sched_svc.remove_translation_job(schedule_id=job_id)
return {"status": "disabled", "job_id": job_id}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:disable_schedule:Function]
# [DEF:delete_schedule:Function]
# @PURPOSE: Delete the schedule for a translation job.
# @PRE: User has translate.schedule.manage permission.
# @POST: Schedule is removed.
@router.delete("/jobs/{job_id}/schedule", status_code=status.HTTP_204_NO_CONTENT)
async def delete_schedule(
job_id: str,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.schedule", "MANAGE")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Delete schedule for a translation job."""
logger.info(f"[translate_routes][delete_schedule] Job: {job_id}, User: {current_user.username}")
try:
scheduler = TranslationScheduler(db, config_manager, current_user.username)
scheduler.delete_schedule(job_id)
from ...dependencies import get_scheduler_service
sched_svc = get_scheduler_service()
sched_svc.remove_translation_job(schedule_id=job_id)
return None
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:delete_schedule:Function]
# [DEF:get_next_executions:Function]
# @PURPOSE: Preview next N executions for a job's schedule.
# @PRE: User has translate.schedule.view permission.
# @POST: Returns next execution times.
@router.get("/jobs/{job_id}/schedule/next-executions")
async def get_next_executions(
job_id: str,
n: int = Query(3, ge=1, le=10),
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.schedule", "VIEW")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Preview next N executions for a job's schedule."""
logger.info(f"[translate_routes][get_next_executions] Job: {job_id}, User: {current_user.username}")
try:
scheduler = TranslationScheduler(db, config_manager, current_user.username)
schedule = scheduler.get_schedule(job_id)
next_times = TranslationScheduler.get_next_executions(
schedule.cron_expression, schedule.timezone, n=n
)
return {
"job_id": job_id,
"cron_expression": schedule.cron_expression,
"timezone": schedule.timezone,
"next_executions": next_times,
}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_next_executions:Function]
# [/DEF:TranslateScheduleRoutesModule:Module]

2145
backend/src/core/superset_client.py
View File

File diff suppressed because it is too large Load Diff

8
backend/src/core/superset_client/__init__.py
View File

@@ -1,12 +1,15 @@
# [DEF:SupersetClientModule:Module]
#
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: superset, api, client, rest, http, dashboard, dataset, import, export
# @PURPOSE: Предоставляет высокоуровневый клиент для взаимодействия с Superset REST API, инкапсулируя логику запросов, обработку ошибок и пагинацию.
# @RELATION: DEPENDS_ON -> [ConfigModels]
# @RELATION: DEPENDS_ON -> [APIClient]
# @RELATION: DEPENDS_ON -> [SupersetAPIError]
# @RELATION: DEPENDS_ON -> [get_filename_from_headers]
# @RELATION: DEPENDS_ON -> [SupersetDatasetsPreviewFiltersMixin]
#
# @INVARIANT: All network operations must use the internal APIClient instance.
# @PUBLIC_API: SupersetClient
#
# @RATIONALE: Decomposed from monolithic superset_client.py (2145 lines) into
@@ -23,6 +26,7 @@ from ._dashboards_crud import SupersetDashboardsCrudMixin
from ._charts import SupersetChartsMixin
from ._datasets import SupersetDatasetsMixin
from ._datasets_preview import SupersetDatasetsPreviewMixin
from ._datasets_preview_filters import SupersetDatasetsPreviewFiltersMixin
from ._databases import SupersetDatabasesMixin
@@ -40,9 +44,11 @@ from ._databases import SupersetDatabasesMixin
# @RELATION: INHERITS -> [SupersetChartsMixin]
# @RELATION: INHERITS -> [SupersetDatasetsMixin]
# @RELATION: INHERITS -> [SupersetDatasetsPreviewMixin]
# @RELATION: INHERITS -> [SupersetDatasetsPreviewFiltersMixin]
# @RELATION: INHERITS -> [SupersetDatabasesMixin]
class SupersetClient(
SupersetDatabasesMixin,
SupersetDatasetsPreviewFiltersMixin,
SupersetDatasetsPreviewMixin,
SupersetDatasetsMixin,
SupersetChartsMixin,

2
backend/src/core/superset_client/_base.py
View File

@@ -1,5 +1,7 @@
# [DEF:SupersetClientBase:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: superset, api, client, base, pagination, auth, import, export
# @PURPOSE: Base class for SupersetClient providing initialization, authentication, pagination, and import/export helpers.
# @RELATION: DEPENDS_ON -> [ConfigModels]
# @RELATION: DEPENDS_ON -> [APIClient]

2
backend/src/core/superset_client/_charts.py
View File

@@ -1,5 +1,7 @@
# [DEF:SupersetChartsMixin:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: superset, charts, list, get, layout
# @PURPOSE: Chart domain mixin for SupersetClient — list, get, extract IDs from layout.
# @RELATION: DEPENDS_ON -> [SupersetClientBase]

2
backend/src/core/superset_client/_dashboards.py
View File

@@ -1,2 +0,0 @@
# This file has been decomposed into _dashboards_list.py, _dashboards_filters.py, _dashboards_crud.py
# See __init__.py for the composed SupersetClient class.

2
backend/src/core/superset_client/_dashboards_crud.py
View File

@@ -1,5 +1,7 @@
# [DEF:SupersetDashboardsCrudMixin:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: superset, dashboards, crud, detail, export, import, delete
# @PURPOSE: Dashboard CRUD mixin for SupersetClient — detail, export, import, delete.
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
# @RELATION: DEPENDS_ON -> [SupersetDashboardsFiltersMixin]

2
backend/src/core/superset_client/_dashboards_filters.py
View File

@@ -1,5 +1,7 @@
# [DEF:SupersetDashboardsFiltersMixin:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: superset, dashboards, filters, permalink, native-filters
# @PURPOSE: Dashboard native filter extraction mixin for SupersetClient.
# @RELATION: DEPENDS_ON -> [SupersetClientBase]

2
backend/src/core/superset_client/_dashboards_list.py
View File

@@ -1,5 +1,7 @@
# [DEF:SupersetDashboardsListMixin:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: superset, dashboards, list, pagination, summary
# @PURPOSE: Dashboard listing mixin for SupersetClient — paginated list, summary projection.
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
# @RELATION: DEPENDS_ON -> [SupersetUserProjectionMixin]

2
backend/src/core/superset_client/_databases.py
View File

@@ -1,5 +1,7 @@
# [DEF:SupersetDatabasesMixin:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: superset, databases, list, get, summary, uuid
# @PURPOSE: Database domain mixin for SupersetClient — list, get, summary, by_uuid.
# @RELATION: DEPENDS_ON -> [SupersetClientBase]

12
backend/src/core/superset_client/_datasets.py
View File

@@ -1,5 +1,7 @@
# [DEF:SupersetDatasetsMixin:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: superset, datasets, list, get, detail, update
# @PURPOSE: Dataset domain mixin for SupersetClient — list, get, detail, update.
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
@@ -147,17 +149,17 @@ class SupersetDatasetsMixin:
)
linked_dashboards = []
database_obj = dataset.get("database")
database_dict = database_obj if isinstance(database_obj, dict) else {}
sql = dataset.get("sql", "")
result = {
"id": dataset.get("id"),
"table_name": dataset.get("table_name"),
"schema": dataset.get("schema"),
"database": (
dataset.get("database", {}).get("database_name", "Unknown")
if isinstance(dataset.get("database"), dict)
else dataset.get("database_name") or "Unknown"
),
"database": database_dict,
"database_id": dataset.get("database_id", database_dict.get("id")),
"database_backend": database_dict.get("backend") or database_dict.get("engine") or "",
"description": dataset.get("description", ""),
"columns": column_info,
"column_count": len(column_info),

180
backend/src/core/superset_client/_datasets_preview.py
View File

@@ -1,13 +1,19 @@
# [DEF:SupersetDatasetsPreviewMixin:Module]
# @COMPLEXITY: 4
# @LAYER: Infra
# @SEMANTICS: superset, datasets, preview, sql, compilation, filters
# @PURPOSE: Dataset preview compilation mixin for SupersetClient — build query context, compile SQL.
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
# @RELATION: DEPENDS_ON -> [SupersetDatasetsMixin]
import json
from copy import deepcopy
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Dict, List, Optional
from ..logger import logger as app_logger, belief_scope
from ..utils.network import SupersetAPIError
# [DEF:SupersetDatasetsPreviewMixin:Class]
# @COMPLEXITY: 4
# @PURPOSE: Mixin providing dataset preview compilation and query context building.
class SupersetDatasetsPreviewMixin:
# [DEF:SupersetClientCompileDatasetPreview:Function]
@@ -221,177 +227,5 @@ class SupersetDatasetsPreviewMixin:
# [/DEF:SupersetClientBuildDatasetPreviewQueryContext:Function]
# [DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]
def _normalize_effective_filters_for_query_context(
self,
effective_filters: List[Dict[str, Any]],
) -> Dict[str, Any]:
with belief_scope(
"SupersetClient._normalize_effective_filters_for_query_context"
):
normalized_filters: List[Dict[str, Any]] = []
merged_extra_form_data: Dict[str, Any] = {}
diagnostics: List[Dict[str, Any]] = []
for item in effective_filters:
if not isinstance(item, dict):
continue
display_name = str(
item.get("display_name")
or item.get("filter_name")
or item.get("variable_name")
or "unresolved_filter"
).strip()
value = item.get("effective_value")
normalized_payload = item.get("normalized_filter_payload")
preserved_clauses: List[Dict[str, Any]] = []
preserved_extra_form_data: Dict[str, Any] = {}
used_preserved_clauses = False
if isinstance(normalized_payload, dict):
raw_clauses = normalized_payload.get("filter_clauses")
if isinstance(raw_clauses, list):
preserved_clauses = [
deepcopy(clause)
for clause in raw_clauses
if isinstance(clause, dict)
]
raw_extra_form_data = normalized_payload.get("extra_form_data")
if isinstance(raw_extra_form_data, dict):
preserved_extra_form_data = deepcopy(raw_extra_form_data)
if isinstance(preserved_extra_form_data, dict):
for key, extra_value in preserved_extra_form_data.items():
if key == "filters":
continue
merged_extra_form_data[key] = deepcopy(extra_value)
outgoing_clauses: List[Dict[str, Any]] = []
if preserved_clauses:
for clause in preserved_clauses:
clause_copy = deepcopy(clause)
if "val" not in clause_copy and value is not None:
clause_copy["val"] = deepcopy(value)
outgoing_clauses.append(clause_copy)
used_preserved_clauses = True
elif preserved_extra_form_data:
outgoing_clauses = []
else:
column = str(
item.get("variable_name") or item.get("filter_name") or ""
).strip()
if column and value is not None:
operator = "IN" if isinstance(value, list) else "=="
outgoing_clauses.append(
{"col": column, "op": operator, "val": value}
)
normalized_filters.extend(outgoing_clauses)
diagnostics.append(
{
"filter_name": display_name,
"value_origin": (
normalized_payload.get("value_origin")
if isinstance(normalized_payload, dict)
else None
),
"used_preserved_clauses": used_preserved_clauses,
"outgoing_clauses": deepcopy(outgoing_clauses),
}
)
app_logger.reason(
"Normalized effective preview filter for Superset query context",
extra={
"filter_name": display_name,
"used_preserved_clauses": used_preserved_clauses,
"outgoing_clauses": outgoing_clauses,
"value_origin": (
normalized_payload.get("value_origin")
if isinstance(normalized_payload, dict)
else "heuristic_reconstruction"
),
},
)
return {
"filters": normalized_filters,
"extra_form_data": merged_extra_form_data,
"diagnostics": diagnostics,
}
# [/DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]
# [DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function]
def _extract_compiled_sql_from_preview_response(
self, response: Any
) -> Dict[str, Any]:
with belief_scope("SupersetClient._extract_compiled_sql_from_preview_response"):
if not isinstance(response, dict):
raise SupersetAPIError(
"Superset preview response was not a JSON object"
)
response_diagnostics: List[Dict[str, Any]] = []
result_payload = response.get("result")
if isinstance(result_payload, list):
for index, item in enumerate(result_payload):
if not isinstance(item, dict):
continue
compiled_sql = str(
item.get("query")
or item.get("sql")
or item.get("compiled_sql")
or ""
).strip()
response_diagnostics.append(
{
"index": index,
"status": item.get("status"),
"applied_filters": item.get("applied_filters"),
"rejected_filters": item.get("rejected_filters"),
"has_query": bool(compiled_sql),
"source": "result_list",
}
)
if compiled_sql:
return {
"compiled_sql": compiled_sql,
"raw_response": response,
"response_diagnostics": response_diagnostics,
}
top_level_candidates: List[Tuple[str, Any]] = [
("query", response.get("query")),
("sql", response.get("sql")),
("compiled_sql", response.get("compiled_sql")),
]
if isinstance(result_payload, dict):
top_level_candidates.extend(
[
("result.query", result_payload.get("query")),
("result.sql", result_payload.get("sql")),
("result.compiled_sql", result_payload.get("compiled_sql")),
]
)
for source, candidate in top_level_candidates:
compiled_sql = str(candidate or "").strip()
response_diagnostics.append(
{"source": source, "has_query": bool(compiled_sql)}
)
if compiled_sql:
return {
"compiled_sql": compiled_sql,
"raw_response": response,
"response_diagnostics": response_diagnostics,
}
raise SupersetAPIError(
"Superset preview response did not expose compiled SQL "
f"(diagnostics={response_diagnostics!r})"
)
# [/DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function]
# [/DEF:SupersetDatasetsPreviewMixin:Class]
# [/DEF:SupersetDatasetsPreviewMixin:Module]

201
backend/src/core/superset_client/_datasets_preview_filters.py Normal file
View File

@@ -0,0 +1,201 @@
# [DEF:SupersetDatasetsPreviewFiltersMixin:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: superset, datasets, preview, filters, normalization, sql, extraction
# @PURPOSE: Filter normalization and SQL extraction helpers for dataset preview compilation.
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
# @RELATION: DEPENDS_ON -> [SupersetDatasetsPreviewMixin]
from copy import deepcopy
from typing import Any, Dict, List, Optional, Tuple
from ..logger import logger as app_logger, belief_scope
from ..utils.network import SupersetAPIError
# [DEF:SupersetDatasetsPreviewFiltersMixin:Class]
# @COMPLEXITY: 3
# @PURPOSE: Mixin providing filter normalization and compiled-SQL extraction for dataset preview operations.
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
# @RELATION: DEPENDS_ON -> [SupersetDatasetsPreviewMixin]
class SupersetDatasetsPreviewFiltersMixin:
# [DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]
# @COMPLEXITY: 3
# @PURPOSE: Convert execution mappings into Superset chart-data filter objects.
def _normalize_effective_filters_for_query_context(
self,
effective_filters: List[Dict[str, Any]],
) -> Dict[str, Any]:
with belief_scope(
"SupersetClient._normalize_effective_filters_for_query_context"
):
normalized_filters: List[Dict[str, Any]] = []
merged_extra_form_data: Dict[str, Any] = {}
diagnostics: List[Dict[str, Any]] = []
for item in effective_filters:
if not isinstance(item, dict):
continue
display_name = str(
item.get("display_name")
or item.get("filter_name")
or item.get("variable_name")
or "unresolved_filter"
).strip()
value = item.get("effective_value")
normalized_payload = item.get("normalized_filter_payload")
preserved_clauses: List[Dict[str, Any]] = []
preserved_extra_form_data: Dict[str, Any] = {}
used_preserved_clauses = False
if isinstance(normalized_payload, dict):
raw_clauses = normalized_payload.get("filter_clauses")
if isinstance(raw_clauses, list):
preserved_clauses = [
deepcopy(clause)
for clause in raw_clauses
if isinstance(clause, dict)
]
raw_extra_form_data = normalized_payload.get("extra_form_data")
if isinstance(raw_extra_form_data, dict):
preserved_extra_form_data = deepcopy(raw_extra_form_data)
if isinstance(preserved_extra_form_data, dict):
for key, extra_value in preserved_extra_form_data.items():
if key == "filters":
continue
merged_extra_form_data[key] = deepcopy(extra_value)
outgoing_clauses: List[Dict[str, Any]] = []
if preserved_clauses:
for clause in preserved_clauses:
clause_copy = deepcopy(clause)
if "val" not in clause_copy and value is not None:
clause_copy["val"] = deepcopy(value)
outgoing_clauses.append(clause_copy)
used_preserved_clauses = True
elif preserved_extra_form_data:
outgoing_clauses = []
else:
column = str(
item.get("variable_name") or item.get("filter_name") or ""
).strip()
if column and value is not None:
operator = "IN" if isinstance(value, list) else "=="
outgoing_clauses.append(
{"col": column, "op": operator, "val": value}
)
normalized_filters.extend(outgoing_clauses)
diagnostics.append(
{
"filter_name": display_name,
"value_origin": (
normalized_payload.get("value_origin")
if isinstance(normalized_payload, dict)
else None
),
"used_preserved_clauses": used_preserved_clauses,
"outgoing_clauses": deepcopy(outgoing_clauses),
}
)
app_logger.reason(
"Normalized effective preview filter for Superset query context",
extra={
"filter_name": display_name,
"used_preserved_clauses": used_preserved_clauses,
"outgoing_clauses": outgoing_clauses,
"value_origin": (
normalized_payload.get("value_origin")
if isinstance(normalized_payload, dict)
else "heuristic_reconstruction"
),
},
)
return {
"filters": normalized_filters,
"extra_form_data": merged_extra_form_data,
"diagnostics": diagnostics,
}
# [/DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]
# [DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function]
# @COMPLEXITY: 3
# @PURPOSE: Normalize compiled SQL from either chart-data or legacy form_data preview responses.
def _extract_compiled_sql_from_preview_response(
self, response: Any
) -> Dict[str, Any]:
with belief_scope("SupersetClient._extract_compiled_sql_from_preview_response"):
if not isinstance(response, dict):
raise SupersetAPIError(
"Superset preview response was not a JSON object"
)
response_diagnostics: List[Dict[str, Any]] = []
result_payload = response.get("result")
if isinstance(result_payload, list):
for index, item in enumerate(result_payload):
if not isinstance(item, dict):
continue
compiled_sql = str(
item.get("query")
or item.get("sql")
or item.get("compiled_sql")
or ""
).strip()
response_diagnostics.append(
{
"index": index,
"status": item.get("status"),
"applied_filters": item.get("applied_filters"),
"rejected_filters": item.get("rejected_filters"),
"has_query": bool(compiled_sql),
"source": "result_list",
}
)
if compiled_sql:
return {
"compiled_sql": compiled_sql,
"raw_response": response,
"response_diagnostics": response_diagnostics,
}
top_level_candidates: List[Tuple[str, Any]] = [
("query", response.get("query")),
("sql", response.get("sql")),
("compiled_sql", response.get("compiled_sql")),
]
if isinstance(result_payload, dict):
top_level_candidates.extend(
[
("result.query", result_payload.get("query")),
("result.sql", result_payload.get("sql")),
("result.compiled_sql", result_payload.get("compiled_sql")),
]
)
for source, candidate in top_level_candidates:
compiled_sql = str(candidate or "").strip()
response_diagnostics.append(
{"source": source, "has_query": bool(compiled_sql)}
)
if compiled_sql:
return {
"compiled_sql": compiled_sql,
"raw_response": response,
"response_diagnostics": response_diagnostics,
}
raise SupersetAPIError(
"Superset preview response did not expose compiled SQL "
f"(diagnostics={response_diagnostics!r})"
)
# [/DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function]
# [/DEF:SupersetDatasetsPreviewFiltersMixin:Class]
# [/DEF:SupersetDatasetsPreviewFiltersMixin:Module]

2
backend/src/core/superset_client/_user_projection.py
View File

@@ -1,5 +1,7 @@
# [DEF:SupersetUserProjection:Module]
# @COMPLEXITY: 2
# @LAYER: Infra
# @SEMANTICS: superset, users, owners, projection, normalization
# @PURPOSE: User/owner payload normalization helpers for Superset client responses.
# @RELATION: DEPENDS_ON -> [SupersetClientBase]

1397
backend/src/core/utils/superset_context_extractor.py
View File

File diff suppressed because it is too large Load Diff

66
backend/src/core/utils/superset_context_extractor/__init__.py Normal file
View File

@@ -0,0 +1,66 @@
# [DEF:SupersetContextExtractorPackage:Module]
# @COMPLEXITY: 4
# @LAYER: Infra
# @SEMANTICS: dataset_review, superset, link_parsing, context_recovery, partial_recovery
# @PURPOSE: Recover dataset and dashboard context from Superset links while preserving explicit partial-recovery markers.
# @RELATION: DEPENDS_ON -> [ImportedFilter]
# @RELATION: DEPENDS_ON -> [TemplateVariable]
# @RELATION: DEPENDS_ON -> [SupersetClient]
# @PRE: Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource.
# @POST: Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary.
# @SIDE_EFFECT: Performs upstream Superset API reads.
# @INVARIANT: Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context.
#
# @RATIONALE: Decomposed from monolithic superset_context_extractor.py (1397 lines) into
# domain-scoped mixins to satisfy INV_7 (module < 400 lines). The composed class
# preserves the original public API surface — all consumers continue to import
# from `src.core.utils.superset_context_extractor` without changes.
# @REJECTED: Keeping a single 1397-line file — violates fractal limit INV_7.
# [/DEF:SupersetContextExtractorPackage:Module]
from ._base import SupersetContextExtractorBase, SupersetParsedContext
from ._parsing import SupersetContextParsingMixin
from ._recovery import SupersetContextRecoveryMixin
from ._templates import SupersetContextTemplatesMixin
from ._filters import SupersetContextFiltersExtractMixin
from ._pii import mask_pii_value, sanitize_imported_filter_for_assistant, _mask_sensitive_text
# [DEF:SupersetContextExtractor:Class]
# @COMPLEXITY: 4
# @PURPOSE: Parse supported Superset URLs and recover canonical dataset/dashboard references for review-session intake.
# @RELATION: DEPENDS_ON -> [Environment]
# @RELATION: INHERITS -> [SupersetContextExtractorBase]
# @RELATION: INHERITS -> [SupersetContextParsingMixin]
# @RELATION: INHERITS -> [SupersetContextRecoveryMixin]
# @RELATION: INHERITS -> [SupersetContextTemplatesMixin]
# @RELATION: INHERITS -> [SupersetContextFiltersExtractMixin]
# @PRE: constructor receives a configured environment with a usable Superset base URL.
# @POST: extractor instance is ready to parse links against one Superset environment.
# @SIDE_EFFECT: downstream parse operations may call Superset APIs through SupersetClient.
class SupersetContextExtractor(
SupersetContextFiltersExtractMixin,
SupersetContextRecoveryMixin,
SupersetContextTemplatesMixin,
SupersetContextParsingMixin,
SupersetContextExtractorBase,
):
"""Composed Superset context extractor.
MRO: FiltersExtractMixin -> RecoveryMixin -> TemplatesMixin -> ParsingMixin -> Base.
All consumers continue to use:
from src.core.utils.superset_context_extractor import SupersetContextExtractor
"""
pass
# [/DEF:SupersetContextExtractor:Class]
__all__ = [
"SupersetContextExtractor",
"SupersetParsedContext",
"mask_pii_value",
"sanitize_imported_filter_for_assistant",
"_mask_sensitive_text",
]

231
backend/src/core/utils/superset_context_extractor/_base.py Normal file
View File

@@ -0,0 +1,231 @@
# [DEF:SupersetContextExtractorBase:Module]
# @COMPLEXITY: 4
# @LAYER: Infra
# @SEMANTICS: dataset_review, superset, link_parsing, context_recovery, partial_recovery
# @PURPOSE: Base class and helpers for recovering dataset/dashboard context from Superset links with explicit partial-recovery markers.
# @RELATION: DEPENDS_ON -> [ImportedFilter]
# @RELATION: DEPENDS_ON -> [TemplateVariable]
# @RELATION: CALLS -> [SupersetClient]
# @PRE: Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource.
# @POST: Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary.
# @SIDE_EFFECT: Performs upstream Superset API reads.
# @INVARIANT: Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context.
# [/DEF:SupersetContextExtractorBase:Module]
# [DEF:_base_imports:Block]
from __future__ import annotations
import json
import re
from copy import deepcopy
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Set, cast
from urllib.parse import parse_qs, unquote, urlparse
from ...config_models import Environment
from ...logger import belief_scope, logger
from ...superset_client import SupersetClient
# [/DEF:_base_imports:Block]
logger = cast(Any, logger)
# [DEF:SupersetParsedContext:Class]
# @COMPLEXITY: 2
# @PURPOSE: Normalized output of Superset link parsing for session intake and recovery.
@dataclass
class SupersetParsedContext:
source_url: str
dataset_ref: str
dataset_id: Optional[int] = None
dashboard_id: Optional[int] = None
chart_id: Optional[int] = None
resource_type: str = "unknown"
query_state: Dict[str, Any] = field(default_factory=dict)
imported_filters: List[Dict[str, Any]] = field(default_factory=list)
unresolved_references: List[str] = field(default_factory=list)
partial_recovery: bool = False
dataset_payload: Optional[Dict[str, Any]] = None
# [/DEF:SupersetParsedContext:Class]
# [DEF:SupersetContextExtractorBase:Class]
# @COMPLEXITY: 4
# @PURPOSE: Bind extractor to one Superset environment and client instance; provide shared URL-parsing helpers.
# @RELATION: DEPENDS_ON -> [Environment]
# @PRE: constructor receives a configured environment with a usable Superset base URL.
# @POST: extractor instance is ready to parse links against one Superset environment.
# @SIDE_EFFECT: downstream parse operations may call Superset APIs through SupersetClient.
class SupersetContextExtractorBase:
# [DEF:SupersetContextExtractorBase.__init__:Function]
# @COMPLEXITY: 2
# @PURPOSE: Bind extractor to one Superset environment and client instance.
def __init__(
self, environment: Environment, client: Optional[SupersetClient] = None
) -> None:
self.environment = environment
self.client = client or SupersetClient(environment)
# [/DEF:SupersetContextExtractorBase.__init__:Function]
# [DEF:SupersetContextExtractorBase.build_recovery_summary:Function]
# @COMPLEXITY: 2
# @PURPOSE: Summarize recovered, partial, and unresolved context for session state and UX.
def build_recovery_summary(
self, parsed_context: SupersetParsedContext
) -> Dict[str, Any]:
return {
"dataset_ref": parsed_context.dataset_ref,
"dataset_id": parsed_context.dataset_id,
"dashboard_id": parsed_context.dashboard_id,
"chart_id": parsed_context.chart_id,
"partial_recovery": parsed_context.partial_recovery,
"unresolved_references": list(parsed_context.unresolved_references),
"imported_filter_count": len(parsed_context.imported_filters),
}
# [/DEF:SupersetContextExtractorBase.build_recovery_summary:Function]
# [DEF:SupersetContextExtractorBase._extract_numeric_identifier:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a numeric identifier from a REST-like Superset URL path.
def _extract_numeric_identifier(
self, path_parts: List[str], resource_name: str
) -> Optional[int]:
if resource_name not in path_parts:
return None
try:
resource_index = path_parts.index(resource_name)
except ValueError:
return None
if resource_index + 1 >= len(path_parts):
return None
candidate = str(path_parts[resource_index + 1]).strip()
if not candidate.isdigit():
return None
return int(candidate)
# [/DEF:SupersetContextExtractorBase._extract_numeric_identifier:Function]
# [DEF:SupersetContextExtractorBase._extract_dashboard_reference:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a dashboard id-or-slug reference from a Superset URL path.
def _extract_dashboard_reference(self, path_parts: List[str]) -> Optional[str]:
if "dashboard" not in path_parts:
return None
try:
resource_index = path_parts.index("dashboard")
except ValueError:
return None
if resource_index + 1 >= len(path_parts):
return None
candidate = str(path_parts[resource_index + 1]).strip()
if not candidate or candidate == "p":
return None
return candidate
# [/DEF:SupersetContextExtractorBase._extract_dashboard_reference:Function]
# [DEF:SupersetContextExtractorBase._extract_dashboard_permalink_key:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a dashboard permalink key from a Superset URL path.
def _extract_dashboard_permalink_key(self, path_parts: List[str]) -> Optional[str]:
if "dashboard" not in path_parts:
return None
try:
resource_index = path_parts.index("dashboard")
except ValueError:
return None
if resource_index + 2 >= len(path_parts):
return None
permalink_marker = str(path_parts[resource_index + 1]).strip()
permalink_key = str(path_parts[resource_index + 2]).strip()
if permalink_marker != "p" or not permalink_key:
return None
return permalink_key
# [/DEF:SupersetContextExtractorBase._extract_dashboard_permalink_key:Function]
# [DEF:SupersetContextExtractorBase._extract_dashboard_id_from_state:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a dashboard identifier from returned permalink state when present.
def _extract_dashboard_id_from_state(self, state: Dict[str, Any]) -> Optional[int]:
return self._search_nested_numeric_key(
payload=state,
candidate_keys={"dashboardId", "dashboard_id", "dashboard_id_value"},
)
# [/DEF:SupersetContextExtractorBase._extract_dashboard_id_from_state:Function]
# [DEF:SupersetContextExtractorBase._extract_chart_id_from_state:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a chart identifier from returned permalink state when dashboard id is absent.
def _extract_chart_id_from_state(self, state: Dict[str, Any]) -> Optional[int]:
return self._search_nested_numeric_key(
payload=state,
candidate_keys={"slice_id", "sliceId", "chartId", "chart_id"},
)
# [/DEF:SupersetContextExtractorBase._extract_chart_id_from_state:Function]
# [DEF:SupersetContextExtractorBase._search_nested_numeric_key:Function]
# @COMPLEXITY: 3
# @PURPOSE: Recursively search nested dict/list payloads for the first numeric value under a candidate key set.
# @RELATION: DEPENDS_ON -> [SupersetContextExtractorBase]
def _search_nested_numeric_key(
self, payload: Any, candidate_keys: Set[str]
) -> Optional[int]:
if isinstance(payload, dict):
for key, value in payload.items():
if key in candidate_keys:
try:
if value is not None:
return int(value)
except (TypeError, ValueError):
pass
found = self._search_nested_numeric_key(value, candidate_keys)
if found is not None:
return found
elif isinstance(payload, list):
for item in payload:
found = self._search_nested_numeric_key(item, candidate_keys)
if found is not None:
return found
return None
# [/DEF:SupersetContextExtractorBase._search_nested_numeric_key:Function]
# [DEF:SupersetContextExtractorBase._decode_query_state:Function]
# @COMPLEXITY: 2
# @PURPOSE: Decode query-string structures used by Superset URL state transport.
def _decode_query_state(self, query_params: Dict[str, List[str]]) -> Dict[str, Any]:
query_state: Dict[str, Any] = {}
for key, values in query_params.items():
if not values:
continue
raw_value = values[-1]
decoded_value = unquote(raw_value)
if key in {"native_filters", "form_data", "q"}:
try:
query_state[key] = json.loads(decoded_value)
continue
except Exception:
logger.explore(
"Failed to decode structured Superset query state; preserving raw value",
extra={"key": key},
)
query_state[key] = decoded_value
return query_state
# [/DEF:SupersetContextExtractorBase._decode_query_state:Function]
# [/DEF:SupersetContextExtractorBase:Class]

261
backend/src/core/utils/superset_context_extractor/_filters.py Normal file
View File

@@ -0,0 +1,261 @@
# [DEF:SupersetContextFiltersExtractMixin:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: superset, filters, extraction, query_state, dataMask, native_filters
# @PURPOSE: Extract normalized imported filters from decoded Superset query state (native_filters, dataMask, native_filter_state, form_data).
# @RELATION: DEPENDS_ON -> [SupersetContextExtractorBase]
# [/DEF:SupersetContextFiltersExtractMixin:Module]
# [DEF:_filters_imports:Block]
from __future__ import annotations
from copy import deepcopy
from typing import Any, Dict, List, cast
from ...logger import logger as app_logger
app_logger = cast(Any, app_logger)
# [/DEF:_filters_imports:Block]
# [DEF:SupersetContextFiltersExtractMixin:Class]
# @COMPLEXITY: 3
# @PURPOSE: Mixin providing query-state filter extraction for the composed SupersetContextExtractor.
class SupersetContextFiltersExtractMixin:
# [DEF:SupersetContextFiltersExtractMixin._extract_imported_filters:Function]
# @COMPLEXITY: 2
# @PURPOSE: Normalize imported filters from decoded query state without fabricating missing values.
def _extract_imported_filters(
self, query_state: Dict[str, Any]
) -> List[Dict[str, Any]]:
imported_filters: List[Dict[str, Any]] = []
native_filters_payload = query_state.get("native_filters")
if isinstance(native_filters_payload, list):
for index, item in enumerate(native_filters_payload):
if not isinstance(item, dict):
continue
filter_name = (
item.get("filter_name")
or item.get("column")
or item.get("name")
or f"native_filter_{index}"
)
direct_clause = None
if item.get("column") and ("value" in item or "val" in item):
direct_clause = {
"col": item.get("column"),
"op": item.get("op")
or (
"IN"
if isinstance(item.get("value"), list)
else "=="
),
"val": item.get("val", item.get("value")),
}
imported_filters.append(
{
"filter_name": str(filter_name),
"raw_value": item.get("value"),
"display_name": item.get("label") or item.get("name"),
"normalized_value": {
"filter_clauses": [direct_clause]
if isinstance(direct_clause, dict)
else [],
"extra_form_data": {},
"value_origin": "native_filters",
},
"source": "superset_url",
"recovery_status": "recovered"
if item.get("value") is not None
else "partial",
"requires_confirmation": item.get("value") is None,
"notes": "Recovered from Superset native filter URL state",
}
)
# Extract filters from permalink dataMask
dashboard_data_mask = query_state.get("dataMask")
if isinstance(dashboard_data_mask, dict):
for filter_key, item in dashboard_data_mask.items():
if not isinstance(item, dict):
continue
filter_state = item.get("filterState")
extra_form_data = item.get("extraFormData")
display_name = None
raw_value = None
normalized_value = {
"filter_clauses": [],
"extra_form_data": deepcopy(extra_form_data)
if isinstance(extra_form_data, dict)
else {},
"value_origin": "unresolved",
}
if isinstance(filter_state, dict):
display_name = filter_state.get("label")
raw_value = filter_state.get("value") or filter_state.get(
"values"
)
if raw_value is not None:
normalized_value["value_origin"] = "filter_state"
if isinstance(extra_form_data, dict):
extra_filters = extra_form_data.get("filters")
if isinstance(extra_filters, list):
normalized_value["filter_clauses"] = [
deepcopy(extra_filter)
for extra_filter in extra_filters
if isinstance(extra_filter, dict)
]
if raw_value is None and normalized_value["filter_clauses"]:
first_filter = normalized_value["filter_clauses"][0]
raw_value = first_filter.get("val")
if raw_value is None:
raw_value = first_filter.get("value")
if raw_value is not None:
normalized_value["value_origin"] = (
"extra_form_data.filters"
)
if raw_value is None and isinstance(extra_form_data, dict):
for field in [
"time_range",
"time_grain_sqla",
"time_column",
"granularity",
]:
if field in extra_form_data:
raw_value = extra_form_data[field]
normalized_value["value_origin"] = (
f"extra_form_data.{field}"
)
break
imported_filters.append(
{
"filter_name": str(item.get("id") or filter_key),
"raw_value": raw_value,
"display_name": display_name,
"normalized_value": normalized_value,
"source": "superset_permalink",
"recovery_status": "recovered"
if raw_value is not None
else "partial",
"requires_confirmation": raw_value is None,
"notes": "Recovered from Superset dashboard permalink state",
}
)
# Extract filters from native_filter_state (fetched via native_filters_key)
native_filter_state = query_state.get("native_filter_state")
if isinstance(native_filter_state, dict):
for filter_key, item in native_filter_state.items():
if not isinstance(item, dict):
continue
filter_id = item.get("id") or filter_key
filter_state = item.get("filterState")
extra_form_data = item.get("extraFormData")
display_name = None
raw_value = None
normalized_value = {
"filter_clauses": [],
"extra_form_data": deepcopy(extra_form_data)
if isinstance(extra_form_data, dict)
else {},
"value_origin": "unresolved",
}
if isinstance(filter_state, dict):
display_name = filter_state.get("label")
raw_value = filter_state.get("value") or filter_state.get(
"values"
)
if raw_value is not None:
normalized_value["value_origin"] = "filter_state"
if isinstance(extra_form_data, dict):
extra_filters = extra_form_data.get("filters")
if isinstance(extra_filters, list):
normalized_value["filter_clauses"] = [
deepcopy(extra_filter)
for extra_filter in extra_filters
if isinstance(extra_filter, dict)
]
if raw_value is None and normalized_value["filter_clauses"]:
first_filter = normalized_value["filter_clauses"][0]
raw_value = first_filter.get("val")
if raw_value is None:
raw_value = first_filter.get("value")
if raw_value is not None:
normalized_value["value_origin"] = (
"extra_form_data.filters"
)
if raw_value is None and isinstance(extra_form_data, dict):
for field in [
"time_range",
"time_grain_sqla",
"time_column",
"granularity",
]:
if field in extra_form_data:
raw_value = extra_form_data[field]
normalized_value["value_origin"] = (
f"extra_form_data.{field}"
)
break
imported_filters.append(
{
"filter_name": str(filter_id),
"raw_value": raw_value,
"display_name": display_name,
"normalized_value": normalized_value,
"source": "superset_native_filters_key",
"recovery_status": "recovered"
if raw_value is not None
else "partial",
"requires_confirmation": raw_value is None,
"notes": "Recovered from Superset native_filters_key state",
}
)
form_data_payload = query_state.get("form_data")
if isinstance(form_data_payload, dict):
extra_filters = form_data_payload.get("extra_filters") or []
for index, item in enumerate(extra_filters):
if not isinstance(item, dict):
continue
filter_name = (
item.get("col")
or item.get("column")
or f"extra_filter_{index}"
)
imported_filters.append(
{
"filter_name": str(filter_name),
"raw_value": item.get("val"),
"display_name": item.get("label"),
"normalized_value": {
"filter_clauses": [deepcopy(item)],
"extra_form_data": {},
"value_origin": "form_data.extra_filters",
},
"source": "superset_url",
"recovery_status": "recovered"
if item.get("val") is not None
else "partial",
"requires_confirmation": item.get("val") is None,
"notes": "Recovered from Superset form_data extra_filters",
}
)
return imported_filters
# [/DEF:SupersetContextFiltersExtractMixin._extract_imported_filters:Function]
# [/DEF:SupersetContextFiltersExtractMixin:Class]

380
backend/src/core/utils/superset_context_extractor/_parsing.py Normal file
View File

@@ -0,0 +1,380 @@
# [DEF:SupersetContextParsingMixin:Module]
# @COMPLEXITY: 4
# @LAYER: Infra
# @SEMANTICS: superset, link_parsing, url, permalink, recovery
# @PURPOSE: Parse supported Superset URLs and recover canonical dataset/dashboard references for review-session intake.
# @RELATION: CALLS -> [SupersetClient]
# @RELATION: CALLS -> [SupersetContextExtractorBase]
# [/DEF:SupersetContextParsingMixin:Module]
# [DEF:_parsing_imports:Block]
from __future__ import annotations
from typing import Any, Dict, List, Optional, cast
from urllib.parse import parse_qs, urlparse
from ...logger import belief_scope, logger
from ._base import SupersetParsedContext
logger = cast(Any, logger)
# [/DEF:_parsing_imports:Block]
# [DEF:SupersetContextParsingMixin:Class]
# @COMPLEXITY: 4
# @PURPOSE: Mixin providing Superset URL parsing logic for the composed SupersetContextExtractor.
class SupersetContextParsingMixin:
# [DEF:SupersetContextParsingMixin.parse_superset_link:Function]
# @COMPLEXITY: 4
# @PURPOSE: Extract candidate identifiers and query state from supported Superset URLs.
# @RELATION: CALLS -> [SupersetClient.get_dashboard_detail]
# @PRE: link is a non-empty Superset URL compatible with the configured environment.
# @POST: returns resolved dataset/dashboard context, preserving explicit partial-recovery state if some identifiers cannot be confirmed.
# @SIDE_EFFECT: may issue Superset API reads to resolve dataset references from dashboard or chart URLs.
# @DATA_CONTRACT: Input[link:str] -> Output[SupersetParsedContext]
def parse_superset_link(self, link: str) -> SupersetParsedContext:
with belief_scope("SupersetContextExtractor.parse_superset_link"):
normalized_link = str(link or "").strip()
if not normalized_link:
logger.explore("Rejected empty Superset link during intake")
raise ValueError("Superset link must be non-empty")
parsed_url = urlparse(normalized_link)
if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc:
logger.explore(
"Superset link is not a parseable absolute URL",
extra={"link": normalized_link},
)
raise ValueError("Superset link must be an absolute http(s) URL")
logger.reason(
"Parsing Superset link for dataset review intake",
extra={"path": parsed_url.path, "query": parsed_url.query},
)
path_parts = [part for part in parsed_url.path.split("/") if part]
query_params = parse_qs(parsed_url.query, keep_blank_values=True)
query_state = self._decode_query_state(query_params)
dataset_id = self._extract_numeric_identifier(path_parts, "dataset")
dashboard_id = self._extract_numeric_identifier(path_parts, "dashboard")
dashboard_ref = self._extract_dashboard_reference(path_parts)
dashboard_permalink_key = self._extract_dashboard_permalink_key(path_parts)
chart_id = self._extract_numeric_identifier(path_parts, "chart")
resource_type = "unknown"
dataset_ref: Optional[str] = None
partial_recovery = False
unresolved_references: List[str] = []
if dataset_id is not None:
resource_type = "dataset"
dataset_ref = f"dataset:{dataset_id}"
logger.reason(
"Resolved direct dataset link",
extra={"dataset_id": dataset_id},
)
elif dashboard_permalink_key is not None:
resource_type = "dashboard"
partial_recovery = True
dataset_ref = f"dashboard_permalink:{dashboard_permalink_key}"
unresolved_references.append(
"dashboard_permalink_dataset_binding_unresolved"
)
logger.reason(
"Resolving dashboard permalink state from Superset",
extra={"permalink_key": dashboard_permalink_key},
)
permalink_payload = self.client.get_dashboard_permalink_state(
dashboard_permalink_key
)
permalink_state = (
permalink_payload.get("state", permalink_payload)
if isinstance(permalink_payload, dict)
else {}
)
if isinstance(permalink_state, dict):
for key, value in permalink_state.items():
query_state.setdefault(key, value)
# Extract filters from permalink dataMask
data_mask = permalink_state.get("dataMask")
if isinstance(data_mask, dict) and data_mask:
query_state["dataMask"] = data_mask
logger.reason(
"Extracted native filters from permalink dataMask",
extra={"filter_count": len(data_mask)},
)
resolved_dashboard_id = self._extract_dashboard_id_from_state(
permalink_state
)
resolved_chart_id = self._extract_chart_id_from_state(
permalink_state
)
if resolved_dashboard_id is not None:
dashboard_id = resolved_dashboard_id
unresolved_references = [
item
for item in unresolved_references
if item
!= "dashboard_permalink_dataset_binding_unresolved"
]
dataset_id, unresolved_references = (
self._recover_dataset_binding_from_dashboard(
dashboard_id=dashboard_id,
dataset_ref=dataset_ref,
unresolved_references=unresolved_references,
)
)
if dataset_id is not None:
dataset_ref = f"dataset:{dataset_id}"
elif resolved_chart_id is not None:
chart_id = resolved_chart_id
unresolved_references = [
item
for item in unresolved_references
if item
!= "dashboard_permalink_dataset_binding_unresolved"
]
try:
chart_payload = self.client.get_chart(chart_id)
chart_data = (
chart_payload.get("result", chart_payload)
if isinstance(chart_payload, dict)
else {}
)
datasource_id = chart_data.get("datasource_id")
if datasource_id is not None:
dataset_id = int(datasource_id)
dataset_ref = f"dataset:{dataset_id}"
logger.reason(
"Recovered dataset reference from permalink chart context",
extra={
"chart_id": chart_id,
"dataset_id": dataset_id,
},
)
else:
unresolved_references.append(
"chart_dataset_binding_unresolved"
)
except Exception as exc:
unresolved_references.append(
"chart_dataset_binding_unresolved"
)
logger.explore(
"Chart lookup failed during permalink recovery",
extra={"chart_id": chart_id, "error": str(exc)},
)
else:
logger.explore(
"Dashboard permalink state was not a structured object",
extra={"permalink_key": dashboard_permalink_key},
)
elif dashboard_id is not None or dashboard_ref is not None:
resource_type = "dashboard"
resolved_dashboard_ref = (
dashboard_id if dashboard_id is not None else dashboard_ref
)
if resolved_dashboard_ref is None:
raise ValueError("Dashboard reference could not be resolved")
logger.reason(
"Resolving dashboard-bound dataset from Superset",
extra={"dashboard_ref": resolved_dashboard_ref},
)
# Resolve dashboard detail first — handles both numeric ID and slug,
# ensuring dashboard_id is available for the native_filters_key fetch below.
dashboard_detail = self.client.get_dashboard_detail(
resolved_dashboard_ref
)
resolved_dashboard_id = dashboard_detail.get("id")
if resolved_dashboard_id is not None:
dashboard_id = int(resolved_dashboard_id)
# Check for native_filters_key in query params and fetch filter state.
# This must run AFTER dashboard_id is resolved from slug above.
native_filters_key = query_params.get("native_filters_key", [None])[0]
if native_filters_key and dashboard_id is not None:
try:
logger.reason(
"Fetching native filter state from Superset",
extra={
"dashboard_id": dashboard_id,
"filter_key": native_filters_key,
},
)
extracted = self.client.extract_native_filters_from_key(
dashboard_id, native_filters_key
)
data_mask = extracted.get("dataMask")
if isinstance(data_mask, dict) and data_mask:
query_state["native_filter_state"] = data_mask
logger.reason(
"Extracted native filter state from Superset via native_filters_key",
extra={"filter_count": len(data_mask)},
)
else:
logger.explore(
"Native filter state returned empty dataMask",
extra={
"dashboard_id": dashboard_id,
"filter_key": native_filters_key,
},
)
except Exception as exc:
logger.explore(
"Failed to fetch native filter state from Superset",
extra={
"dashboard_id": dashboard_id,
"filter_key": native_filters_key,
"error": str(exc),
},
)
datasets = dashboard_detail.get("datasets") or []
if datasets:
first_dataset = datasets[0]
resolved_dataset_id = first_dataset.get("id")
if resolved_dataset_id is not None:
dataset_id = int(resolved_dataset_id)
dataset_ref = f"dataset:{dataset_id}"
logger.reason(
"Recovered dataset reference from dashboard context",
extra={
"dashboard_id": dashboard_id,
"dataset_id": dataset_id,
"dataset_count": len(datasets),
},
)
if len(datasets) > 1:
partial_recovery = True
unresolved_references.append(
"multiple_dashboard_datasets"
)
else:
partial_recovery = True
unresolved_references.append("dashboard_dataset_id_missing")
else:
partial_recovery = True
unresolved_references.append("dashboard_dataset_binding_missing")
elif chart_id is not None:
resource_type = "chart"
partial_recovery = True
unresolved_references.append("chart_dataset_binding_unresolved")
dataset_ref = f"chart:{chart_id}"
logger.reason(
"Accepted chart link with explicit partial recovery",
extra={"chart_id": chart_id},
)
else:
logger.explore(
"Unsupported Superset link shape encountered",
extra={"path": parsed_url.path},
)
raise ValueError("Unsupported Superset link shape")
dataset_payload: Optional[Dict[str, Any]] = None
if dataset_id is not None:
try:
dataset_payload = self.client.get_dataset_detail(dataset_id)
table_name = str(
dataset_payload.get("table_name") or ""
).strip()
schema_name = str(
dataset_payload.get("schema") or ""
).strip()
if table_name:
dataset_ref = (
f"{schema_name}.{table_name}"
if schema_name
else table_name
)
logger.reason(
"Canonicalized dataset reference from dataset detail",
extra={
"dataset_ref": dataset_ref,
"dataset_id": dataset_id,
},
)
except Exception as exc:
partial_recovery = True
unresolved_references.append("dataset_detail_lookup_failed")
logger.explore(
"Dataset detail lookup failed during link parsing; keeping session usable",
extra={"dataset_id": dataset_id, "error": str(exc)},
)
imported_filters = self._extract_imported_filters(query_state)
result = SupersetParsedContext(
source_url=normalized_link,
dataset_ref=dataset_ref or "unresolved",
dataset_id=dataset_id,
dashboard_id=dashboard_id,
chart_id=chart_id,
resource_type=resource_type,
query_state=query_state,
imported_filters=imported_filters,
unresolved_references=unresolved_references,
partial_recovery=partial_recovery,
dataset_payload=dataset_payload,
)
logger.reflect(
"Superset link parsing completed",
extra={
"dataset_ref": result.dataset_ref,
"dataset_id": result.dataset_id,
"dashboard_id": result.dashboard_id,
"chart_id": result.chart_id,
"partial_recovery": result.partial_recovery,
"unresolved_references": result.unresolved_references,
"imported_filters": len(result.imported_filters),
},
)
return result
# [/DEF:SupersetContextParsingMixin.parse_superset_link:Function]
# [DEF:SupersetContextParsingMixin._recover_dataset_binding_from_dashboard:Function]
# @COMPLEXITY: 3
# @PURPOSE: Recover a dataset binding from resolved dashboard context while preserving explicit unresolved markers.
# @RELATION: CALLS -> [SupersetClient.get_dashboard_detail]
def _recover_dataset_binding_from_dashboard(
self,
dashboard_id: int,
dataset_ref: Optional[str],
unresolved_references: List[str],
) -> tuple[Optional[int], List[str]]:
dashboard_detail = self.client.get_dashboard_detail(dashboard_id)
datasets = dashboard_detail.get("datasets") or []
if datasets:
first_dataset = datasets[0]
resolved_dataset_id = first_dataset.get("id")
if resolved_dataset_id is not None:
resolved_dataset = int(resolved_dataset_id)
logger.reason(
"Recovered dataset reference from dashboard permalink context",
extra={
"dashboard_id": dashboard_id,
"dataset_id": resolved_dataset,
"dataset_count": len(datasets),
"dataset_ref": dataset_ref,
},
)
if (
len(datasets) > 1
and "multiple_dashboard_datasets" not in unresolved_references
):
unresolved_references.append("multiple_dashboard_datasets")
return resolved_dataset, unresolved_references
if "dashboard_dataset_id_missing" not in unresolved_references:
unresolved_references.append("dashboard_dataset_id_missing")
return None, unresolved_references
if "dashboard_dataset_binding_missing" not in unresolved_references:
unresolved_references.append("dashboard_dataset_binding_missing")
return None, unresolved_references
# [/DEF:SupersetContextParsingMixin._recover_dataset_binding_from_dashboard:Function]
# [/DEF:SupersetContextParsingMixin:Class]

112
backend/src/core/utils/superset_context_extractor/_pii.py Normal file
View File

@@ -0,0 +1,112 @@
# [DEF:SupersetContextExtractorPII:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: superset, pii, masking, sanitization, privacy
# @PURPOSE: PII redaction helpers for assistant-facing dataset-review context — mask emails, UUIDs, long digit strings, and mixed identifiers.
# [/DEF:SupersetContextExtractorPII:Module]
# [DEF:_pii_imports:Block]
import re
from copy import deepcopy
from typing import Any, Dict, List, Tuple
# [/DEF:_pii_imports:Block]
_EMAIL_PATTERN = re.compile(
r"(?P<local>[A-Za-z0-9._%+-]+)@(?P<domain>[A-Za-z0-9.-]+\.[A-Za-z]{2,})"
)
_UUID_PATTERN = re.compile(
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
)
_LONG_DIGIT_PATTERN = re.compile(r"\b\d{6,}\b")
_MIXED_IDENTIFIER_PATTERN = re.compile(
r"\b(?=[A-Za-z0-9_-]{10,}\b)(?=.*[A-Za-z])(?=.*\d)[A-Za-z0-9_-]+\b"
)
# [DEF:mask_pii_value:Function]
# @COMPLEXITY: 2
# @PURPOSE: Redact likely sensitive identifiers while preserving structural hints for assistant-facing dataset-review context.
def mask_pii_value(value: Any) -> Tuple[Any, bool]:
if isinstance(value, str):
return _mask_sensitive_text(value)
if isinstance(value, list):
masked_any = False
masked_list: List[Any] = []
for item in value:
masked_item, item_masked = mask_pii_value(item)
masked_list.append(masked_item)
masked_any = masked_any or item_masked
return masked_list, masked_any
if isinstance(value, dict):
masked_any = False
masked_dict: Dict[str, Any] = {}
for key, item in value.items():
masked_item, item_masked = mask_pii_value(item)
masked_dict[key] = masked_item
masked_any = masked_any or item_masked
return masked_dict, masked_any
return value, False
# [/DEF:mask_pii_value:Function]
# [DEF:sanitize_imported_filter_for_assistant:Function]
# @COMPLEXITY: 2
# @PURPOSE: Build assistant-safe imported-filter payload with raw-value redaction metadata when sensitive tokens are detected.
def sanitize_imported_filter_for_assistant(
filter_payload: Dict[str, Any],
) -> Dict[str, Any]:
sanitized = deepcopy(filter_payload)
masked_raw_value, was_masked = mask_pii_value(filter_payload.get("raw_value"))
sanitized["raw_value"] = masked_raw_value
sanitized["raw_value_masked"] = bool(
filter_payload.get("raw_value_masked", False) or was_masked
)
return sanitized
# [/DEF:sanitize_imported_filter_for_assistant:Function]
# [DEF:_mask_sensitive_text:Function]
# @COMPLEXITY: 2
# @PURPOSE: Redact likely PII substrings from one string while preserving non-sensitive delimiters and token shape.
def _mask_sensitive_text(value: str) -> Tuple[str, bool]:
masked = value
masked_any = False
def _replace_email(match: re.Match[str]) -> str:
nonlocal masked_any
masked_any = True
domain = match.group("domain")
return f"***@{domain}"
def _replace_uuid(match: re.Match[str]) -> str:
nonlocal masked_any
masked_any = True
token = match.group(0)
return "********-****-****-****-" + token[-12:]
def _replace_long_digits(match: re.Match[str]) -> str:
nonlocal masked_any
masked_any = True
token = match.group(0)
return "*" * max(len(token) - 2, 1) + token[-2:]
def _replace_mixed_identifier(match: re.Match[str]) -> str:
nonlocal masked_any
token = match.group(0)
if token.isupper() and len(token) <= 5:
return token
masked_any = True
return token[:2] + "***" + token[-2:]
masked = _EMAIL_PATTERN.sub(_replace_email, masked)
masked = _UUID_PATTERN.sub(_replace_uuid, masked)
masked = _LONG_DIGIT_PATTERN.sub(_replace_long_digits, masked)
masked = _MIXED_IDENTIFIER_PATTERN.sub(_replace_mixed_identifier, masked)
return masked, masked_any
# [/DEF:_mask_sensitive_text:Function]

314
backend/src/core/utils/superset_context_extractor/_recovery.py Normal file
View File

@@ -0,0 +1,314 @@
# [DEF:SupersetContextRecoveryMixin:Module]
# @COMPLEXITY: 4
# @LAYER: Infra
# @SEMANTICS: superset, filters, recovery, imported_filters
# @PURPOSE: Recover imported filters from Superset parsed context and dashboard metadata.
# @RELATION: CALLS -> [SupersetClient]
# @RELATION: DEPENDS_ON -> [SupersetContextExtractorBase]
# [/DEF:SupersetContextRecoveryMixin:Module]
# [DEF:_recovery_imports:Block]
from __future__ import annotations
import json
from copy import deepcopy
from typing import Any, Dict, List, Set, cast
from ...logger import belief_scope, logger
from ._base import SupersetParsedContext
logger = cast(Any, logger)
# [/DEF:_recovery_imports:Block]
# [DEF:SupersetContextRecoveryMixin:Class]
# @COMPLEXITY: 4
# @PURPOSE: Mixin providing filter recovery for the composed SupersetContextExtractor.
class SupersetContextRecoveryMixin:
# [DEF:SupersetContextRecoveryMixin.recover_imported_filters:Function]
# @COMPLEXITY: 4
# @PURPOSE: Build imported filter entries from URL state and Superset-side saved context.
# @RELATION: CALLS -> [SupersetClient.get_dashboard]
# @PRE: parsed_context comes from a successful Superset link parse for one environment.
# @POST: returns explicit recovered and partial filter entries with preserved provenance and confirmation requirements.
# @SIDE_EFFECT: may issue Superset reads for dashboard metadata enrichment.
# @DATA_CONTRACT: Input[SupersetParsedContext] -> Output[List[Dict[str,Any]]]
def recover_imported_filters(
self, parsed_context: SupersetParsedContext
) -> List[Dict[str, Any]]:
with belief_scope("SupersetContextExtractor.recover_imported_filters"):
recovered_filters: List[Dict[str, Any]] = []
seen_filter_keys: Set[str] = set()
metadata_filters: List[Dict[str, Any]] = []
metadata_filters_by_id: Dict[str, Dict[str, Any]] = {}
def merge_recovered_filter(candidate: Dict[str, Any]) -> None:
filter_key = candidate["filter_name"].strip().lower()
existing_index = next(
(
index
for index, existing in enumerate(recovered_filters)
if existing["filter_name"].strip().lower() == filter_key
),
None,
)
if existing_index is None:
seen_filter_keys.add(filter_key)
recovered_filters.append(candidate)
return
existing = recovered_filters[existing_index]
if existing.get("display_name") in {
None,
"",
existing.get("filter_name"),
} and candidate.get("display_name"):
existing["display_name"] = candidate["display_name"]
if (
existing.get("raw_value") is None
and candidate.get("raw_value") is not None
):
existing["raw_value"] = candidate["raw_value"]
existing["confidence_state"] = candidate.get(
"confidence_state", "imported"
)
existing["requires_confirmation"] = candidate.get(
"requires_confirmation", False
)
existing["recovery_status"] = candidate.get(
"recovery_status", "recovered"
)
existing["source"] = candidate.get(
"source", existing.get("source")
)
if (
existing.get("normalized_value") is None
and candidate.get("normalized_value") is not None
):
existing["normalized_value"] = deepcopy(
candidate["normalized_value"]
)
if (
existing.get("notes")
and candidate.get("notes")
and candidate["notes"] not in existing["notes"]
):
existing["notes"] = f"{existing['notes']}; {candidate['notes']}"
if parsed_context.dashboard_id is not None:
try:
dashboard_payload = self.client.get_dashboard(
parsed_context.dashboard_id
)
dashboard_record = (
dashboard_payload.get("result", dashboard_payload)
if isinstance(dashboard_payload, dict)
else {}
)
json_metadata = dashboard_record.get("json_metadata")
if isinstance(json_metadata, str) and json_metadata.strip():
json_metadata = json.loads(json_metadata)
if not isinstance(json_metadata, dict):
json_metadata = {}
native_filter_configuration = (
json_metadata.get("native_filter_configuration") or []
)
default_filters = json_metadata.get("default_filters") or {}
if isinstance(default_filters, str) and default_filters.strip():
try:
default_filters = json.loads(default_filters)
except Exception:
logger.explore(
"Superset default_filters payload was not valid JSON",
extra={
"dashboard_id": parsed_context.dashboard_id
},
)
default_filters = {}
for item in native_filter_configuration:
if not isinstance(item, dict):
continue
filter_name = str(
item.get("name")
or item.get("filter_name")
or item.get("column")
or ""
).strip()
if not filter_name:
continue
display_name = (
item.get("label") or item.get("name") or filter_name
)
filter_id = str(item.get("id") or "").strip()
default_value = None
if isinstance(default_filters, dict):
default_value = default_filters.get(filter_name)
metadata_filter = self._normalize_imported_filter_payload(
{
"filter_name": filter_name,
"display_name": display_name,
"raw_value": default_value,
"source": "superset_native",
"recovery_status": "recovered"
if default_value is not None
else "partial",
"requires_confirmation": default_value is None,
"notes": "Recovered from Superset dashboard native filter configuration",
},
default_source="superset_native",
default_note="Recovered from Superset dashboard native filter configuration",
)
metadata_filters.append(metadata_filter)
if filter_id:
metadata_filters_by_id[filter_id.lower()] = {
"filter_name": filter_name,
"display_name": display_name,
}
except Exception as exc:
logger.explore(
"Dashboard native filter enrichment failed; preserving partial imported filters",
extra={
"dashboard_id": parsed_context.dashboard_id,
"error": str(exc),
"filter_count": len(recovered_filters),
},
)
metadata_filters = []
metadata_filters_by_id = {}
for item in parsed_context.imported_filters:
normalized = self._normalize_imported_filter_payload(
item,
default_source="superset_url",
default_note="Recovered from Superset URL state",
)
metadata_match = metadata_filters_by_id.get(
normalized["filter_name"].strip().lower()
)
if metadata_match is not None:
normalized["filter_name"] = metadata_match["filter_name"]
normalized["display_name"] = metadata_match["display_name"]
normalized["notes"] = (
"Recovered from Superset URL state and reconciled against dashboard native filter metadata"
)
merge_recovered_filter(normalized)
logger.reflect(
"Recovered filter from URL state",
extra={
"filter_name": normalized["filter_name"],
"source": normalized["source"],
"has_value": normalized["raw_value"] is not None,
"canonicalized": metadata_match is not None,
},
)
if parsed_context.dashboard_id is None:
logger.reflect(
"Imported filter recovery completed without dashboard enrichment",
extra={
"dashboard_id": None,
"filter_count": len(recovered_filters),
"partial_recovery": parsed_context.partial_recovery,
},
)
return recovered_filters
for saved_filter in metadata_filters:
merge_recovered_filter(saved_filter)
logger.reflect(
"Recovered filter from dashboard metadata",
extra={
"filter_name": saved_filter["filter_name"],
"has_value": saved_filter["raw_value"] is not None,
},
)
if not recovered_filters:
recovered_filters.append(
self._normalize_imported_filter_payload(
{
"filter_name": f"dashboard_{parsed_context.dashboard_id}_filters",
"display_name": "Dashboard native filters",
"raw_value": None,
"source": "superset_native",
"recovery_status": "partial",
"requires_confirmation": True,
"notes": "Superset dashboard filter configuration could not be recovered fully",
},
default_source="superset_native",
default_note="Superset dashboard filter configuration could not be recovered fully",
)
)
logger.reflect(
"Imported filter recovery completed with dashboard enrichment",
extra={
"dashboard_id": parsed_context.dashboard_id,
"filter_count": len(recovered_filters),
"partial_entries": len(
[
item
for item in recovered_filters
if item["recovery_status"] == "partial"
]
),
},
)
return recovered_filters
# [/DEF:SupersetContextRecoveryMixin.recover_imported_filters:Function]
# [DEF:SupersetContextRecoveryMixin._normalize_imported_filter_payload:Function]
# @COMPLEXITY: 2
# @PURPOSE: Normalize one imported-filter payload with explicit provenance and confirmation state.
def _normalize_imported_filter_payload(
self,
payload: Dict[str, Any],
default_source: str,
default_note: str,
) -> Dict[str, Any]:
raw_value = payload.get("raw_value")
if "raw_value" not in payload and "value" in payload:
raw_value = payload.get("value")
recovery_status = (
str(
payload.get("recovery_status")
or ("recovered" if raw_value is not None else "partial")
)
.strip()
.lower()
)
requires_confirmation = bool(
payload.get(
"requires_confirmation",
raw_value is None or recovery_status != "recovered",
)
)
return {
"filter_name": str(
payload.get("filter_name") or "unresolved_filter"
).strip(),
"display_name": payload.get("display_name"),
"raw_value": raw_value,
"normalized_value": payload.get("normalized_value"),
"source": str(payload.get("source") or default_source),
"confidence_state": "imported"
if raw_value is not None
else "unresolved",
"requires_confirmation": requires_confirmation,
"recovery_status": recovery_status,
"notes": str(payload.get("notes") or default_note),
}
# [/DEF:SupersetContextRecoveryMixin._normalize_imported_filter_payload:Function]
# [/DEF:SupersetContextRecoveryMixin:Class]

255
backend/src/core/utils/superset_context_extractor/_templates.py Normal file
View File

@@ -0,0 +1,255 @@
# [DEF:SupersetContextTemplatesMixin:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: superset, template_variables, jinja, discovery, deterministic
# @PURPOSE: Deterministically detect runtime variables and Jinja references from dataset query-bearing fields without execution.
# @RELATION: DEPENDS_ON -> [TemplateVariable]
# @RELATION: DEPENDS_ON -> [SupersetContextExtractorBase]
# [/DEF:SupersetContextTemplatesMixin:Module]
# [DEF:_templates_imports:Block]
from __future__ import annotations
import re
from typing import Any, Dict, List, Optional, Set, cast
from ...logger import belief_scope, logger
logger = cast(Any, logger)
# [/DEF:_templates_imports:Block]
# [DEF:SupersetContextTemplatesMixin:Class]
# @COMPLEXITY: 3
# @PURPOSE: Mixin providing template variable discovery for the composed SupersetContextExtractor.
class SupersetContextTemplatesMixin:
# [DEF:SupersetContextTemplatesMixin.discover_template_variables:Function]
# @COMPLEXITY: 4
# @PURPOSE: Detect runtime variables and Jinja references from dataset query-bearing fields.
# @PRE: dataset_payload is a Superset dataset-detail style payload with query-bearing fields when available.
# @POST: returns deduplicated explicit variable records without executing Jinja or fabricating runtime values.
# @SIDE_EFFECT: none.
# @DATA_CONTRACT: Input[dataset_payload:Dict[str,Any]] -> Output[List[Dict[str,Any]]]
def discover_template_variables(
self, dataset_payload: Dict[str, Any]
) -> List[Dict[str, Any]]:
with belief_scope("SupersetContextExtractor.discover_template_variables"):
discovered: List[Dict[str, Any]] = []
seen_variable_names: Set[str] = set()
for expression_source in self._collect_query_bearing_expressions(
dataset_payload
):
for filter_match in re.finditer(
r"filter_values\(\s*['\"]([^'\"]+)['\"]\s*\)",
expression_source,
flags=re.IGNORECASE,
):
variable_name = str(filter_match.group(1) or "").strip()
if not variable_name:
continue
self._append_template_variable(
discovered=discovered,
seen_variable_names=seen_variable_names,
variable_name=variable_name,
expression_source=expression_source,
variable_kind="native_filter",
is_required=True,
default_value=None,
)
for url_param_match in re.finditer(
r"url_param\(\s*['\"]([^'\"]+)['\"]\s*(?:,\s*([^)]+))?\)",
expression_source,
flags=re.IGNORECASE,
):
variable_name = str(url_param_match.group(1) or "").strip()
if not variable_name:
continue
default_literal = url_param_match.group(2)
self._append_template_variable(
discovered=discovered,
seen_variable_names=seen_variable_names,
variable_name=variable_name,
expression_source=expression_source,
variable_kind="parameter",
is_required=default_literal is None,
default_value=self._normalize_default_literal(
default_literal
),
)
for jinja_match in re.finditer(
r"\{\{\s*(.*?)\s*\}\}", expression_source, flags=re.DOTALL
):
expression = str(jinja_match.group(1) or "").strip()
if not expression:
continue
if any(
token in expression
for token in (
"filter_values(",
"url_param(",
"get_filters(",
)
):
continue
variable_name = self._extract_primary_jinja_identifier(expression)
if not variable_name:
continue
self._append_template_variable(
discovered=discovered,
seen_variable_names=seen_variable_names,
variable_name=variable_name,
expression_source=expression_source,
variable_kind="derived"
if "." in expression or "|" in expression
else "parameter",
is_required=True,
default_value=None,
)
logger.reflect(
"Template variable discovery completed deterministically",
extra={
"dataset_id": dataset_payload.get("id"),
"variable_count": len(discovered),
"variable_names": [
item["variable_name"] for item in discovered
],
},
)
return discovered
# [/DEF:SupersetContextTemplatesMixin.discover_template_variables:Function]
# [DEF:SupersetContextTemplatesMixin._collect_query_bearing_expressions:Function]
# @COMPLEXITY: 3
# @PURPOSE: Collect SQL and expression-bearing dataset fields for deterministic template-variable discovery.
# @RELATION: DEPENDS_ON -> [SupersetContextTemplatesMixin.discover_template_variables]
def _collect_query_bearing_expressions(
self, dataset_payload: Dict[str, Any]
) -> List[str]:
expressions: List[str] = []
def append_expression(candidate: Any) -> None:
if not isinstance(candidate, str):
return
normalized = candidate.strip()
if normalized:
expressions.append(normalized)
append_expression(dataset_payload.get("sql"))
append_expression(dataset_payload.get("query"))
append_expression(dataset_payload.get("template_sql"))
metrics_payload = dataset_payload.get("metrics") or []
if isinstance(metrics_payload, list):
for metric in metrics_payload:
if isinstance(metric, str):
append_expression(metric)
continue
if not isinstance(metric, dict):
continue
append_expression(metric.get("expression"))
append_expression(metric.get("sqlExpression"))
append_expression(metric.get("metric_name"))
columns_payload = dataset_payload.get("columns") or []
if isinstance(columns_payload, list):
for column in columns_payload:
if not isinstance(column, dict):
continue
append_expression(column.get("sqlExpression"))
append_expression(column.get("expression"))
return expressions
# [/DEF:SupersetContextTemplatesMixin._collect_query_bearing_expressions:Function]
# [DEF:SupersetContextTemplatesMixin._append_template_variable:Function]
# @COMPLEXITY: 2
# @PURPOSE: Append one deduplicated template-variable descriptor.
def _append_template_variable(
self,
discovered: List[Dict[str, Any]],
seen_variable_names: Set[str],
variable_name: str,
expression_source: str,
variable_kind: str,
is_required: bool,
default_value: Any,
) -> None:
normalized_name = str(variable_name or "").strip()
if not normalized_name:
return
seen_key = normalized_name.lower()
if seen_key in seen_variable_names:
return
seen_variable_names.add(seen_key)
discovered.append(
{
"variable_name": normalized_name,
"expression_source": expression_source,
"variable_kind": variable_kind,
"is_required": is_required,
"default_value": default_value,
"mapping_status": "unmapped",
}
)
# [/DEF:SupersetContextTemplatesMixin._append_template_variable:Function]
# [DEF:SupersetContextTemplatesMixin._extract_primary_jinja_identifier:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a deterministic primary identifier from a Jinja expression without executing it.
def _extract_primary_jinja_identifier(self, expression: str) -> Optional[str]:
matched = re.match(r"([A-Za-z_][A-Za-z0-9_]*)", expression.strip())
if matched is None:
return None
candidate = matched.group(1)
if candidate in {
"if",
"else",
"for",
"set",
"True",
"False",
"none",
"None",
}:
return None
return candidate
# [/DEF:SupersetContextTemplatesMixin._extract_primary_jinja_identifier:Function]
# [DEF:SupersetContextTemplatesMixin._normalize_default_literal:Function]
# @COMPLEXITY: 2
# @PURPOSE: Normalize literal default fragments from template helper calls into JSON-safe values.
def _normalize_default_literal(self, literal: Optional[str]) -> Any:
normalized_literal = str(literal or "").strip()
if not normalized_literal:
return None
if (
normalized_literal.startswith("'") and normalized_literal.endswith("'")
) or (
normalized_literal.startswith('"') and normalized_literal.endswith('"')
):
return normalized_literal[1:-1]
lowered = normalized_literal.lower()
if lowered in {"true", "false"}:
return lowered == "true"
if lowered in {"null", "none"}:
return None
try:
return int(normalized_literal)
except ValueError:
try:
return float(normalized_literal)
except ValueError:
return normalized_literal
# [/DEF:SupersetContextTemplatesMixin._normalize_default_literal:Function]
# [/DEF:SupersetContextTemplatesMixin:Class]

66
backend/src/services/git/__init__.py Normal file
View File

@@ -0,0 +1,66 @@
# [DEF:GitServiceModule:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: git, service, decomposition, mixin, composition
# @PURPOSE: Composed GitService via multiple inheritance from domain-specific mixins.
# @RELATION: DEPENDS_ON -> [GitServiceBase]
# @RELATION: DEPENDS_ON -> [GitServiceBranchMixin]
# @RELATION: DEPENDS_ON -> [GitServiceSyncMixin]
# @RELATION: DEPENDS_ON -> [GitServiceStatusMixin]
# @RELATION: DEPENDS_ON -> [GitServiceMergeMixin]
# @RELATION: DEPENDS_ON -> [GitServiceUrlMixin]
# @RELATION: DEPENDS_ON -> [GitServiceGiteaMixin]
# @RELATION: DEPENDS_ON -> [GitServiceGithubMixin]
# @RELATION: DEPENDS_ON -> [GitServiceGitlabMixin]
#
# @RATIONALE: Decomposed from monolithic git_service.py (2101 lines) into
# domain-scoped mixins to satisfy INV_7 (module < 400 lines). The composed class
# preserves the original public API surface — all consumers continue to import
# `from src.services.git_service import GitService` without changes.
# @REJECTED: Keeping a single 2101-line file — violates fractal limit INV_7.
from ._base import GitServiceBase
from ._branch import GitServiceBranchMixin
from ._sync import GitServiceSyncMixin
from ._status import GitServiceStatusMixin
from ._merge import GitServiceMergeMixin
from ._url import GitServiceUrlMixin
from ._gitea import GitServiceGiteaMixin
from ._remote_providers import GitServiceGithubMixin, GitServiceGitlabMixin
__all__ = ["GitService"]
# [DEF:GitService:Class]
# @COMPLEXITY: 3
# @PURPOSE: Composed GitService class providing all Git operations — init, clone, branches, commits, push, pull,
# merge, status, diff, history, and provider API operations (Gitea, GitHub, GitLab).
# @RELATION: INHERITS -> [GitServiceBase]
# @RELATION: INHERITS -> [GitServiceBranchMixin]
# @RELATION: INHERITS -> [GitServiceSyncMixin]
# @RELATION: INHERITS -> [GitServiceStatusMixin]
# @RELATION: INHERITS -> [GitServiceMergeMixin]
# @RELATION: INHERITS -> [GitServiceUrlMixin]
# @RELATION: INHERITS -> [GitServiceGiteaMixin]
# @RELATION: INHERITS -> [GitServiceGithubMixin]
# @RELATION: INHERITS -> [GitServiceGitlabMixin]
class GitService(
GitServiceGiteaMixin,
GitServiceGithubMixin,
GitServiceGitlabMixin,
GitServiceMergeMixin,
GitServiceSyncMixin,
GitServiceStatusMixin,
GitServiceBranchMixin,
GitServiceUrlMixin,
GitServiceBase,
):
"""
Composed GitService — all Git operations via domain-specific mixins.
MRO ensures provider mixins and domain mixins resolve before the base class.
All consumers continue to use: from src.services.git_service import GitService
"""
pass
# [/DEF:GitService:Class]
# [/DEF:GitServiceModule:Module]

319
backend/src/services/git/_base.py Normal file
View File

@@ -0,0 +1,319 @@
# [DEF:GitServiceBase:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: git, service, repository, version_control, initialization
# @PURPOSE: Core GitService base class — initialization, path resolution, repo lifecycle (init/delete/get), and identity configuration.
# @RELATION: INHERITED_BY -> [GitService]
# @RELATION: DEPENDS_ON -> [SessionLocal]
# @RELATION: DEPENDS_ON -> [AppConfigRecord]
# @RELATION: DEPENDS_ON -> [GitRepository]
import os
import re
import shutil
from pathlib import Path
from typing import Any, Dict, List, Optional
from git import Repo
from git.exc import InvalidGitRepositoryError, NoSuchPathError
from fastapi import HTTPException
from src.core.logger import logger, belief_scope
from src.models.git import GitRepository
from src.models.config import AppConfigRecord
from src.core.database import SessionLocal
# [DEF:GitServiceBase:Class]
# @COMPLEXITY: 3
# @PURPOSE: Base class for GitService providing initialization, path resolution, repository lifecycle, and identity.
class GitServiceBase:
# [DEF:GitService_init:Function]
# @PURPOSE: Initializes the GitService with a base path for repositories.
# @PARAM: base_path (str) - Root directory for all Git clones.
# @PRE: base_path is a valid string path.
# @POST: GitService is initialized; base_path directory exists.
def __init__(self, base_path: str = "git_repos"):
with belief_scope("GitService.__init__"):
backend_root = Path(__file__).parents[3]
self.legacy_base_path = str((backend_root / "git_repos").resolve())
self._uses_default_base_path = base_path == "git_repos"
self.base_path = self._resolve_base_path(base_path)
self._ensure_base_path_exists()
# [/DEF:GitService_init:Function]
# [DEF:_ensure_base_path_exists:Function]
# @PURPOSE: Ensure the repositories root directory exists and is a directory.
# @PRE: self.base_path is resolved to filesystem path.
# @POST: self.base_path exists as directory or raises ValueError.
def _ensure_base_path_exists(self) -> None:
base = Path(self.base_path)
if base.exists() and not base.is_dir():
raise ValueError(f"Git repositories base path is not a directory: {self.base_path}")
try:
base.mkdir(parents=True, exist_ok=True)
except (PermissionError, OSError) as e:
logger.warning(
f"[_ensure_base_path_exists][Coherence:Failed] Cannot create Git repositories base path: {self.base_path}. Error: {e}"
)
raise ValueError(f"Cannot create Git repositories base path: {self.base_path}. {e}")
# [/DEF:_ensure_base_path_exists:Function]
# [DEF:_resolve_base_path:Function]
# @PURPOSE: Resolve base repository directory from explicit argument or global storage settings.
# @PRE: base_path is a string path.
# @POST: Returns absolute path for Git repositories root.
# @RETURN: str
def _resolve_base_path(self, base_path: str) -> str:
backend_root = Path(__file__).parents[3]
fallback_path = str((backend_root / base_path).resolve())
if base_path != "git_repos":
return fallback_path
try:
session = SessionLocal()
try:
config_row = session.query(AppConfigRecord).filter(AppConfigRecord.id == "global").first()
finally:
session.close()
payload = (config_row.payload if config_row and config_row.payload else {}) if config_row else {}
storage_cfg = payload.get("settings", {}).get("storage", {}) if isinstance(payload, dict) else {}
root_path = str(storage_cfg.get("root_path", "")).strip()
repo_path = str(storage_cfg.get("repo_path", "")).strip()
if not root_path:
return fallback_path
project_root = Path(__file__).parents[4]
root = Path(root_path)
if not root.is_absolute():
root = (project_root / root).resolve()
repo_root = Path(repo_path) if repo_path else Path("repositorys")
if repo_root.is_absolute():
return str(repo_root.resolve())
return str((root / repo_root).resolve())
except Exception as e:
logger.warning(f"[_resolve_base_path][Coherence:Failed] Falling back to default path: {e}")
return fallback_path
# [/DEF:_resolve_base_path:Function]
# [DEF:_normalize_repo_key:Function]
# @PURPOSE: Convert user/dashboard-provided key to safe filesystem directory name.
# @PRE: repo_key can be None/empty.
# @POST: Returns normalized non-empty key.
# @RETURN: str
def _normalize_repo_key(self, repo_key: Optional[str]) -> str:
raw_key = str(repo_key or "").strip().lower()
normalized = re.sub(r"[^a-z0-9._-]+", "-", raw_key).strip("._-")
return normalized or "dashboard"
# [/DEF:_normalize_repo_key:Function]
# [DEF:_update_repo_local_path:Function]
# @PURPOSE: Persist repository local_path in GitRepository table when record exists.
# @PRE: dashboard_id is valid integer.
# @POST: local_path is updated for existing record.
def _update_repo_local_path(self, dashboard_id: int, local_path: str) -> None:
try:
session = SessionLocal()
try:
db_repo = (
session.query(GitRepository)
.filter(GitRepository.dashboard_id == int(dashboard_id))
.first()
)
if db_repo:
db_repo.local_path = local_path
session.commit()
finally:
session.close()
except Exception as e:
logger.warning(f"[_update_repo_local_path][Coherence:Failed] {e}")
# [/DEF:_update_repo_local_path:Function]
# [DEF:_migrate_repo_directory:Function]
# @PURPOSE: Move legacy repository directory to target path and sync DB metadata.
# @PRE: source_path exists.
# @POST: Repository content available at target_path.
# @RETURN: str
def _migrate_repo_directory(self, dashboard_id: int, source_path: str, target_path: str) -> str:
source_abs = os.path.abspath(source_path)
target_abs = os.path.abspath(target_path)
if source_abs == target_abs:
return source_abs
if os.path.exists(target_abs):
logger.warning(f"[_migrate_repo_directory][Action] Target already exists, keeping source path: {target_abs}")
return source_abs
Path(target_abs).parent.mkdir(parents=True, exist_ok=True)
try:
os.replace(source_abs, target_abs)
except OSError:
shutil.move(source_abs, target_abs)
self._update_repo_local_path(dashboard_id, target_abs)
logger.info(
f"[_migrate_repo_directory][Coherence:OK] Repository migrated for dashboard {dashboard_id}: {source_abs} -> {target_abs}"
)
return target_abs
# [/DEF:_migrate_repo_directory:Function]
# [DEF:_get_repo_path:Function]
# @PURPOSE: Resolves the local filesystem path for a dashboard's repository.
# @PARAM: dashboard_id (int)
# @PARAM: repo_key (Optional[str]) - Slug-like key used when DB local_path is absent.
# @PRE: dashboard_id is an integer.
# @POST: Returns DB-local_path when present, otherwise base_path/<normalized repo_key>.
# @RETURN: str
def _get_repo_path(self, dashboard_id: int, repo_key: Optional[str] = None) -> str:
with belief_scope("GitService._get_repo_path"):
if dashboard_id is None:
raise ValueError("dashboard_id cannot be None")
self._ensure_base_path_exists()
fallback_key = repo_key if repo_key is not None else str(dashboard_id)
normalized_key = self._normalize_repo_key(fallback_key)
target_path = os.path.join(self.base_path, normalized_key)
if not self._uses_default_base_path:
return target_path
try:
session = SessionLocal()
try:
db_repo = (
session.query(GitRepository)
.filter(GitRepository.dashboard_id == int(dashboard_id))
.first()
)
finally:
session.close()
if db_repo and db_repo.local_path:
db_path = os.path.abspath(db_repo.local_path)
if os.path.exists(db_path):
if (
os.path.abspath(self.base_path) != os.path.abspath(self.legacy_base_path)
and db_path.startswith(os.path.abspath(self.legacy_base_path) + os.sep)
):
return self._migrate_repo_directory(dashboard_id, db_path, target_path)
return db_path
except Exception as e:
logger.warning(f"[_get_repo_path][Coherence:Failed] Could not resolve local_path from DB: {e}")
legacy_id_path = os.path.join(self.legacy_base_path, str(dashboard_id))
if os.path.exists(legacy_id_path) and not os.path.exists(target_path):
return self._migrate_repo_directory(dashboard_id, legacy_id_path, target_path)
if os.path.exists(target_path):
self._update_repo_local_path(dashboard_id, target_path)
return target_path
# [/DEF:_get_repo_path:Function]
# [DEF:init_repo:Function]
# @PURPOSE: Initialize or clone a repository for a dashboard.
# @PARAM: dashboard_id (int), remote_url (str), pat (str), repo_key (Optional[str]).
# @PRE: dashboard_id is int, remote_url is valid Git URL, pat is provided.
# @POST: Repository is cloned or opened at the local path.
# @RETURN: Repo
def init_repo(self, dashboard_id: int, remote_url: str, pat: str, repo_key: Optional[str] = None) -> Repo:
with belief_scope("GitService.init_repo"):
self._ensure_base_path_exists()
repo_path = self._get_repo_path(dashboard_id, repo_key=repo_key or str(dashboard_id))
Path(repo_path).parent.mkdir(parents=True, exist_ok=True)
if pat and "://" in remote_url:
proto, rest = remote_url.split("://", 1)
auth_url = f"{proto}://oauth2:{pat}@{rest}"
else:
auth_url = remote_url
if os.path.exists(repo_path):
logger.info(f"[init_repo][Action] Opening existing repo at {repo_path}")
try:
repo = Repo(repo_path)
except (InvalidGitRepositoryError, NoSuchPathError):
logger.warning(f"[init_repo][Action] Existing path is not a Git repository, recreating: {repo_path}")
stale_path = Path(repo_path)
if stale_path.exists():
shutil.rmtree(stale_path, ignore_errors=True)
if stale_path.exists():
try:
stale_path.unlink()
except Exception:
pass
repo = Repo.clone_from(auth_url, repo_path)
self._ensure_gitflow_branches(repo, dashboard_id)
return repo
logger.info(f"[init_repo][Action] Cloning {remote_url} to {repo_path}")
repo = Repo.clone_from(auth_url, repo_path)
self._ensure_gitflow_branches(repo, dashboard_id)
return repo
# [/DEF:init_repo:Function]
# [DEF:delete_repo:Function]
# @PURPOSE: Remove local repository and DB binding for a dashboard.
# @PRE: dashboard_id is a valid integer.
# @POST: Local path is deleted when present and GitRepository row is removed.
def delete_repo(self, dashboard_id: int) -> None:
with belief_scope("GitService.delete_repo"):
repo_path = self._get_repo_path(dashboard_id)
removed_files = False
if os.path.exists(repo_path):
if os.path.isdir(repo_path):
shutil.rmtree(repo_path)
else:
os.remove(repo_path)
removed_files = True
session = SessionLocal()
try:
db_repo = (
session.query(GitRepository)
.filter(GitRepository.dashboard_id == int(dashboard_id))
.first()
)
if db_repo:
session.delete(db_repo)
session.commit()
return
if removed_files:
return
raise HTTPException(
status_code=404,
detail=f"Repository for dashboard {dashboard_id} not found",
)
except HTTPException:
session.rollback()
raise
except Exception as e:
session.rollback()
logger.error(f"[delete_repo][Coherence:Failed] Failed to delete repository for dashboard {dashboard_id}: {e}")
raise HTTPException(status_code=500, detail=f"Failed to delete repository: {str(e)}")
finally:
session.close()
# [/DEF:delete_repo:Function]
# [DEF:get_repo:Function]
# @PURPOSE: Get Repo object for a dashboard.
# @PRE: Repository must exist on disk for the given dashboard_id.
# @POST: Returns a GitPython Repo instance for the dashboard.
# @RETURN: Repo
def get_repo(self, dashboard_id: int) -> Repo:
with belief_scope("GitService.get_repo"):
repo_path = self._get_repo_path(dashboard_id)
if not os.path.exists(repo_path):
logger.error(f"[get_repo][Coherence:Failed] Repository for dashboard {dashboard_id} does not exist")
raise HTTPException(status_code=404, detail=f"Repository for dashboard {dashboard_id} not found")
try:
return Repo(repo_path)
except Exception as e:
logger.error(f"[get_repo][Coherence:Failed] Failed to open repository at {repo_path}: {e}")
raise HTTPException(status_code=500, detail="Failed to open local Git repository")
# [/DEF:get_repo:Function]
# [DEF:configure_identity:Function]
# @PURPOSE: Configure repository-local Git committer identity for user-scoped operations.
# @PRE: dashboard_id repository exists; git_username/git_email may be empty.
# @POST: Repository config has user.name and user.email when both identity values are provided.
def configure_identity(self, dashboard_id: int, git_username: Optional[str], git_email: Optional[str]) -> None:
with belief_scope("GitService.configure_identity"):
normalized_username = str(git_username or "").strip()
normalized_email = str(git_email or "").strip()
if not normalized_username or not normalized_email:
return
repo = self.get_repo(dashboard_id)
try:
with repo.config_writer(config_level="repository") as config_writer:
config_writer.set_value("user", "name", normalized_username)
config_writer.set_value("user", "email", normalized_email)
logger.info("[configure_identity][Action] Applied repository-local git identity for dashboard %s", dashboard_id)
except Exception as e:
logger.error(f"[configure_identity][Coherence:Failed] Failed to configure git identity: {e}")
raise HTTPException(status_code=500, detail=f"Failed to configure git identity: {str(e)}")
# [/DEF:configure_identity:Function]
# [/DEF:GitServiceBase:Class]
# [/DEF:GitServiceBase:Module]

192
backend/src/services/git/_branch.py Normal file
View File

@@ -0,0 +1,192 @@
# [DEF:GitServiceBranchMixin:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: git, branches, commits, checkout, gitflow
# @PURPOSE: Branch and commit operations for GitService — gitflow branches, list/create/checkout branches, commit changes.
# @RELATION: USED_BY -> [GitService]
import os
from typing import Any, Dict, List, Optional
from datetime import datetime
from git import Repo
from git.exc import GitCommandError
from fastapi import HTTPException
from src.core.logger import logger, belief_scope
# [DEF:GitServiceBranchMixin:Class]
# @COMPLEXITY: 3
# @PURPOSE: Mixin providing branch and commit operations for GitService.
class GitServiceBranchMixin:
# [DEF:_ensure_gitflow_branches:Function]
# @PURPOSE: Ensure standard GitFlow branches (main/dev/preprod) exist locally and on origin.
# @PRE: repo is a valid GitPython Repo instance.
# @POST: main, dev, preprod are available in local repository and pushed to origin when available.
def _ensure_gitflow_branches(self, repo: Repo, dashboard_id: int) -> None:
with belief_scope("GitService._ensure_gitflow_branches"):
required_branches = ["main", "dev", "preprod"]
local_heads = {head.name: head for head in getattr(repo, "heads", [])}
base_commit = None
try:
base_commit = repo.head.commit
except Exception:
base_commit = None
if "main" in local_heads:
base_commit = local_heads["main"].commit
if base_commit is None:
logger.warning(
f"[_ensure_gitflow_branches][Action] Skipping branch bootstrap for dashboard {dashboard_id}: repository has no commits"
)
return
if "main" not in local_heads:
local_heads["main"] = repo.create_head("main", base_commit)
logger.info(f"[_ensure_gitflow_branches][Action] Created local branch main for dashboard {dashboard_id}")
for branch_name in ("dev", "preprod"):
if branch_name in local_heads:
continue
local_heads[branch_name] = repo.create_head(branch_name, local_heads["main"].commit)
logger.info(
f"[_ensure_gitflow_branches][Action] Created local branch {branch_name} for dashboard {dashboard_id}"
)
try:
origin = repo.remote(name="origin")
except ValueError:
logger.info(
f"[_ensure_gitflow_branches][Action] Remote origin is not configured for dashboard {dashboard_id}; skipping remote branch creation"
)
return
remote_branch_names = set()
try:
origin.fetch()
for ref in origin.refs:
remote_head = getattr(ref, "remote_head", None)
if remote_head:
remote_branch_names.add(str(remote_head))
except Exception as e:
logger.warning(f"[_ensure_gitflow_branches][Action] Failed to fetch origin refs: {e}")
for branch_name in required_branches:
if branch_name in remote_branch_names:
continue
try:
origin.push(refspec=f"{branch_name}:{branch_name}")
logger.info(
f"[_ensure_gitflow_branches][Action] Pushed branch {branch_name} to origin for dashboard {dashboard_id}"
)
except Exception as e:
logger.error(f"[_ensure_gitflow_branches][Coherence:Failed] Failed to push branch {branch_name} to origin: {e}")
raise HTTPException(
status_code=500,
detail=f"Failed to create default branch '{branch_name}' on remote: {str(e)}",
)
try:
repo.git.checkout("dev")
logger.info(f"[_ensure_gitflow_branches][Action] Checked out default branch dev for dashboard {dashboard_id}")
except Exception as e:
logger.warning(f"[_ensure_gitflow_branches][Action] Could not checkout dev branch for dashboard {dashboard_id}: {e}")
# [/DEF:_ensure_gitflow_branches:Function]
# [DEF:list_branches:Function]
# @PURPOSE: List all branches for a dashboard's repository.
# @PRE: Repository for dashboard_id exists.
# @POST: Returns a list of branch metadata dictionaries.
# @RETURN: List[dict]
def list_branches(self, dashboard_id: int) -> List[dict]:
with belief_scope("GitService.list_branches"):
repo = self.get_repo(dashboard_id)
logger.info(f"[list_branches][Action] Listing branches for {dashboard_id}. Refs: {repo.refs}")
branches = []
for ref in repo.refs:
try:
name = ref.name.replace('refs/heads/', '').replace('refs/remotes/origin/', '')
if any(b['name'] == name for b in branches):
continue
branches.append({
"name": name,
"commit_hash": ref.commit.hexsha if hasattr(ref, 'commit') else "0000000",
"is_remote": ref.is_remote() if hasattr(ref, 'is_remote') else False,
"last_updated": datetime.fromtimestamp(ref.commit.committed_date) if hasattr(ref, 'commit') else datetime.utcnow()
})
except Exception as e:
logger.warning(f"[list_branches][Action] Skipping ref {ref}: {e}")
try:
active_name = repo.active_branch.name
if not any(b['name'] == active_name for b in branches):
branches.append({
"name": active_name, "commit_hash": "0000000",
"is_remote": False, "last_updated": datetime.utcnow()
})
except Exception as e:
logger.warning(f"[list_branches][Action] Could not determine active branch: {e}")
if not branches:
branches.append({
"name": "dev", "commit_hash": "0000000",
"is_remote": False, "last_updated": datetime.utcnow()
})
return branches
# [/DEF:list_branches:Function]
# [DEF:create_branch:Function]
# @PURPOSE: Create a new branch from an existing one.
# @PARAM: name (str) - New branch name.
# @PARAM: from_branch (str) - Source branch.
# @PRE: Repository exists; name is valid; from_branch exists or repo is empty.
# @POST: A new branch is created in the repository.
def create_branch(self, dashboard_id: int, name: str, from_branch: str = "main"):
with belief_scope("GitService.create_branch"):
repo = self.get_repo(dashboard_id)
logger.info(f"[create_branch][Action] Creating branch {name} from {from_branch}")
if not repo.heads and not repo.remotes:
logger.warning("[create_branch][Action] Repository is empty. Creating initial commit to enable branching.")
readme_path = os.path.join(repo.working_dir, "README.md")
if not os.path.exists(readme_path):
with open(readme_path, "w") as f:
f.write(f"# Dashboard {dashboard_id}\nGit repository for Superset dashboard integration.")
repo.index.add(["README.md"])
repo.index.commit("Initial commit")
try:
repo.commit(from_branch)
except Exception:
logger.warning(f"[create_branch][Action] Source branch {from_branch} not found, using HEAD")
from_branch = repo.head
try:
new_branch = repo.create_head(name, from_branch)
return new_branch
except Exception as e:
logger.error(f"[create_branch][Coherence:Failed] {e}")
raise
# [/DEF:create_branch:Function]
# [DEF:checkout_branch:Function]
# @PURPOSE: Switch to a specific branch.
# @PRE: Repository exists and the specified branch name exists.
# @POST: The repository working directory is updated to the specified branch.
def checkout_branch(self, dashboard_id: int, name: str):
with belief_scope("GitService.checkout_branch"):
repo = self.get_repo(dashboard_id)
logger.info(f"[checkout_branch][Action] Checking out branch {name}")
repo.git.checkout(name)
# [/DEF:checkout_branch:Function]
# [DEF:commit_changes:Function]
# @PURPOSE: Stage and commit changes.
# @PARAM: message (str) - Commit message.
# @PARAM: files (List[str]) - Optional list of specific files to stage.
# @PRE: Repository exists and has changes (dirty) or files are specified.
# @POST: Changes are staged and a new commit is created.
def commit_changes(self, dashboard_id: int, message: str, files: List[str] = None):
with belief_scope("GitService.commit_changes"):
repo = self.get_repo(dashboard_id)
if not repo.is_dirty(untracked_files=True) and not files:
logger.info(f"[commit_changes][Action] No changes to commit for dashboard {dashboard_id}")
return
if files:
logger.info(f"[commit_changes][Action] Staging files: {files}")
repo.index.add(files)
else:
logger.info("[commit_changes][Action] Staging all changes")
repo.git.add(A=True)
repo.index.commit(message)
logger.info(f"[commit_changes][Coherence:OK] Committed changes with message: {message}")
# [/DEF:commit_changes:Function]
# [/DEF:GitServiceBranchMixin:Class]
# [/DEF:GitServiceBranchMixin:Module]

262
backend/src/services/git/_gitea.py Normal file
View File

@@ -0,0 +1,262 @@
# [DEF:GitServiceGiteaMixin:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: git, gitea, api, provider, connection_test
# @PURPOSE: Gitea API operations for GitService — connection testing, repository CRUD, and pull request creation.
# @RELATION: USED_BY -> [GitService]
import httpx
from typing import Any, Dict, List, Optional
from urllib.parse import quote
from fastapi import HTTPException
from src.core.logger import logger, belief_scope
from src.models.git import GitProvider
# [DEF:GitServiceGiteaMixin:Class]
# @COMPLEXITY: 3
# @PURPOSE: Mixin providing Gitea API operations for GitService.
class GitServiceGiteaMixin:
# [DEF:test_connection:Function]
# @PURPOSE: Test connection to Git provider using PAT.
# @PARAM: provider (GitProvider), url (str), pat (str)
# @PRE: provider is valid; url is a valid HTTP(S) URL; pat is provided.
# @POST: Returns True if connection to the provider's API succeeds.
# @RETURN: bool
async def test_connection(self, provider: GitProvider, url: str, pat: str) -> bool:
with belief_scope("GitService.test_connection"):
if ".local" in url or "localhost" in url:
logger.info("[test_connection][Action] Local/Offline mode detected for URL")
return True
if not url.startswith(('http://', 'https://')):
logger.error(f"[test_connection][Coherence:Failed] Invalid URL protocol: {url}")
return False
if not pat or not pat.strip():
logger.error("[test_connection][Coherence:Failed] Git PAT is missing or empty")
return False
pat = pat.strip()
try:
async with httpx.AsyncClient() as client:
if provider == GitProvider.GITHUB:
headers = {"Authorization": f"token {pat}"}
api_url = "https://api.github.com/user" if "github.com" in url else f"{url.rstrip('/')}/api/v3/user"
resp = await client.get(api_url, headers=headers)
elif provider == GitProvider.GITLAB:
headers = {"PRIVATE-TOKEN": pat}
api_url = f"{url.rstrip('/')}/api/v4/user"
resp = await client.get(api_url, headers=headers)
elif provider == GitProvider.GITEA:
headers = {"Authorization": f"token {pat}"}
api_url = f"{url.rstrip('/')}/api/v1/user"
resp = await client.get(api_url, headers=headers)
else:
return False
if resp.status_code != 200:
logger.error(f"[test_connection][Coherence:Failed] Git connection test failed for {provider} at {api_url}. Status: {resp.status_code}")
return resp.status_code == 200
except Exception as e:
logger.error(f"[test_connection][Coherence:Failed] Error testing git connection: {e}")
return False
# [/DEF:test_connection:Function]
# [DEF:_gitea_headers:Function]
# @PURPOSE: Build Gitea API authorization headers.
# @PRE: pat is provided.
# @POST: Returns headers with token auth.
# @RETURN: Dict[str, str]
def _gitea_headers(self, pat: str) -> Dict[str, str]:
token = (pat or "").strip()
if not token:
raise HTTPException(status_code=400, detail="Git PAT is required for Gitea operations")
return {
"Authorization": f"token {token}",
"Content-Type": "application/json",
"Accept": "application/json",
}
# [/DEF:_gitea_headers:Function]
# [DEF:_gitea_request:Function]
# @PURPOSE: Execute HTTP request against Gitea API with stable error mapping.
# @PRE: method and endpoint are valid.
# @POST: Returns decoded JSON payload.
# @RETURN: Any
async def _gitea_request(
self, method: str, server_url: str, pat: str, endpoint: str,
payload: Optional[Dict[str, Any]] = None,
) -> Any:
base_url = self._normalize_git_server_url(server_url)
url = f"{base_url}/api/v1{endpoint}"
headers = self._gitea_headers(pat)
try:
async with httpx.AsyncClient(timeout=20.0) as client:
response = await client.request(method=method, url=url, headers=headers, json=payload)
except Exception as e:
logger.error(f"[gitea_request][Coherence:Failed] Network error: {e}")
raise HTTPException(status_code=503, detail=f"Gitea API is unavailable: {str(e)}")
if response.status_code >= 400:
detail = response.text
try:
parsed = response.json()
detail = parsed.get("message") or parsed.get("error") or detail
except Exception:
pass
logger.error(f"[gitea_request][Coherence:Failed] method={method} endpoint={endpoint} status={response.status_code} detail={detail}")
raise HTTPException(status_code=response.status_code, detail=f"Gitea API error: {detail}")
if response.status_code == 204:
return None
return response.json()
# [/DEF:_gitea_request:Function]
# [DEF:get_gitea_current_user:Function]
# @PURPOSE: Resolve current Gitea user for PAT.
# @PRE: server_url and pat are valid.
# @POST: Returns current username.
# @RETURN: str
async def get_gitea_current_user(self, server_url: str, pat: str) -> str:
payload = await self._gitea_request("GET", server_url, pat, "/user")
username = payload.get("login") or payload.get("username")
if not username:
raise HTTPException(status_code=500, detail="Failed to resolve Gitea username")
return str(username)
# [/DEF:get_gitea_current_user:Function]
# [DEF:list_gitea_repositories:Function]
# @PURPOSE: List repositories visible to authenticated Gitea user.
# @PRE: server_url and pat are valid.
# @POST: Returns repository list from Gitea.
# @RETURN: List[dict]
async def list_gitea_repositories(self, server_url: str, pat: str) -> List[dict]:
payload = await self._gitea_request("GET", server_url, pat, "/user/repos?limit=100&page=1")
if not isinstance(payload, list):
return []
return payload
# [/DEF:list_gitea_repositories:Function]
# [DEF:create_gitea_repository:Function]
# @PURPOSE: Create repository in Gitea for authenticated user.
# @PRE: name is non-empty and PAT has repo creation permission.
# @POST: Returns created repository payload.
# @RETURN: dict
async def create_gitea_repository(
self, server_url: str, pat: str, name: str, private: bool = True,
description: Optional[str] = None, auto_init: bool = True, default_branch: Optional[str] = "main",
) -> Dict[str, Any]:
payload: Dict[str, Any] = {"name": name, "private": bool(private), "auto_init": bool(auto_init)}
if description:
payload["description"] = description
if default_branch:
payload["default_branch"] = default_branch
created = await self._gitea_request("POST", server_url, pat, "/user/repos", payload=payload)
if not isinstance(created, dict):
raise HTTPException(status_code=500, detail="Unexpected Gitea response while creating repository")
return created
# [/DEF:create_gitea_repository:Function]
# [DEF:delete_gitea_repository:Function]
# @PURPOSE: Delete repository in Gitea.
# @PRE: owner and repo_name are non-empty.
# @POST: Repository deleted on Gitea server.
async def delete_gitea_repository(self, server_url: str, pat: str, owner: str, repo_name: str) -> None:
if not owner or not repo_name:
raise HTTPException(status_code=400, detail="owner and repo_name are required")
await self._gitea_request("DELETE", server_url, pat, f"/repos/{owner}/{repo_name}")
# [/DEF:delete_gitea_repository:Function]
# [DEF:_gitea_branch_exists:Function]
# @PURPOSE: Check whether a branch exists in Gitea repository.
# @PRE: owner/repo/branch are non-empty.
# @POST: Returns True when branch exists, False when 404.
# @RETURN: bool
async def _gitea_branch_exists(self, server_url: str, pat: str, owner: str, repo: str, branch: str) -> bool:
if not owner or not repo or not branch:
return False
endpoint = f"/repos/{owner}/{repo}/branches/{quote(branch, safe='')}"
try:
await self._gitea_request("GET", server_url, pat, endpoint)
return True
except HTTPException as exc:
if exc.status_code == 404:
return False
raise
# [/DEF:_gitea_branch_exists:Function]
# [DEF:_build_gitea_pr_404_detail:Function]
# @PURPOSE: Build actionable error detail for Gitea PR 404 responses.
# @PRE: owner/repo/from_branch/to_branch are provided.
# @POST: Returns specific branch-missing message when detected.
# @RETURN: Optional[str]
async def _build_gitea_pr_404_detail(
self, server_url: str, pat: str, owner: str, repo: str, from_branch: str, to_branch: str,
) -> Optional[str]:
source_exists = await self._gitea_branch_exists(
server_url=server_url, pat=pat, owner=owner, repo=repo, branch=from_branch,
)
target_exists = await self._gitea_branch_exists(
server_url=server_url, pat=pat, owner=owner, repo=repo, branch=to_branch,
)
if not source_exists:
return f"Gitea branch not found: source branch '{from_branch}' in {owner}/{repo}"
if not target_exists:
return f"Gitea branch not found: target branch '{to_branch}' in {owner}/{repo}"
return None
# [/DEF:_build_gitea_pr_404_detail:Function]
# [DEF:create_gitea_pull_request:Function]
# @PURPOSE: Create pull request in Gitea.
# @PRE: Config and remote URL are valid.
# @POST: Returns normalized PR metadata.
# @RETURN: Dict[str, Any]
async def create_gitea_pull_request(
self, server_url: str, pat: str, remote_url: str, from_branch: str, to_branch: str,
title: str, description: Optional[str] = None,
) -> Dict[str, Any]:
identity = self._parse_remote_repo_identity(remote_url)
req_payload = {"title": title, "head": from_branch, "base": to_branch, "body": description or ""}
endpoint = f"/repos/{identity['owner']}/{identity['repo']}/pulls"
active_server_url = server_url
try:
data = await self._gitea_request("POST", active_server_url, pat, endpoint, payload=req_payload)
except HTTPException as exc:
fallback_url = self._derive_server_url_from_remote(remote_url)
normalized_primary = self._normalize_git_server_url(server_url)
should_retry_with_fallback = (
exc.status_code == 404 and fallback_url and fallback_url != normalized_primary
)
if should_retry_with_fallback:
logger.warning(
"[create_gitea_pull_request][Action] Primary Gitea URL not found, retrying with remote host: %s",
fallback_url,
)
active_server_url = fallback_url
try:
data = await self._gitea_request("POST", active_server_url, pat, endpoint, payload=req_payload)
except HTTPException as retry_exc:
if retry_exc.status_code == 404:
branch_detail = await self._build_gitea_pr_404_detail(
server_url=active_server_url, pat=pat,
owner=identity["owner"], repo=identity["repo"],
from_branch=from_branch, to_branch=to_branch,
)
if branch_detail:
raise HTTPException(status_code=400, detail=branch_detail)
raise
else:
if exc.status_code == 404:
branch_detail = await self._build_gitea_pr_404_detail(
server_url=active_server_url, pat=pat,
owner=identity["owner"], repo=identity["repo"],
from_branch=from_branch, to_branch=to_branch,
)
if branch_detail:
raise HTTPException(status_code=400, detail=branch_detail)
raise
if not isinstance(data, dict):
raise HTTPException(status_code=500, detail="Unexpected Gitea response while creating pull request")
return {
"id": data.get("number") or data.get("id"),
"url": data.get("html_url") or data.get("url"),
"status": data.get("state") or "open",
}
# [/DEF:create_gitea_pull_request:Function]
# [/DEF:GitServiceGiteaMixin:Class]
# [/DEF:GitServiceGiteaMixin:Module]

278
backend/src/services/git/_merge.py Normal file
View File

@@ -0,0 +1,278 @@
# [DEF:GitServiceMergeMixin:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: git, merge, conflicts, resolution, promote
# @PURPOSE: Merge operations for GitService — conflict detection, resolution, abort, continue, and direct promote.
# @RELATION: USED_BY -> [GitService]
import os
from pathlib import Path
from typing import Any, Dict, List, Optional
from git import Repo
from git.exc import GitCommandError
from git.objects.blob import Blob
from fastapi import HTTPException
from src.core.logger import logger, belief_scope
# [DEF:GitServiceMergeMixin:Class]
# @COMPLEXITY: 3
# @PURPOSE: Mixin providing merge operations for GitService.
class GitServiceMergeMixin:
# [DEF:_read_blob_text:Function]
# @PURPOSE: Read text from a Git blob.
def _read_blob_text(self, blob: Blob) -> str:
with belief_scope("GitService._read_blob_text"):
if blob is None:
return ""
try:
return blob.data_stream.read().decode("utf-8", errors="replace")
except Exception:
return ""
# [/DEF:_read_blob_text:Function]
# [DEF:_get_unmerged_file_paths:Function]
# @PURPOSE: List files with merge conflicts.
def _get_unmerged_file_paths(self, repo: Repo) -> List[str]:
with belief_scope("GitService._get_unmerged_file_paths"):
try:
return sorted(list(repo.index.unmerged_blobs().keys()))
except Exception:
return []
# [/DEF:_get_unmerged_file_paths:Function]
# [DEF:_build_unfinished_merge_payload:Function]
# @PURPOSE: Build payload for unfinished merge state.
def _build_unfinished_merge_payload(self, repo: Repo) -> Dict[str, Any]:
with belief_scope("GitService._build_unfinished_merge_payload"):
merge_head_path = os.path.join(repo.git_dir, "MERGE_HEAD")
merge_head_value = ""
merge_msg_preview = ""
current_branch = "unknown"
try:
merge_head_value = Path(merge_head_path).read_text(encoding="utf-8").strip()
except Exception:
merge_head_value = "<unreadable>"
try:
merge_msg_path = os.path.join(repo.git_dir, "MERGE_MSG")
if os.path.exists(merge_msg_path):
merge_msg_preview = (
Path(merge_msg_path).read_text(encoding="utf-8").strip().splitlines()[:1] or [""]
)[0]
except Exception:
merge_msg_preview = "<unreadable>"
try:
current_branch = repo.active_branch.name
except Exception:
current_branch = "detached_or_unknown"
conflicts_count = len(self._get_unmerged_file_paths(repo))
return {
"error_code": "GIT_UNFINISHED_MERGE",
"message": (
"\u0412 \u0440\u0435\u043f\u043e\u0437\u0438\u0442\u043e\u0440\u0438\u0438 \u0435\u0441\u0442\u044c \u043d\u0435\u0437\u0430\u0432\u0435\u0440\u0448\u0451\u043d\u043d\u043e\u0435 \u0441\u043b\u0438\u044f\u043d\u0438\u0435. "
"\u0417\u0430\u0432\u0435\u0440\u0448\u0438\u0442\u0435 \u0438\u043b\u0438 \u043e\u0442\u043c\u0435\u043d\u0438\u0442\u0435 \u0441\u043b\u0438\u044f\u043d\u0438\u0435 \u0432\u0440\u0443\u0447\u043d\u0443\u044e."
),
"repository_path": repo.working_tree_dir,
"git_dir": repo.git_dir,
"current_branch": current_branch,
"merge_head": merge_head_value,
"merge_message_preview": merge_msg_preview,
"conflicts_count": conflicts_count,
"next_steps": [
"\u041e\u0442\u043a\u0440\u043e\u0439\u0442\u0435 \u043b\u043e\u043a\u0430\u043b\u044c\u043d\u044b\u0439 \u0440\u0435\u043f\u043e\u0437\u0438\u0442\u043e\u0440\u0438\u0439 \u043f\u043e \u043f\u0443\u0442\u0438 repository_path",
"\u041f\u0440\u043e\u0432\u0435\u0440\u044c\u0442\u0435 \u0441\u043e\u0441\u0442\u043e\u044f\u043d\u0438\u0435: git status",
"\u0420\u0430\u0437\u0440\u0435\u0448\u0438\u0442\u0435 \u043a\u043e\u043d\u0444\u043b\u0438\u043a\u0442\u044b \u0438 \u0432\u044b\u043f\u043e\u043b\u043d\u0438\u0442\u0435 commit, \u043b\u0438\u0431\u043e \u043e\u0442\u043c\u0435\u043d\u0438\u0442\u0435: git merge --abort",
"\u041f\u043e\u0441\u043b\u0435 \u0437\u0430\u0432\u0435\u0440\u0448\u0435\u043d\u0438\u044f/\u043e\u0442\u043c\u0435\u043d\u044b \u0441\u043b\u0438\u044f\u043d\u0438\u044f \u043f\u043e\u0432\u0442\u043e\u0440\u0438\u0442\u0435 Pull \u0438\u0437 \u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441\u0430",
],
"manual_commands": ["git status", "git add <resolved-files>", 'git commit -m "resolve merge conflicts"', "git merge --abort"],
}
# [/DEF:_build_unfinished_merge_payload:Function]
# [DEF:get_merge_status:Function]
# @PURPOSE: Get current merge status for a dashboard repository.
def get_merge_status(self, dashboard_id: int) -> Dict[str, Any]:
with belief_scope("GitService.get_merge_status"):
repo = self.get_repo(dashboard_id)
merge_head_path = os.path.join(repo.git_dir, "MERGE_HEAD")
if not os.path.exists(merge_head_path):
current_branch = "unknown"
try:
current_branch = repo.active_branch.name
except Exception:
current_branch = "detached_or_unknown"
return {
"has_unfinished_merge": False,
"repository_path": repo.working_tree_dir,
"git_dir": repo.git_dir,
"current_branch": current_branch,
"merge_head": None,
"merge_message_preview": None,
"conflicts_count": 0,
}
payload = self._build_unfinished_merge_payload(repo)
return {
"has_unfinished_merge": True,
"repository_path": payload["repository_path"],
"git_dir": payload["git_dir"],
"current_branch": payload["current_branch"],
"merge_head": payload["merge_head"],
"merge_message_preview": payload["merge_message_preview"],
"conflicts_count": int(payload.get("conflicts_count") or 0),
}
# [/DEF:get_merge_status:Function]
# [DEF:get_merge_conflicts:Function]
# @PURPOSE: List all files with conflicts and their contents.
def get_merge_conflicts(self, dashboard_id: int) -> List[Dict[str, Any]]:
with belief_scope("GitService.get_merge_conflicts"):
repo = self.get_repo(dashboard_id)
conflicts = []
unmerged = repo.index.unmerged_blobs()
for file_path, stages in unmerged.items():
mine_blob = None
theirs_blob = None
for stage, blob in stages:
if stage == 2:
mine_blob = blob
elif stage == 3:
theirs_blob = blob
conflicts.append({
"file_path": file_path,
"mine": self._read_blob_text(mine_blob) if mine_blob else "",
"theirs": self._read_blob_text(theirs_blob) if theirs_blob else "",
})
return sorted(conflicts, key=lambda item: item["file_path"])
# [/DEF:get_merge_conflicts:Function]
# [DEF:resolve_merge_conflicts:Function]
# @PURPOSE: Resolve conflicts using specified strategy.
def resolve_merge_conflicts(self, dashboard_id: int, resolutions: List[Dict[str, Any]]) -> List[str]:
with belief_scope("GitService.resolve_merge_conflicts"):
repo = self.get_repo(dashboard_id)
resolved_files: List[str] = []
repo_root = os.path.abspath(str(repo.working_tree_dir or ""))
if not repo_root:
raise HTTPException(status_code=500, detail="Repository working tree directory is unavailable")
for item in resolutions or []:
file_path = str(item.get("file_path") or "").strip()
strategy = str(item.get("resolution") or "").strip().lower()
content = item.get("content")
if not file_path:
raise HTTPException(status_code=400, detail="resolution.file_path is required")
if strategy not in {"mine", "theirs", "manual"}:
raise HTTPException(status_code=400, detail=f"Unsupported resolution strategy: {strategy}")
if strategy == "mine":
repo.git.checkout("--ours", "--", file_path)
elif strategy == "theirs":
repo.git.checkout("--theirs", "--", file_path)
else:
abs_target = os.path.abspath(os.path.join(repo_root, file_path))
if abs_target != repo_root and not abs_target.startswith(repo_root + os.sep):
raise HTTPException(status_code=400, detail=f"Invalid conflict file path: {file_path}")
os.makedirs(os.path.dirname(abs_target), exist_ok=True)
with open(abs_target, "w", encoding="utf-8") as file_obj:
file_obj.write(str(content or ""))
repo.git.add(file_path)
resolved_files.append(file_path)
return resolved_files
# [/DEF:resolve_merge_conflicts:Function]
# [DEF:abort_merge:Function]
# @PURPOSE: Abort ongoing merge.
def abort_merge(self, dashboard_id: int) -> Dict[str, Any]:
with belief_scope("GitService.abort_merge"):
repo = self.get_repo(dashboard_id)
try:
repo.git.merge("--abort")
except GitCommandError as e:
details = str(e)
lowered = details.lower()
if "there is no merge to abort" in lowered or "no merge to abort" in lowered:
return {"status": "no_merge_in_progress"}
raise HTTPException(status_code=409, detail=f"Cannot abort merge: {details}")
return {"status": "aborted"}
# [/DEF:abort_merge:Function]
# [DEF:continue_merge:Function]
# @PURPOSE: Finalize merge after conflict resolution.
def continue_merge(self, dashboard_id: int, message: Optional[str] = None) -> Dict[str, Any]:
with belief_scope("GitService.continue_merge"):
repo = self.get_repo(dashboard_id)
unmerged_files = self._get_unmerged_file_paths(repo)
if unmerged_files:
raise HTTPException(
status_code=409,
detail={
"error_code": "GIT_MERGE_CONFLICTS_REMAIN",
"message": "\u041d\u0435\u0432\u043e\u0437\u043c\u043e\u0436\u043d\u043e \u0437\u0430\u0432\u0435\u0440\u0448\u0438\u0442\u044c merge: \u043e\u0441\u0442\u0430\u043b\u0438\u0441\u044c \u043d\u0435\u0440\u0430\u0437\u0440\u0435\u0448\u0451\u043d\u043d\u044b\u0435 \u043a\u043e\u043d\u0444\u043b\u0438\u043a\u0442\u044b.",
"unresolved_files": unmerged_files,
},
)
try:
normalized_message = str(message or "").strip()
if normalized_message:
repo.git.commit("-m", normalized_message)
else:
repo.git.commit("--no-edit")
except GitCommandError as e:
details = str(e)
lowered = details.lower()
if "nothing to commit" in lowered:
return {"status": "already_clean"}
raise HTTPException(status_code=409, detail=f"Cannot continue merge: {details}")
commit_hash = ""
try:
commit_hash = repo.head.commit.hexsha
except Exception:
commit_hash = ""
return {"status": "committed", "commit_hash": commit_hash}
# [/DEF:continue_merge:Function]
# [DEF:promote_direct_merge:Function]
# @PURPOSE: Perform direct merge between branches in local repo and push target branch.
# @PRE: Repository exists and both branches are valid.
# @POST: Target branch contains merged changes from source branch.
# @RETURN: Dict[str, Any]
def promote_direct_merge(self, dashboard_id: int, from_branch: str, to_branch: str) -> Dict[str, Any]:
with belief_scope("GitService.promote_direct_merge"):
if not from_branch or not to_branch:
raise HTTPException(status_code=400, detail="from_branch and to_branch are required")
repo = self.get_repo(dashboard_id)
source = from_branch.strip()
target = to_branch.strip()
if source == target:
raise HTTPException(status_code=400, detail="from_branch and to_branch must be different")
try:
origin = repo.remote(name="origin")
except ValueError:
raise HTTPException(status_code=400, detail="Remote 'origin' not configured")
try:
origin.fetch()
if source not in [head.name for head in repo.heads]:
if f"origin/{source}" in [ref.name for ref in repo.refs]:
repo.git.checkout("-b", source, f"origin/{source}")
else:
raise HTTPException(status_code=404, detail=f"Source branch '{source}' not found")
if target in [head.name for head in repo.heads]:
repo.git.checkout(target)
elif f"origin/{target}" in [ref.name for ref in repo.refs]:
repo.git.checkout("-b", target, f"origin/{target}")
else:
raise HTTPException(status_code=404, detail=f"Target branch '{target}' not found")
try:
origin.pull(target)
except Exception:
pass
repo.git.merge(source, "--no-ff", "-m", f"chore(flow): promote {source} -> {target}")
origin.push(refspec=f"{target}:{target}")
except HTTPException:
raise
except Exception as e:
message = str(e)
if "CONFLICT" in message.upper():
raise HTTPException(status_code=409, detail=f"Merge conflict during direct promote: {message}")
raise HTTPException(status_code=500, detail=f"Direct promote failed: {message}")
return {"mode": "direct", "from_branch": source, "to_branch": target, "status": "merged"}
# [/DEF:promote_direct_merge:Function]
# [/DEF:GitServiceMergeMixin:Class]
# [/DEF:GitServiceMergeMixin:Module]

186
backend/src/services/git/_remote_providers.py Normal file
View File

@@ -0,0 +1,186 @@
# [DEF:GitServiceRemoteMixin:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: git, github, gitlab, api, provider, repository, pull_request, merge_request
# @PURPOSE: GitHub and GitLab provider operations for GitService — repository creation and PR/MR creation.
# @RELATION: USED_BY -> [GitService]
import httpx
from typing import Any, Dict, Optional
from urllib.parse import quote
from fastapi import HTTPException
from src.core.logger import logger, belief_scope
# [DEF:GitServiceGithubMixin:Class]
# @COMPLEXITY: 3
# @PURPOSE: Mixin providing GitHub API operations for GitService.
class GitServiceGithubMixin:
# [DEF:create_github_repository:Function]
# @PURPOSE: Create repository in GitHub or GitHub Enterprise.
# @PRE: PAT has repository create permission.
# @POST: Returns created repository payload.
# @RETURN: dict
async def create_github_repository(
self, server_url: str, pat: str, name: str, private: bool = True,
description: Optional[str] = None, auto_init: bool = True, default_branch: Optional[str] = "main",
) -> Dict[str, Any]:
base_url = self._normalize_git_server_url(server_url)
if "github.com" in base_url:
api_url = "https://api.github.com/user/repos"
else:
api_url = f"{base_url}/api/v3/user/repos"
headers = {
"Authorization": f"token {pat.strip()}",
"Content-Type": "application/json",
"Accept": "application/vnd.github+json",
}
payload: Dict[str, Any] = {"name": name, "private": bool(private), "auto_init": bool(auto_init)}
if description:
payload["description"] = description
if default_branch:
payload["default_branch"] = default_branch
try:
async with httpx.AsyncClient(timeout=20.0) as client:
response = await client.post(api_url, headers=headers, json=payload)
except Exception as e:
raise HTTPException(status_code=503, detail=f"GitHub API is unavailable: {str(e)}")
if response.status_code >= 400:
detail = response.text
try:
parsed = response.json()
detail = parsed.get("message") or detail
except Exception:
pass
raise HTTPException(status_code=response.status_code, detail=f"GitHub API error: {detail}")
return response.json()
# [/DEF:create_github_repository:Function]
# [DEF:create_github_pull_request:Function]
# @PURPOSE: Create pull request in GitHub or GitHub Enterprise.
# @PRE: Config and remote URL are valid.
# @POST: Returns normalized PR metadata.
# @RETURN: Dict[str, Any]
async def create_github_pull_request(
self, server_url: str, pat: str, remote_url: str, from_branch: str, to_branch: str,
title: str, description: Optional[str] = None, draft: bool = False,
) -> Dict[str, Any]:
identity = self._parse_remote_repo_identity(remote_url)
base_url = self._normalize_git_server_url(server_url)
if "github.com" in base_url:
api_url = f"https://api.github.com/repos/{identity['namespace']}/{identity['repo']}/pulls"
else:
api_url = f"{base_url}/api/v3/repos/{identity['namespace']}/{identity['repo']}/pulls"
headers = {
"Authorization": f"token {pat.strip()}",
"Content-Type": "application/json",
"Accept": "application/vnd.github+json",
}
payload = {
"title": title, "head": from_branch, "base": to_branch,
"body": description or "", "draft": bool(draft),
}
try:
async with httpx.AsyncClient(timeout=20.0) as client:
response = await client.post(api_url, headers=headers, json=payload)
except Exception as e:
raise HTTPException(status_code=503, detail=f"GitHub API is unavailable: {str(e)}")
if response.status_code >= 400:
detail = response.text
try:
detail = response.json().get("message") or detail
except Exception:
pass
raise HTTPException(status_code=response.status_code, detail=f"GitHub API error: {detail}")
data = response.json()
return {"id": data.get("number") or data.get("id"), "url": data.get("html_url") or data.get("url"), "status": data.get("state") or "open"}
# [/DEF:create_github_pull_request:Function]
# [DEF:GitServiceGitlabMixin:Class]
# @COMPLEXITY: 3
# @PURPOSE: Mixin providing GitLab API operations for GitService.
class GitServiceGitlabMixin:
# [DEF:create_gitlab_repository:Function]
# @PURPOSE: Create repository(project) in GitLab.
# @PRE: PAT has api scope.
# @POST: Returns created repository payload.
# @RETURN: dict
async def create_gitlab_repository(
self, server_url: str, pat: str, name: str, private: bool = True,
description: Optional[str] = None, auto_init: bool = True, default_branch: Optional[str] = "main",
) -> Dict[str, Any]:
base_url = self._normalize_git_server_url(server_url)
api_url = f"{base_url}/api/v4/projects"
headers = {"PRIVATE-TOKEN": pat.strip(), "Content-Type": "application/json", "Accept": "application/json"}
payload: Dict[str, Any] = {
"name": name, "visibility": "private" if private else "public",
"initialize_with_readme": bool(auto_init),
}
if description:
payload["description"] = description
if default_branch:
payload["default_branch"] = default_branch
try:
async with httpx.AsyncClient(timeout=20.0) as client:
response = await client.post(api_url, headers=headers, json=payload)
except Exception as e:
raise HTTPException(status_code=503, detail=f"GitLab API is unavailable: {str(e)}")
if response.status_code >= 400:
detail = response.text
try:
parsed = response.json()
if isinstance(parsed, dict):
detail = parsed.get("message") or detail
except Exception:
pass
raise HTTPException(status_code=response.status_code, detail=f"GitLab API error: {detail}")
data = response.json()
if "clone_url" not in data:
data["clone_url"] = data.get("http_url_to_repo")
if "html_url" not in data:
data["html_url"] = data.get("web_url")
if "ssh_url" not in data:
data["ssh_url"] = data.get("ssh_url_to_repo")
if "full_name" not in data:
data["full_name"] = data.get("path_with_namespace") or data.get("name")
return data
# [/DEF:create_gitlab_repository:Function]
# [DEF:create_gitlab_merge_request:Function]
# @PURPOSE: Create merge request in GitLab.
# @PRE: Config and remote URL are valid.
# @POST: Returns normalized MR metadata.
# @RETURN: Dict[str, Any]
async def create_gitlab_merge_request(
self, server_url: str, pat: str, remote_url: str, from_branch: str, to_branch: str,
title: str, description: Optional[str] = None, remove_source_branch: bool = False,
) -> Dict[str, Any]:
identity = self._parse_remote_repo_identity(remote_url)
base_url = self._normalize_git_server_url(server_url)
project_id = quote(identity["full_name"], safe="")
api_url = f"{base_url}/api/v4/projects/{project_id}/merge_requests"
headers = {"PRIVATE-TOKEN": pat.strip(), "Content-Type": "application/json", "Accept": "application/json"}
payload = {
"source_branch": from_branch, "target_branch": to_branch, "title": title,
"description": description or "", "remove_source_branch": bool(remove_source_branch),
}
try:
async with httpx.AsyncClient(timeout=20.0) as client:
response = await client.post(api_url, headers=headers, json=payload)
except Exception as e:
raise HTTPException(status_code=503, detail=f"GitLab API is unavailable: {str(e)}")
if response.status_code >= 400:
detail = response.text
try:
parsed = response.json()
if isinstance(parsed, dict):
detail = parsed.get("message") or detail
except Exception:
pass
raise HTTPException(status_code=response.status_code, detail=f"GitLab API error: {detail}")
data = response.json()
return {"id": data.get("iid") or data.get("id"), "url": data.get("web_url") or data.get("url"), "status": data.get("state") or "opened"}
# [/DEF:create_gitlab_merge_request:Function]
# [/DEF:GitServiceGitlabMixin:Class]
# [/DEF:GitServiceRemoteMixin:Module]

164
backend/src/services/git/_status.py Normal file
View File

@@ -0,0 +1,164 @@
# [DEF:GitServiceStatusMixin:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: git, status, diff, history, porcelain
# @PURPOSE: Status, diff, and commit history operations for GitService using git status --porcelain to avoid --cached flag issues.
# @RELATION: USED_BY -> [GitService]
from typing import Any, Dict, List
from datetime import datetime
from git import Repo
from src.core.logger import logger, belief_scope
# [DEF:GitServiceStatusMixin:Class]
# @COMPLEXITY: 3
# @PURPOSE: Mixin providing repository status, diff, and commit history for GitService.
class GitServiceStatusMixin:
# [DEF:_parse_status_porcelain:Function]
# @COMPLEXITY: 2
# @PURPOSE: Parse git status --porcelain output into staged, modified, and untracked file lists.
# @PRE: `repo` is an open GitPython Repo instance.
# @POST: Returns (staged, modified, untracked) tuple of file path lists.
# @RATIONALE: Avoids repo.is_dirty() / repo.index.diff("HEAD") which internally
# call git diff --cached, a flag unsupported in some Git environments (exit 129).
# Using git status --porcelain is self-contained and avoids the --cached flag entirely.
def _parse_status_porcelain(self, repo) -> tuple[list[str], list[str], list[str]]:
with belief_scope("GitService._parse_status_porcelain"):
staged: list[str] = []
modified: list[str] = []
untracked: list[str] = []
try:
output = repo.git.status("--porcelain")
except Exception:
logger.warning("[_parse_status_porcelain][Coherence:Failed] git status --porcelain failed")
return staged, modified, untracked
for line in output.split("\n"):
if not line:
continue
if line.startswith("??"):
untracked.append(line[2:].strip())
continue
if line.startswith("!!"):
continue
if len(line) < 3:
continue
x = line[0]
y = line[1]
rest = line[3:]
path = rest.split(" -> ")[-1].strip() if " -> " in rest else rest.strip()
if x != " " and x != "?":
staged.append(path)
if y != " " and y != "?":
if path not in modified:
modified.append(path)
return staged, modified, untracked
# [/DEF:_parse_status_porcelain:Function]
# [DEF:get_status:Function]
# @PURPOSE: Get current repository status (dirty files, untracked, etc.)
# @PRE: Repository for dashboard_id exists.
# @POST: Returns a dictionary representing the Git status.
# @RETURN: dict
def get_status(self, dashboard_id: int) -> dict:
with belief_scope("GitService.get_status"):
repo = self.get_repo(dashboard_id)
has_commits = False
try:
repo.head.commit
has_commits = True
except (ValueError, Exception):
has_commits = False
current_branch = repo.active_branch.name
tracking_branch = None
has_upstream = False
ahead_count = 0
behind_count = 0
try:
tracking_branch = repo.active_branch.tracking_branch()
has_upstream = tracking_branch is not None
except Exception:
tracking_branch = None
has_upstream = False
if has_upstream and tracking_branch is not None:
try:
ahead_count = sum(1 for _ in repo.iter_commits(f"{tracking_branch.name}..{current_branch}"))
behind_count = sum(1 for _ in repo.iter_commits(f"{current_branch}..{tracking_branch.name}"))
except Exception:
ahead_count = 0
behind_count = 0
staged_files, modified_files, untracked_files = self._parse_status_porcelain(repo)
is_dirty = bool(staged_files or modified_files or untracked_files)
is_diverged = ahead_count > 0 and behind_count > 0
if is_diverged:
sync_state = "DIVERGED"
elif behind_count > 0:
sync_state = "BEHIND_REMOTE"
elif ahead_count > 0:
sync_state = "AHEAD_REMOTE"
elif is_dirty or modified_files or staged_files or untracked_files:
sync_state = "CHANGES"
else:
sync_state = "SYNCED"
return {
"is_dirty": is_dirty,
"untracked_files": untracked_files,
"modified_files": modified_files,
"staged_files": staged_files,
"current_branch": current_branch,
"upstream_branch": tracking_branch.name if tracking_branch is not None else None,
"has_upstream": has_upstream,
"ahead_count": ahead_count,
"behind_count": behind_count,
"is_diverged": is_diverged,
"sync_state": sync_state,
}
# [/DEF:get_status:Function]
# [DEF:get_diff:Function]
# @PURPOSE: Generate diff for a file or the whole repository.
# @PARAM: file_path (str) - Optional specific file.
# @PARAM: staged (bool) - Whether to show staged changes.
# @PRE: Repository for dashboard_id exists.
# @POST: Returns the diff text as a string.
# @RETURN: str
def get_diff(self, dashboard_id: int, file_path: str = None, staged: bool = False) -> str:
with belief_scope("GitService.get_diff"):
repo = self.get_repo(dashboard_id)
diff_args = []
if staged:
diff_args.append("--staged")
if file_path:
return repo.git.diff(*diff_args, "--", file_path)
return repo.git.diff(*diff_args)
# [/DEF:get_diff:Function]
# [DEF:get_commit_history:Function]
# @PURPOSE: Retrieve commit history for a repository.
# @PARAM: limit (int) - Max number of commits to return.
# @PRE: Repository for dashboard_id exists.
# @POST: Returns a list of dictionaries for each commit in history.
# @RETURN: List[dict]
def get_commit_history(self, dashboard_id: int, limit: int = 50) -> List[dict]:
with belief_scope("GitService.get_commit_history"):
repo = self.get_repo(dashboard_id)
commits = []
try:
if not repo.heads and not repo.remotes:
return []
for commit in repo.iter_commits(max_count=limit):
commits.append({
"hash": commit.hexsha,
"author": commit.author.name,
"email": commit.author.email,
"timestamp": datetime.fromtimestamp(commit.committed_date),
"message": commit.message.strip(),
"files_changed": list(commit.stats.files.keys())
})
except Exception as e:
logger.warning(f"[get_commit_history][Action] Could not retrieve commit history for dashboard {dashboard_id}: {e}")
return []
return commits
# [/DEF:get_commit_history:Function]
# [/DEF:GitServiceStatusMixin:Class]
# [/DEF:GitServiceStatusMixin:Module]

177
backend/src/services/git/_sync.py Normal file
View File

@@ -0,0 +1,177 @@
# [DEF:GitServiceSyncMixin:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: git, push, pull, sync, remote
# @PURPOSE: Push and pull operations for GitService with origin host auto-alignment.
# @RELATION: USED_BY -> [GitService]
# @RELATION: DEPENDS_ON -> [GitServiceUrlMixin]
import os
from typing import Optional
from git import Repo
from git.exc import GitCommandError
from fastapi import HTTPException
from src.core.database import SessionLocal
from src.core.logger import logger, belief_scope
from src.models.git import GitRepository, GitServerConfig
# [DEF:GitServiceSyncMixin:Class]
# @COMPLEXITY: 3
# @PURPOSE: Mixin providing push and pull operations with origin host alignment.
class GitServiceSyncMixin:
# [DEF:push_changes:Function]
# @PURPOSE: Push local commits to remote.
# @PRE: Repository exists and has an 'origin' remote.
# @POST: Local branch commits are pushed to origin.
def push_changes(self, dashboard_id: int):
with belief_scope("GitService.push_changes"):
repo = self.get_repo(dashboard_id)
if not repo.heads:
logger.warning(f"[push_changes][Coherence:Failed] No local branches to push for dashboard {dashboard_id}")
return
try:
origin = repo.remote(name='origin')
except ValueError:
logger.error(f"[push_changes][Coherence:Failed] Remote 'origin' not found for dashboard {dashboard_id}")
raise HTTPException(status_code=400, detail="Remote 'origin' not configured")
try:
origin_urls = list(origin.urls)
except Exception:
origin_urls = []
binding_remote_url = None
binding_config_id = None
binding_config_url = None
try:
session = SessionLocal()
try:
db_repo = (
session.query(GitRepository)
.filter(GitRepository.dashboard_id == int(dashboard_id))
.first()
)
if db_repo:
binding_remote_url = db_repo.remote_url
binding_config_id = db_repo.config_id
db_config = (
session.query(GitServerConfig)
.filter(GitServerConfig.id == db_repo.config_id)
.first()
)
if db_config:
binding_config_url = db_config.url
finally:
session.close()
except Exception as diag_error:
logger.warning(
"[push_changes][Action] Failed to load repository binding diagnostics for dashboard %s: %s",
dashboard_id, diag_error,
)
realigned_origin_url = self._align_origin_host_with_config(
dashboard_id=dashboard_id,
origin=origin,
config_url=binding_config_url,
current_origin_url=(origin_urls[0] if origin_urls else None),
binding_remote_url=binding_remote_url,
)
try:
origin_urls = list(origin.urls)
except Exception:
origin_urls = []
logger.info(
"[push_changes][Action] Push diagnostics dashboard=%s config_id=%s config_url=%s binding_remote_url=%s origin_urls=%s origin_realigned=%s",
dashboard_id, binding_config_id, binding_config_url, binding_remote_url, origin_urls, bool(realigned_origin_url),
)
try:
current_branch = repo.active_branch
logger.info(f"[push_changes][Action] Pushing branch {current_branch.name} to origin")
tracking_branch = None
try:
tracking_branch = current_branch.tracking_branch()
except Exception:
tracking_branch = None
if tracking_branch is None:
repo.git.push("--set-upstream", "origin", f"{current_branch.name}:{current_branch.name}")
else:
push_info = origin.push(refspec=f'{current_branch.name}:{current_branch.name}')
for info in push_info:
if info.flags & info.ERROR:
logger.error(f"[push_changes][Coherence:Failed] Error pushing ref {info.remote_ref_string}: {info.summary}")
raise Exception(f"Git push error for {info.remote_ref_string}: {info.summary}")
except GitCommandError as e:
details = str(e)
lowered = details.lower()
if "non-fast-forward" in lowered or "rejected" in lowered:
raise HTTPException(
status_code=409,
detail="Push rejected: remote branch contains newer commits. Run Pull first, resolve conflicts if any, then push again.",
)
logger.error(f"[push_changes][Coherence:Failed] Failed to push changes: {e}")
raise HTTPException(status_code=500, detail=f"Git push failed: {details}")
except Exception as e:
logger.error(f"[push_changes][Coherence:Failed] Failed to push changes: {e}")
raise HTTPException(status_code=500, detail=f"Git push failed: {str(e)}")
# [/DEF:push_changes:Function]
# [DEF:pull_changes:Function]
# @PURPOSE: Pull changes from remote.
# @PRE: Repository exists and has an 'origin' remote.
# @POST: Changes from origin are pulled and merged into the active branch.
def pull_changes(self, dashboard_id: int):
with belief_scope("GitService.pull_changes"):
repo = self.get_repo(dashboard_id)
merge_head_path = os.path.join(repo.git_dir, "MERGE_HEAD")
if os.path.exists(merge_head_path):
payload = self._build_unfinished_merge_payload(repo)
logger.warning(
"[pull_changes][Action] Unfinished merge detected for dashboard %s (repo_path=%s git_dir=%s branch=%s merge_head=%s merge_msg=%s)",
dashboard_id, payload["repository_path"], payload["git_dir"],
payload["current_branch"], payload["merge_head"], payload["merge_message_preview"],
)
raise HTTPException(status_code=409, detail=payload)
try:
origin = repo.remote(name='origin')
current_branch = repo.active_branch.name
try:
origin_urls = list(origin.urls)
except Exception:
origin_urls = []
logger.info(
"[pull_changes][Action] Pull diagnostics dashboard=%s repo_path=%s branch=%s origin_urls=%s",
dashboard_id, repo.working_tree_dir, current_branch, origin_urls,
)
origin.fetch(prune=True)
remote_ref = f"origin/{current_branch}"
has_remote_branch = any(ref.name == remote_ref for ref in repo.refs)
logger.info(
"[pull_changes][Action] Pull remote branch check dashboard=%s branch=%s remote_ref=%s exists=%s",
dashboard_id, current_branch, remote_ref, has_remote_branch,
)
if not has_remote_branch:
raise HTTPException(
status_code=409,
detail=f"Remote branch '{current_branch}' does not exist yet. Push this branch first.",
)
logger.info(f"[pull_changes][Action] Pulling changes from origin/{current_branch}")
repo.git.pull("--no-rebase", "origin", current_branch)
except ValueError:
logger.error(f"[pull_changes][Coherence:Failed] Remote 'origin' not found for dashboard {dashboard_id}")
raise HTTPException(status_code=400, detail="Remote 'origin' not configured")
except GitCommandError as e:
details = str(e)
lowered = details.lower()
if "conflict" in lowered or "not possible to fast-forward" in lowered:
raise HTTPException(
status_code=409,
detail="Pull requires conflict resolution. Resolve conflicts in repository and repeat operation.",
)
logger.error(f"[pull_changes][Coherence:Failed] Failed to pull changes: {e}")
raise HTTPException(status_code=500, detail=f"Git pull failed: {details}")
except HTTPException:
raise
except Exception as e:
logger.error(f"[pull_changes][Coherence:Failed] Failed to pull changes: {e}")
raise HTTPException(status_code=500, detail=f"Git pull failed: {str(e)}")
# [/DEF:pull_changes:Function]
# [/DEF:GitServiceSyncMixin:Class]
# [/DEF:GitServiceSyncMixin:Module]

212
backend/src/services/git/_url.py Normal file
View File

@@ -0,0 +1,212 @@
# [DEF:GitServiceUrlMixin:Module]
# @COMPLEXITY: 3
# @LAYER: Infra
# @SEMANTICS: git, url, parsing, normalization, host_alignment
# @PURPOSE: URL helper mixin for GitService — parse, normalize, align, and strip credentials from Git URLs.
# @RELATION: USED_BY -> [GitServiceSyncMixin]
# @RELATION: USED_BY -> [GitServiceGiteaMixin]
# @RELATION: USED_BY -> [GitServiceRemoteMixin]
import os
from typing import Any, Dict, List, Optional
from urllib.parse import quote, urlparse
from fastapi import HTTPException
from src.core.database import SessionLocal
from src.core.logger import logger, belief_scope
from src.models.git import GitRepository, GitServerConfig
# [DEF:GitServiceUrlMixin:Class]
# @COMPLEXITY: 3
# @PURPOSE: URL helper methods for GitService — extract host, strip credentials, replace host, align origin, parse remote identity, derive server URL, normalize server URL.
class GitServiceUrlMixin:
# [DEF:_extract_http_host:Function]
# @PURPOSE: Extract normalized host[:port] from HTTP(S) URL.
# @PRE: url_value may be empty.
# @POST: Returns lowercase host token or None.
# @RETURN: Optional[str]
def _extract_http_host(self, url_value: Optional[str]) -> Optional[str]:
normalized = str(url_value or "").strip()
if not normalized:
return None
try:
parsed = urlparse(normalized)
except Exception:
return None
if parsed.scheme not in {"http", "https"}:
return None
host = parsed.hostname
if not host:
return None
if parsed.port:
return f"{host.lower()}:{parsed.port}"
return host.lower()
# [/DEF:_extract_http_host:Function]
# [DEF:_strip_url_credentials:Function]
# @PURPOSE: Remove credentials from URL while preserving scheme/host/path.
# @PRE: url_value may contain credentials.
# @POST: Returns URL without username/password.
# @RETURN: str
def _strip_url_credentials(self, url_value: str) -> str:
normalized = str(url_value or "").strip()
if not normalized:
return normalized
try:
parsed = urlparse(normalized)
except Exception:
return normalized
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
return normalized
host = parsed.hostname
if parsed.port:
host = f"{host}:{parsed.port}"
return parsed._replace(netloc=host).geturl()
# [/DEF:_strip_url_credentials:Function]
# [DEF:_replace_host_in_url:Function]
# @PURPOSE: Replace source URL host with host from configured server URL.
# @PRE: source_url and config_url are HTTP(S) URLs.
# @POST: Returns source URL with updated host (credentials preserved) or None.
# @RETURN: Optional[str]
def _replace_host_in_url(self, source_url: Optional[str], config_url: Optional[str]) -> Optional[str]:
source = str(source_url or "").strip()
config = str(config_url or "").strip()
if not source or not config:
return None
try:
source_parsed = urlparse(source)
config_parsed = urlparse(config)
except Exception:
return None
if source_parsed.scheme not in {"http", "https"} or config_parsed.scheme not in {"http", "https"}:
return None
if not source_parsed.hostname or not config_parsed.hostname:
return None
target_host = config_parsed.hostname
if config_parsed.port:
target_host = f"{target_host}:{config_parsed.port}"
auth_part = ""
if source_parsed.username:
auth_part = quote(source_parsed.username, safe="")
if source_parsed.password is not None:
auth_part = f"{auth_part}:{quote(source_parsed.password, safe='')}"
auth_part = f"{auth_part}@"
new_netloc = f"{auth_part}{target_host}"
return source_parsed._replace(netloc=new_netloc).geturl()
# [/DEF:_replace_host_in_url:Function]
# [DEF:_align_origin_host_with_config:Function]
# @PURPOSE: Auto-align local origin host to configured Git server host when they drift.
# @PRE: origin remote exists.
# @POST: origin URL host updated and DB binding normalized when mismatch detected.
# @RETURN: Optional[str]
def _align_origin_host_with_config(
self,
dashboard_id: int,
origin,
config_url: Optional[str],
current_origin_url: Optional[str],
binding_remote_url: Optional[str],
) -> Optional[str]:
config_host = self._extract_http_host(config_url)
source_origin_url = str(current_origin_url or "").strip() or str(binding_remote_url or "").strip()
origin_host = self._extract_http_host(source_origin_url)
if not config_host or not origin_host:
return None
if config_host == origin_host:
return None
aligned_url = self._replace_host_in_url(source_origin_url, config_url)
if not aligned_url:
return None
logger.warning(
"[_align_origin_host_with_config][Action] Host mismatch for dashboard %s: config_host=%s origin_host=%s, applying origin.set_url",
dashboard_id, config_host, origin_host,
)
try:
origin.set_url(aligned_url)
except Exception as e:
logger.warning(
"[_align_origin_host_with_config][Coherence:Failed] Failed to set origin URL for dashboard %s: %s",
dashboard_id, e,
)
return None
try:
session = SessionLocal()
try:
db_repo = (
session.query(GitRepository)
.filter(GitRepository.dashboard_id == int(dashboard_id))
.first()
)
if db_repo:
db_repo.remote_url = self._strip_url_credentials(aligned_url)
session.commit()
finally:
session.close()
except Exception as e:
logger.warning(
"[_align_origin_host_with_config][Action] Failed to persist aligned remote_url for dashboard %s: %s",
dashboard_id, e,
)
return aligned_url
# [/DEF:_align_origin_host_with_config:Function]
# [DEF:_parse_remote_repo_identity:Function]
# @PURPOSE: Parse owner/repo from remote URL for Git server API operations.
# @PRE: remote_url is a valid git URL.
# @POST: Returns owner/repo tokens.
# @RETURN: Dict[str, str]
def _parse_remote_repo_identity(self, remote_url: str) -> Dict[str, str]:
normalized = str(remote_url or "").strip()
if not normalized:
raise HTTPException(status_code=400, detail="Repository remote_url is empty")
if normalized.startswith("git@"):
path = normalized.split(":", 1)[1] if ":" in normalized else ""
else:
parsed = urlparse(normalized)
path = parsed.path or ""
path = path.strip("/")
if path.endswith(".git"):
path = path[:-4]
parts = [segment for segment in path.split("/") if segment]
if len(parts) < 2:
raise HTTPException(status_code=400, detail=f"Cannot parse repository owner/name from remote URL: {remote_url}")
owner = parts[0]
repo = parts[-1]
namespace = "/".join(parts[:-1])
return {"owner": owner, "repo": repo, "namespace": namespace, "full_name": f"{namespace}/{repo}"}
# [/DEF:_parse_remote_repo_identity:Function]
# [DEF:_derive_server_url_from_remote:Function]
# @PURPOSE: Build API base URL from remote repository URL without credentials.
# @PRE: remote_url may be any git URL.
# @POST: Returns normalized http(s) base URL or None when derivation is impossible.
# @RETURN: Optional[str]
def _derive_server_url_from_remote(self, remote_url: str) -> Optional[str]:
normalized = str(remote_url or "").strip()
if not normalized or normalized.startswith("git@"):
return None
parsed = urlparse(normalized)
if parsed.scheme not in {"http", "https"}:
return None
if not parsed.hostname:
return None
netloc = parsed.hostname
if parsed.port:
netloc = f"{netloc}:{parsed.port}"
return f"{parsed.scheme}://{netloc}".rstrip("/")
# [/DEF:_derive_server_url_from_remote:Function]
# [DEF:_normalize_git_server_url:Function]
# @PURPOSE: Normalize Git server URL for provider API calls.
# @PRE: raw_url is non-empty.
# @POST: Returns URL without trailing slash.
# @RETURN: str
def _normalize_git_server_url(self, raw_url: str) -> str:
normalized = (raw_url or "").strip()
if not normalized:
raise HTTPException(status_code=400, detail="Git server URL is required")
return normalized.rstrip("/")
# [/DEF:_normalize_git_server_url:Function]
# [/DEF:GitServiceUrlMixin:Class]
# [/DEF:GitServiceUrlMixin:Module]

2112
backend/src/services/git_service.py
View File

File diff suppressed because it is too large Load Diff