test: massive coverage expansion — 15 new test modules + assistant tool fixes + orthogonal testing
- 10 translate plugin test files (100% coverage on 12 modules) - assistant/handler tools: 85+ tests covering dispatch, registry, resolvers, routes, llm_planner, 13 tool handlers - clean release: artifact_catalog_loader, mappers, approval, publication tests - API routes: translate_helpers, validation_service extensions, datasets to 100% - notifications: providers/service tests - services: profile_preference_service - docs/orthogonal-test-report.md — full speckit.tests audit - Fixes: 3 git_base async mock failures, 4 assistant handler permission-check patches - .gitignore: coverage artifacts
This commit is contained in:
145
backend/src/services/auth_service.py,cover
Normal file
145
backend/src/services/auth_service.py,cover
Normal file
@@ -0,0 +1,145 @@
|
||||
# #region auth_service [C:5] [TYPE Module] [SEMANTICS sqlalchemy, auth, credential, session, jwt]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Orchestrates credential authentication and ADFS JIT user provisioning.
|
||||
# @LAYER Domain
|
||||
# @RELATION DEPENDS_ON -> [AuthRepository]
|
||||
# @RELATION DEPENDS_ON -> [verify_password]
|
||||
# @RELATION DEPENDS_ON -> [create_access_token]
|
||||
# @RELATION DEPENDS_ON -> [User]
|
||||
# @RELATION DEPENDS_ON -> [Role]
|
||||
# @INVARIANT Authentication succeeds only for active users with valid credentials; issued sessions encode subject and scopes from assigned roles.
|
||||
# @PRE Core auth models and security utilities available.
|
||||
# @POST User identity verified and session tokens issued according to role scopes.
|
||||
# @SIDE_EFFECT Writes last login timestamps and JIT-provisions external users.
|
||||
# @DATA_CONTRACT [Credentials | ADFSClaims] -> [UserEntity | SessionToken]
|
||||
|
||||
> from datetime import UTC, datetime
|
||||
> from typing import Any
|
||||
|
||||
> from sqlalchemy.orm import Session
|
||||
|
||||
> from ..core.auth.jwt import create_access_token
|
||||
> from ..core.auth.logger import log_security_event
|
||||
> from ..core.auth.repository import AuthRepository
|
||||
> from ..core.auth.security import verify_password
|
||||
> from ..core.logger import belief_scope
|
||||
> from ..models.auth import User
|
||||
|
||||
|
||||
# #region AuthService [C:3] [TYPE Class]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Provides high-level authentication services.
|
||||
# @RELATION DEPENDS_ON -> [AuthRepository]
|
||||
# @RELATION DEPENDS_ON -> [User]
|
||||
# @RELATION DEPENDS_ON -> [Role]
|
||||
> class AuthService:
|
||||
# region AuthService_init [TYPE Function]
|
||||
# @PURPOSE: Initializes the authentication service with repository access over an active DB session.
|
||||
# @PRE db is a valid SQLAlchemy Session instance bound to the auth persistence context.
|
||||
# @POST self.repo is initialized and ready for auth user/role CRUD operations.
|
||||
# @SIDE_EFFECT Allocates AuthRepository and binds it to the provided Session.
|
||||
# @DATA_CONTRACT Input(Session) -> Model(AuthRepository)
|
||||
# @PARAM db (Session) - SQLAlchemy session.
|
||||
> def __init__(self, db: Session):
|
||||
> self.db = db
|
||||
> self.repo = AuthRepository(db)
|
||||
|
||||
# endregion AuthService_init
|
||||
|
||||
# region AuthService.authenticate_user [TYPE Function]
|
||||
# @PURPOSE: Validates credentials and account state for local username/password authentication.
|
||||
# @PRE username and password are non-empty credential inputs.
|
||||
# @POST Returns User only when user exists, is active, and password hash verification succeeds; otherwise returns None.
|
||||
# @SIDE_EFFECT Persists last_login update for successful authentications via repository.
|
||||
# @DATA_CONTRACT Input(str username, str password) -> Output(User | None)
|
||||
# @RELATION DEPENDS_ON ->[AuthRepository]
|
||||
# @RELATION CALLS ->[verify_password]
|
||||
# @RELATION DEPENDS_ON ->[User]
|
||||
# @PARAM username (str) - The username.
|
||||
# @PARAM password (str) - The plain password.
|
||||
# @RETURN Optional[User] - The authenticated user or None.
|
||||
> def authenticate_user(self, username: str, password: str) -> User | None:
|
||||
> with belief_scope("auth.authenticate_user"):
|
||||
> user = self.repo.get_user_by_username(username)
|
||||
> if not user or not user.is_active:
|
||||
> return None
|
||||
|
||||
> if not verify_password(password, user.password_hash):
|
||||
> return None
|
||||
|
||||
# Update last login
|
||||
> user.last_login = datetime.now(UTC)
|
||||
> self.db.commit()
|
||||
> self.db.refresh(user)
|
||||
|
||||
> return user
|
||||
|
||||
# endregion AuthService.authenticate_user
|
||||
|
||||
# region AuthService.create_session [TYPE Function]
|
||||
# @PURPOSE: Issues an access token payload for an already authenticated user.
|
||||
# @PRE user is a valid User entity containing username and iterable roles with role.name values.
|
||||
# @POST Returns session dict with non-empty access_token and token_type='bearer'.
|
||||
# @SIDE_EFFECT Generates signed JWT via auth JWT provider.
|
||||
# @DATA_CONTRACT Input(User) -> Output(Dict[str, str]{access_token, token_type})
|
||||
# @RELATION CALLS ->[create_access_token]
|
||||
# @RELATION DEPENDS_ON ->[User]
|
||||
# @RELATION DEPENDS_ON ->[Role]
|
||||
# @PARAM user (User) - The authenticated user.
|
||||
# @RETURN Dict[str, str] - Session data.
|
||||
> def create_session(self, user: User) -> dict[str, str]:
|
||||
> with belief_scope("auth.create_session"):
|
||||
> roles = [role.name for role in user.roles]
|
||||
> access_token = create_access_token(
|
||||
> data={"sub": user.username, "scopes": roles}
|
||||
> )
|
||||
> return {"access_token": access_token, "token_type": "bearer"}
|
||||
|
||||
# endregion AuthService.create_session
|
||||
|
||||
# region AuthService.provision_adfs_user [TYPE Function]
|
||||
# @PURPOSE: Performs ADFS Just-In-Time provisioning and role synchronization from AD group mappings.
|
||||
# @PRE user_info contains identity claims where at least one of 'upn' or 'email' is present; 'groups' may be absent.
|
||||
# @POST Returns persisted user entity with roles synchronized to mapped AD groups and refreshed state.
|
||||
# @SIDE_EFFECT May insert new User, mutate user.roles, commit transaction, and refresh ORM state.
|
||||
# @DATA_CONTRACT Input(Dict[str, Any]{upn|email, email, groups[]}) -> Output(User persisted)
|
||||
# @RELATION DEPENDS_ON ->[AuthRepository]
|
||||
# @RELATION DEPENDS_ON ->[User]
|
||||
# @RELATION DEPENDS_ON ->[Role]
|
||||
# @PARAM user_info (Dict[str, Any]) - Claims from ADFS token.
|
||||
# @RETURN User - The provisioned user.
|
||||
> def provision_adfs_user(self, user_info: dict[str, Any]) -> User:
|
||||
> with belief_scope("auth.provision_adfs_user"):
|
||||
> username = user_info.get("upn") or user_info.get("email")
|
||||
> email = user_info.get("email")
|
||||
> groups = user_info.get("groups", [])
|
||||
|
||||
> user = self.repo.get_user_by_username(username)
|
||||
> if not user:
|
||||
> user = User(
|
||||
> username=username,
|
||||
> email=email,
|
||||
> full_name=user_info.get("name"),
|
||||
> auth_source="ADFS",
|
||||
> is_active=True,
|
||||
> is_ad_user=True,
|
||||
> )
|
||||
> self.db.add(user)
|
||||
> log_security_event("USER_PROVISIONED", username, {"source": "ADFS"})
|
||||
|
||||
# Sync roles from AD groups
|
||||
> mapped_roles = self.repo.get_roles_by_ad_groups(groups)
|
||||
> user.roles = mapped_roles
|
||||
|
||||
> user.last_login = datetime.now(UTC)
|
||||
> self.db.commit()
|
||||
> self.db.refresh(user)
|
||||
|
||||
> return user
|
||||
|
||||
# endregion AuthService.provision_adfs_user
|
||||
|
||||
|
||||
# #endregion AuthService
|
||||
|
||||
# #endregion auth_service
|
||||
10
backend/src/services/git_service.py,cover
Normal file
10
backend/src/services/git_service.py,cover
Normal file
@@ -0,0 +1,10 @@
|
||||
# #region git_service [C:1] [TYPE Module:Tombstone] [SEMANTICS git, service, shim, re-export, decomissioned]
|
||||
# @BRIEF Re-export shim — GitService has been decomposed into services/git/ package.
|
||||
# All consumers continue to import from this same path without changes.
|
||||
# @RELATION CALLS -> [GitServiceModule]
|
||||
# @RATIONALE Monolithic GitService (2101 lines) was decomposed into 8 domain-specific mixins
|
||||
# under services/git/ to satisfy INV_7 (< 400 lines per module). This shim preserves
|
||||
# the original import path for all 5 consumers.
|
||||
# @REJECTED Breaking 5 consumer imports to remove this shim — unacceptable migration cost.
|
||||
> from src.services.git import GitService # noqa: F401
|
||||
# #endregion git_service
|
||||
412
backend/src/services/health_service.py,cover
Normal file
412
backend/src/services/health_service.py,cover
Normal file
@@ -0,0 +1,412 @@
|
||||
# #region health_service [C:3] [TYPE Module] [SEMANTICS sqlalchemy, health, dashboard, validation, aggregate]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Business logic for aggregating dashboard health status from validation records.
|
||||
# @LAYER Service
|
||||
# @RELATION DEPENDS_ON -> [ValidationRecord]
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [TaskCleanupService]
|
||||
# @RELATION DEPENDS_ON -> [TaskManager]
|
||||
|
||||
> from datetime import UTC
|
||||
> import json
|
||||
> import os
|
||||
> import time
|
||||
> from typing import Any, cast
|
||||
|
||||
> from sqlalchemy import func
|
||||
> from sqlalchemy.orm import Session
|
||||
|
||||
> from ..core.logger import logger
|
||||
> from ..core.task_manager import TaskManager
|
||||
> from ..core.utils.client_registry import get_superset_client
|
||||
> from ..core.task_manager.cleanup import TaskCleanupService
|
||||
> from ..models.llm import ValidationRecord
|
||||
> from ..schemas.health import DashboardHealthItem, HealthSummaryResponse
|
||||
|
||||
|
||||
> def _empty_dashboard_meta() -> dict[str, str | None]:
|
||||
> return cast(dict[str, str | None], {"slug": None, "title": None})
|
||||
|
||||
|
||||
# #region HealthService [C:4] [TYPE Class]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Aggregate latest dashboard validation state and manage persisted health report lifecycle.
|
||||
# @PRE Service is constructed with a live SQLAlchemy session and optional config manager.
|
||||
# @POST Exposes health summary aggregation and validation report deletion operations.
|
||||
# @SIDE_EFFECT Maintains in-memory dashboard metadata caches and may coordinate cleanup through collaborators.
|
||||
# @DATA_CONTRACT Input[Session, Optional[Any]] -> Output[HealthSummaryResponse|bool]
|
||||
# @RELATION DEPENDS_ON -> [ValidationRecord]
|
||||
# @RELATION DEPENDS_ON -> [DashboardHealthItem]
|
||||
# @RELATION DEPENDS_ON -> [HealthSummaryResponse]
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [TaskCleanupService]
|
||||
# @RELATION DEPENDS_ON -> [TaskManager]
|
||||
> class HealthService:
|
||||
> _dashboard_summary_cache: dict[
|
||||
> str, tuple[float, dict[str, dict[str, str | None]]]
|
||||
> ] = {}
|
||||
> _dashboard_summary_cache_ttl_seconds = 60.0
|
||||
|
||||
> """
|
||||
> @PURPOSE: Service for managing and querying dashboard health data.
|
||||
> """
|
||||
|
||||
# region HealthService_init [TYPE Function]
|
||||
# @PURPOSE: Initialize health service with DB session and optional config access for dashboard metadata resolution.
|
||||
# @PRE db is a valid SQLAlchemy session.
|
||||
# @POST Service is ready to aggregate summaries and delete health reports.
|
||||
# @SIDE_EFFECT Initializes per-instance dashboard metadata cache.
|
||||
# @DATA_CONTRACT Input[db: Session, config_manager: Optional[Any]] -> Output[HealthService]
|
||||
# @RELATION BINDS_TO ->[HealthService]
|
||||
> def __init__(self, db: Session, config_manager=None):
|
||||
> self.db = db
|
||||
> self.config_manager = config_manager
|
||||
> self._dashboard_meta_cache: dict[tuple[str, str], dict[str, str | None]] = {}
|
||||
|
||||
# endregion HealthService_init
|
||||
|
||||
# region _prime_dashboard_meta_cache [TYPE Function]
|
||||
# @PURPOSE: Warm dashboard slug/title cache with one Superset list fetch per environment.
|
||||
# @PRE records may contain mixed numeric and slug dashboard identifiers.
|
||||
# @POST Numeric dashboard ids for known environments are cached when discoverable.
|
||||
# @SIDE_EFFECT May call Superset dashboard list API once per referenced environment.
|
||||
# @DATA_CONTRACT Input[records: List[ValidationRecord]] -> Output[None]
|
||||
# @RELATION DEPENDS_ON ->[ValidationRecord]
|
||||
# @RELATION DEPENDS_ON ->[ConfigManager]
|
||||
# @RELATION DEPENDS_ON ->[SupersetClient]
|
||||
> async def _prime_dashboard_meta_cache(self, records: list[ValidationRecord]) -> None:
|
||||
> if not self.config_manager or not records:
|
||||
! return
|
||||
|
||||
> numeric_ids_by_env: dict[str, set[str]] = {}
|
||||
> for record in records:
|
||||
> environment_id = str(record.environment_id or "").strip()
|
||||
> dashboard_id = str(record.dashboard_id or "").strip()
|
||||
> if not environment_id or not dashboard_id or not dashboard_id.isdigit():
|
||||
! continue
|
||||
> cache_key = (environment_id, dashboard_id)
|
||||
> if cache_key in self._dashboard_meta_cache:
|
||||
! continue
|
||||
> numeric_ids_by_env.setdefault(environment_id, set()).add(dashboard_id)
|
||||
|
||||
> if not numeric_ids_by_env:
|
||||
! return
|
||||
|
||||
> environments = {
|
||||
> str(getattr(env, "id", "")).strip(): env
|
||||
> for env in self.config_manager.get_environments()
|
||||
> if str(getattr(env, "id", "")).strip()
|
||||
> }
|
||||
|
||||
> for environment_id, dashboard_ids in numeric_ids_by_env.items():
|
||||
> env = environments.get(environment_id)
|
||||
> if not env:
|
||||
! for dashboard_id in dashboard_ids:
|
||||
! self._dashboard_meta_cache[(environment_id, dashboard_id)] = (
|
||||
! _empty_dashboard_meta()
|
||||
! )
|
||||
! continue
|
||||
|
||||
> try:
|
||||
> cached_meta = self.__class__._dashboard_summary_cache.get(
|
||||
> environment_id
|
||||
> )
|
||||
> dashboard_meta_map: dict[str, dict[str, str | None]]
|
||||
> if (
|
||||
> cached_meta is not None
|
||||
> and (time.monotonic() - cached_meta[0])
|
||||
> < self.__class__._dashboard_summary_cache_ttl_seconds
|
||||
> ):
|
||||
! cached_meta_data = cast(
|
||||
! tuple[float, dict[str, dict[str, str | None]]],
|
||||
! cached_meta,
|
||||
! )
|
||||
! dashboard_meta_map = cached_meta_data[1]
|
||||
> else:
|
||||
> client = await get_superset_client(env)
|
||||
> dashboards = await client.get_dashboards_summary()
|
||||
> dashboard_meta_map = {
|
||||
> str(item.get("id")): {
|
||||
> "slug": item.get("slug"),
|
||||
> "title": item.get("title"),
|
||||
> }
|
||||
> for item in dashboards
|
||||
> if str(item.get("id") or "").strip()
|
||||
> }
|
||||
> self.__class__._dashboard_summary_cache[environment_id] = (
|
||||
> time.monotonic(),
|
||||
> dashboard_meta_map,
|
||||
> )
|
||||
> for dashboard_id in dashboard_ids:
|
||||
> self._dashboard_meta_cache[(environment_id, dashboard_id)] = (
|
||||
> dashboard_meta_map.get(
|
||||
> dashboard_id,
|
||||
> _empty_dashboard_meta(),
|
||||
> )
|
||||
> )
|
||||
! except Exception as exc:
|
||||
! logger.warning(
|
||||
! "[HealthService][EXT:method:_prime_dashboard_meta_cache] Failed to preload dashboard metadata for env=%s: %s",
|
||||
! environment_id,
|
||||
! exc,
|
||||
! )
|
||||
! for dashboard_id in dashboard_ids:
|
||||
! self._dashboard_meta_cache[(environment_id, dashboard_id)] = (
|
||||
! _empty_dashboard_meta()
|
||||
! )
|
||||
|
||||
# endregion _prime_dashboard_meta_cache
|
||||
|
||||
# region _resolve_dashboard_meta [TYPE Function]
|
||||
# @PURPOSE: Resolve slug/title for a dashboard referenced by persisted validation record.
|
||||
# @PRE dashboard_id may be numeric or slug-like; environment_id may be empty.
|
||||
# @POST Returns dict with `slug` and `title` keys, using cache when possible.
|
||||
# @SIDE_EFFECT Writes default cache entries for unresolved numeric dashboard ids.
|
||||
> def _resolve_dashboard_meta(
|
||||
> self, dashboard_id: str, environment_id: str | None
|
||||
> ) -> dict[str, str | None]:
|
||||
> normalized_dashboard_id = str(dashboard_id or "").strip()
|
||||
> normalized_environment_id = str(environment_id or "").strip()
|
||||
> if not normalized_dashboard_id:
|
||||
! return _empty_dashboard_meta()
|
||||
|
||||
> if not normalized_dashboard_id.isdigit():
|
||||
! return {"slug": normalized_dashboard_id, "title": None}
|
||||
|
||||
> if not self.config_manager or not normalized_environment_id:
|
||||
! return _empty_dashboard_meta()
|
||||
|
||||
> cache_key = (normalized_environment_id, normalized_dashboard_id)
|
||||
> cached = self._dashboard_meta_cache.get(cache_key)
|
||||
> if cached is not None:
|
||||
> return cached
|
||||
|
||||
! meta = _empty_dashboard_meta()
|
||||
! self._dashboard_meta_cache[cache_key] = meta
|
||||
! return meta
|
||||
|
||||
# endregion _resolve_dashboard_meta
|
||||
|
||||
# region get_health_summary [TYPE Function]
|
||||
# @PURPOSE: Aggregate latest validation status per dashboard and enrich rows with dashboard slug/title.
|
||||
# @PRE environment_id may be omitted to aggregate across all environments.
|
||||
# @POST Returns HealthSummaryResponse with counts and latest record row per dashboard.
|
||||
# @SIDE_EFFECT May call Superset API to resolve dashboard metadata.
|
||||
# @DATA_CONTRACT Input[environment_id: Optional[str]] -> Output[HealthSummaryResponse]
|
||||
# @RELATION CALLS ->[EXT:method:_prime_dashboard_meta_cache]
|
||||
# @RELATION CALLS ->[EXT:method:_resolve_dashboard_meta]
|
||||
> async def get_health_summary(
|
||||
> self, environment_id: str = ""
|
||||
> ) -> HealthSummaryResponse:
|
||||
> """
|
||||
> @PURPOSE: Aggregates the latest validation status for all dashboards.
|
||||
> @PRE: environment_id (optional) to filter by environment.
|
||||
> @POST: Returns a HealthSummaryResponse with aggregated status counts and items.
|
||||
> """
|
||||
# [REASON] We need the latest ValidationRecord for each unique dashboard_id.
|
||||
# We use a subquery to find the max timestamp per dashboard_id.
|
||||
|
||||
> subquery = self.db.query(
|
||||
> ValidationRecord.dashboard_id,
|
||||
> func.max(ValidationRecord.timestamp).label("max_ts"),
|
||||
> )
|
||||
> if environment_id:
|
||||
> subquery = subquery.filter(
|
||||
> ValidationRecord.environment_id == environment_id
|
||||
> )
|
||||
> subquery = subquery.group_by(ValidationRecord.dashboard_id).subquery()
|
||||
|
||||
> query = self.db.query(ValidationRecord).join(
|
||||
> subquery,
|
||||
> (ValidationRecord.dashboard_id == subquery.c.dashboard_id)
|
||||
> & (ValidationRecord.timestamp == subquery.c.max_ts),
|
||||
> )
|
||||
|
||||
> records = query.all()
|
||||
|
||||
> await self._prime_dashboard_meta_cache(records)
|
||||
|
||||
> items = []
|
||||
> pass_count = 0
|
||||
> warn_count = 0
|
||||
> fail_count = 0
|
||||
> unknown_count = 0
|
||||
|
||||
> for rec in records:
|
||||
> record = cast(Any, rec)
|
||||
> status = str(record.status or "").upper()
|
||||
> if status == "PASS":
|
||||
> pass_count += 1
|
||||
! elif status == "WARN":
|
||||
! warn_count += 1
|
||||
! elif status == "FAIL":
|
||||
! fail_count += 1
|
||||
! else:
|
||||
! unknown_count += 1
|
||||
! status = "UNKNOWN"
|
||||
|
||||
> record_id = str(record.id or "")
|
||||
> dashboard_id = str(record.dashboard_id or "")
|
||||
> resolved_environment_id = (
|
||||
> str(record.environment_id)
|
||||
> if record.environment_id is not None
|
||||
> else None
|
||||
> )
|
||||
> response_environment_id = (
|
||||
> resolved_environment_id
|
||||
> if resolved_environment_id is not None
|
||||
> else "unknown"
|
||||
> )
|
||||
> task_id = str(record.task_id) if record.task_id is not None else None
|
||||
> summary = str(record.summary) if record.summary is not None else None
|
||||
> timestamp = cast(Any, record.timestamp)
|
||||
# Ensure timestamp is timezone-aware (DB stores naive UTC)
|
||||
> if timestamp is not None and timestamp.tzinfo is None:
|
||||
! timestamp = timestamp.replace(tzinfo=UTC)
|
||||
|
||||
# ── v2 fields ──────────────────────────────────────────────
|
||||
> execution_path = str(record.execution_path) if record.execution_path else None
|
||||
> issues_count = len(record.issues) if record.issues else 0
|
||||
> timings = dict(record.timings) if record.timings else None
|
||||
> token_usage = dict(record.token_usage) if record.token_usage else None
|
||||
> screenshot_paths = list(record.screenshot_paths) if record.screenshot_paths else None
|
||||
|
||||
# Extract chunk_count from raw_response (stored as JSON in result_payload)
|
||||
> chunk_count: int | None = None
|
||||
> if record.raw_response:
|
||||
! try:
|
||||
! raw = json.loads(record.raw_response)
|
||||
! chunk_count = int(raw["chunk_count"]) if raw.get("chunk_count") else None
|
||||
! except (json.JSONDecodeError, KeyError, TypeError, ValueError):
|
||||
! pass
|
||||
|
||||
> meta = self._resolve_dashboard_meta(dashboard_id, resolved_environment_id)
|
||||
> dashboard_name = meta.get("title") or meta.get("slug") or dashboard_id
|
||||
> items.append(
|
||||
> DashboardHealthItem(
|
||||
> record_id=record_id,
|
||||
> dashboard_id=dashboard_id,
|
||||
> dashboard_slug=meta.get("slug"),
|
||||
> dashboard_title=meta.get("title"),
|
||||
> environment_id=response_environment_id,
|
||||
> status=status,
|
||||
> last_check=timestamp,
|
||||
> task_id=task_id,
|
||||
> run_id=str(record.run_id) if record.run_id else None,
|
||||
> policy_id=str(record.policy_id) if record.policy_id else None,
|
||||
> summary=summary,
|
||||
> execution_path=execution_path,
|
||||
> issues_count=issues_count,
|
||||
> timings=timings,
|
||||
> token_usage=token_usage,
|
||||
> screenshot_paths=screenshot_paths,
|
||||
> chunk_count=chunk_count,
|
||||
> dashboard_name=dashboard_name,
|
||||
> )
|
||||
> )
|
||||
|
||||
> logger.info(
|
||||
> f"[HealthService][get_health_summary] Aggregated {len(items)} dashboard health records."
|
||||
> )
|
||||
|
||||
> return HealthSummaryResponse(
|
||||
> items=items,
|
||||
> pass_count=pass_count,
|
||||
> warn_count=warn_count,
|
||||
> fail_count=fail_count,
|
||||
> unknown_count=unknown_count,
|
||||
> )
|
||||
|
||||
# endregion get_health_summary
|
||||
|
||||
# region delete_validation_report [TYPE Function]
|
||||
# @PURPOSE: Delete one persisted health report and optionally clean linked task/log artifacts.
|
||||
# @PRE record_id is a validation record identifier.
|
||||
# @POST Returns True only when a matching record was deleted.
|
||||
# @SIDE_EFFECT Deletes DB rows, optional screenshot file, and optional task/log persistence.
|
||||
# @DATA_CONTRACT Input[record_id: str, task_manager: Optional[TaskManager]] -> Output[bool]
|
||||
# @RELATION DEPENDS_ON ->[ValidationRecord]
|
||||
# @RELATION DEPENDS_ON ->[TaskManager]
|
||||
# @RELATION DEPENDS_ON ->[TaskCleanupService]
|
||||
> def delete_validation_report(
|
||||
> self, record_id: str, task_manager: TaskManager | None = None
|
||||
> ) -> bool:
|
||||
! record = (
|
||||
! self.db.query(ValidationRecord)
|
||||
! .filter(ValidationRecord.id == record_id)
|
||||
! .first()
|
||||
! )
|
||||
! if not record:
|
||||
! return False
|
||||
|
||||
! peer_query = self.db.query(ValidationRecord).filter(
|
||||
! ValidationRecord.dashboard_id == record.dashboard_id
|
||||
! )
|
||||
! if record.environment_id is None:
|
||||
! peer_query = peer_query.filter(ValidationRecord.environment_id.is_(None))
|
||||
! else:
|
||||
! peer_query = peer_query.filter(
|
||||
! ValidationRecord.environment_id == record.environment_id
|
||||
! )
|
||||
|
||||
! records_to_delete = peer_query.all()
|
||||
! screenshot_paths = [
|
||||
! str(item.screenshot_path or "").strip()
|
||||
! for item in records_to_delete
|
||||
! if str(item.screenshot_path or "").strip()
|
||||
! ]
|
||||
! task_ids = {
|
||||
! str(item.task_id or "").strip()
|
||||
! for item in records_to_delete
|
||||
! if str(item.task_id or "").strip()
|
||||
! }
|
||||
|
||||
! logger.info(
|
||||
! "[HealthService][delete_validation_report] Removing %s validation record(s) for dashboard=%s environment=%s triggered_by_record=%s",
|
||||
! len(records_to_delete),
|
||||
! record.dashboard_id,
|
||||
! record.environment_id,
|
||||
! record_id,
|
||||
! )
|
||||
|
||||
! for item in records_to_delete:
|
||||
! self.db.delete(item)
|
||||
! self.db.commit()
|
||||
|
||||
! for screenshot_path in screenshot_paths:
|
||||
! try:
|
||||
! if os.path.exists(screenshot_path):
|
||||
! os.remove(screenshot_path)
|
||||
! except OSError as exc:
|
||||
! logger.warning(
|
||||
! "[HealthService][delete_validation_report] Failed to remove screenshot %s: %s",
|
||||
! screenshot_path,
|
||||
! exc,
|
||||
! )
|
||||
|
||||
! if task_ids and task_manager and self.config_manager:
|
||||
! try:
|
||||
! cleanup_service = TaskCleanupService(
|
||||
! task_manager.persistence_service,
|
||||
! task_manager.log_persistence_service,
|
||||
! self.config_manager,
|
||||
! )
|
||||
! for task_id in task_ids:
|
||||
! task_manager.tasks.pop(task_id, None)
|
||||
! cleanup_service.delete_task_with_logs(task_id)
|
||||
! except Exception as exc:
|
||||
! logger.warning(
|
||||
! "[HealthService][delete_validation_report] Failed to cleanup linked task/logs for dashboard=%s environment=%s: %s",
|
||||
! record.dashboard_id,
|
||||
! record.environment_id,
|
||||
! exc,
|
||||
! )
|
||||
|
||||
! return True
|
||||
|
||||
# endregion delete_validation_report
|
||||
|
||||
|
||||
# #endregion HealthService
|
||||
|
||||
# #endregion health_service
|
||||
267
backend/src/services/llm_prompt_templates.py,cover
Normal file
267
backend/src/services/llm_prompt_templates.py,cover
Normal file
@@ -0,0 +1,267 @@
|
||||
# #region llm_prompt_templates [C:2] [TYPE Module] [SEMANTICS llm, prompt, template, normalization]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Provide default LLM prompt templates and normalization helpers for runtime usage.
|
||||
# @LAYER Domain
|
||||
# @RELATION DEPENDS_ON -> [ConfigManager]
|
||||
# @INVARIANT All required prompt template keys are always present after normalization.
|
||||
|
||||
> from __future__ import annotations
|
||||
|
||||
> from copy import deepcopy
|
||||
> from typing import Any
|
||||
> import warnings
|
||||
|
||||
> from ..core.logger import logger
|
||||
|
||||
# #region DEFAULT_LLM_PROMPTS [C:2] [TYPE Constant]
|
||||
# @ingroup Services
|
||||
# @BRIEF Default prompt templates used by documentation, dashboard validation, and git commit generation.
|
||||
> DEFAULT_LLM_PROMPTS: dict[str, str] = {
|
||||
> "dashboard_validation_prompt": ( # Legacy v1 — deprecated for new tasks, kept for backward compat
|
||||
> "Analyze the attached dashboard screenshot and the following execution logs for health and visual issues.\n\n"
|
||||
> "Logs:\n"
|
||||
> "{logs}\n\n"
|
||||
> "Provide the analysis in JSON format with the following structure:\n"
|
||||
> "{\n"
|
||||
> ' "status": "PASS" | "WARN" | "FAIL",\n'
|
||||
> ' "summary": "Short summary of findings",\n'
|
||||
> ' "issues": [\n'
|
||||
> " {\n"
|
||||
> ' "severity": "WARN" | "FAIL",\n'
|
||||
> ' "message": "Description of the issue",\n'
|
||||
> ' "location": "Optional location info (e.g. chart name)"\n'
|
||||
> " }\n"
|
||||
> " ]\n"
|
||||
> "}"
|
||||
> ),
|
||||
> "dashboard_validation_prompt_multimodal": ( # Path A — image-focused (v2)
|
||||
> "Analyze the attached {total_chunks} dashboard tab screenshots and execution logs for visual and data health issues.\n\n"
|
||||
> "Screenshots are attached IN ORDER — screenshot 0 is the first tab, screenshot 1 is the second, etc.\n"
|
||||
> "Tab order:\n"
|
||||
> "{tab_list}\n\n"
|
||||
> "Logs:\n"
|
||||
> "{logs}\n\n"
|
||||
> "Provide the analysis in JSON format. Each issue MUST specify the tab_index field "
|
||||
> "(0-based index of the screenshot/tab where the issue appears, or -1 if not tab-specific):\n"
|
||||
> "{\n"
|
||||
> ' "status": "PASS" | "WARN" | "FAIL",\n'
|
||||
> ' "summary": "Short summary of findings",\n'
|
||||
> ' "issues": [\n'
|
||||
> " {\n"
|
||||
> ' "severity": "WARN" | "FAIL",\n'
|
||||
> ' "message": "Description of the issue",\n'
|
||||
> ' "location": "Where the issue is located (tab name, chart name, screen region)",\n'
|
||||
> ' "tab_index": "0-based tab index (-1 if unknown)"\n'
|
||||
> " }\n"
|
||||
> " ]\n"
|
||||
> "}"
|
||||
> ),
|
||||
> "dashboard_validation_prompt_text": ( # Path B — topology-focused (v2)
|
||||
> "Analyze the following dashboard topology, dataset health, and execution logs for "
|
||||
> "data consistency and KXD connectivity issues.\n\n"
|
||||
> "Dashboard structure (tabs are indicated in the topology below):\n"
|
||||
> "{topology}\n\n"
|
||||
> "Dataset health:\n"
|
||||
> "{dataset_health}\n\n"
|
||||
> "Logs:\n"
|
||||
> "{logs}\n\n"
|
||||
> "Provide the analysis in JSON format. Each issue MUST specify the tab_index field "
|
||||
> "(0-based index of the tab where the issue appears, or -1 if not tab-specific):\n"
|
||||
> "{\n"
|
||||
> ' "status": "PASS" | "WARN" | "FAIL",\n'
|
||||
> ' "summary": "Short summary of findings",\n'
|
||||
> ' "issues": [\n'
|
||||
> " {\n"
|
||||
> ' "severity": "WARN" | "FAIL",\n'
|
||||
> ' "message": "Description of the issue",\n'
|
||||
> ' "location": "Where the issue is located (tab name, chart name, dataset name)",\n'
|
||||
> ' "tab_index": "0-based tab index (-1 if unknown)"\n'
|
||||
> " }\n"
|
||||
> " ]\n"
|
||||
> "}"
|
||||
> ),
|
||||
> "documentation_prompt": (
|
||||
> "Generate professional documentation for the following dataset and its columns.\n"
|
||||
> "Dataset: {dataset_name}\n"
|
||||
> "Columns: {columns_json}\n\n"
|
||||
> "Provide the documentation in JSON format:\n"
|
||||
> "{\n"
|
||||
> ' "dataset_description": "General description of the dataset",\n'
|
||||
> ' "column_descriptions": [\n'
|
||||
> " {\n"
|
||||
> ' "name": "column_name",\n'
|
||||
> ' "description": "Generated description"\n'
|
||||
> " }\n"
|
||||
> " ]\n"
|
||||
> "}"
|
||||
> ),
|
||||
> "git_commit_prompt": (
|
||||
> "Generate a concise and professional git commit message based on the following diff and recent history.\n"
|
||||
> "Use Conventional Commits format (e.g., feat: ..., fix: ..., docs: ...).\n\n"
|
||||
> "Recent History:\n"
|
||||
> "{history}\n\n"
|
||||
> "Diff:\n"
|
||||
> "{diff}\n\n"
|
||||
> "Commit Message:"
|
||||
> ),
|
||||
> }
|
||||
# #endregion DEFAULT_LLM_PROMPTS
|
||||
|
||||
|
||||
# #region DEFAULT_LLM_PROVIDER_BINDINGS [C:2] [TYPE Constant]
|
||||
# @ingroup Services
|
||||
# @BRIEF Default provider binding per task domain.
|
||||
> DEFAULT_LLM_PROVIDER_BINDINGS: dict[str, str] = {
|
||||
> "documentation": "",
|
||||
> "git_commit": "",
|
||||
> }
|
||||
# #endregion DEFAULT_LLM_PROVIDER_BINDINGS
|
||||
|
||||
|
||||
# #region DEFAULT_LLM_ASSISTANT_SETTINGS [C:2] [TYPE Constant]
|
||||
# @ingroup Services
|
||||
# @BRIEF Default planner settings for assistant chat intent model/provider resolution.
|
||||
> DEFAULT_LLM_ASSISTANT_SETTINGS: dict[str, str] = {
|
||||
> "assistant_planner_provider": "",
|
||||
> "assistant_planner_model": "",
|
||||
> }
|
||||
# #endregion DEFAULT_LLM_ASSISTANT_SETTINGS
|
||||
|
||||
|
||||
# #region normalize_llm_settings [C:3] [TYPE Function]
|
||||
# @ingroup Services
|
||||
# @BRIEF Ensure llm settings contain stable schema with prompts section and default templates.
|
||||
# @PRE llm_settings is dictionary-like value or None.
|
||||
# @POST Returned dict contains prompts with all required template keys.
|
||||
# @RELATION DEPENDS_ON -> LLMProviderService
|
||||
> def normalize_llm_settings(llm_settings: Any) -> dict[str, Any]:
|
||||
> normalized: dict[str, Any] = {
|
||||
> "providers": [],
|
||||
> "default_provider": "",
|
||||
> "prompts": {},
|
||||
> "provider_bindings": {},
|
||||
> **DEFAULT_LLM_ASSISTANT_SETTINGS,
|
||||
> }
|
||||
> if isinstance(llm_settings, dict):
|
||||
> normalized.update(
|
||||
> {
|
||||
> k: v
|
||||
> for k, v in llm_settings.items()
|
||||
> if k
|
||||
> in (
|
||||
> "providers",
|
||||
> "default_provider",
|
||||
> "prompts",
|
||||
> "provider_bindings",
|
||||
> "assistant_planner_provider",
|
||||
> "assistant_planner_model",
|
||||
> )
|
||||
> }
|
||||
> )
|
||||
> prompts = normalized.get("prompts") if isinstance(normalized.get("prompts"), dict) else {}
|
||||
> merged_prompts = deepcopy(DEFAULT_LLM_PROMPTS)
|
||||
> merged_prompts.update({k: v for k, v in prompts.items() if isinstance(v, str) and v.strip()})
|
||||
> normalized["prompts"] = merged_prompts
|
||||
> bindings = normalized.get("provider_bindings") if isinstance(normalized.get("provider_bindings"), dict) else {}
|
||||
> merged_bindings = deepcopy(DEFAULT_LLM_PROVIDER_BINDINGS)
|
||||
> merged_bindings.update({k: v for k, v in bindings.items() if isinstance(v, str)})
|
||||
> normalized["provider_bindings"] = merged_bindings
|
||||
> for key, default_value in DEFAULT_LLM_ASSISTANT_SETTINGS.items():
|
||||
> value = normalized.get(key, default_value)
|
||||
> normalized[key] = value.strip() if isinstance(value, str) else default_value
|
||||
> return normalized
|
||||
# #endregion normalize_llm_settings
|
||||
|
||||
# #region is_multimodal_model [C:3] [TYPE Function]
|
||||
# @ingroup Services
|
||||
# @BRIEF Heuristically determine whether model supports image input required for dashboard validation.
|
||||
# @DEPRECATED Use the explicit `db_provider.is_multimodal` flag instead (see migration 9f8e7d6c5b4a).
|
||||
# @RATIONALE Added import warnings + warnings.warn(DeprecationWarning) to is_multimodal_model as a deprecation shim.
|
||||
|
||||
# Replaced by an explicit boolean flag on `LLMProvider` that users control via checkbox in UI.
|
||||
# This function is retained only as a backward-compatibility shim for the Alembic migration
|
||||
# backfill and must not be imported in new production code.
|
||||
# @REJECTED Keeping the function as a backward-compatibility shim; do not use for new validation.
|
||||
# @PRE model_name may be empty or mixed-case.
|
||||
# @POST Returns True when model likely supports multimodal input.
|
||||
# @RELATION DEPENDS_ON -> LLMProviderService
|
||||
> def is_multimodal_model(model_name: str, provider_type: str | None = None) -> bool:
|
||||
> warnings.warn(
|
||||
> "is_multimodal_model is deprecated; use LLMProvider.is_multimodal instead",
|
||||
> DeprecationWarning,
|
||||
> stacklevel=2,
|
||||
> )
|
||||
> token = (model_name or "").strip().lower()
|
||||
> if not token:
|
||||
> return False
|
||||
> text_only_markers = (
|
||||
> "text-only",
|
||||
> "embedding",
|
||||
> "rerank",
|
||||
> "whisper",
|
||||
> "tts",
|
||||
> "transcribe",
|
||||
> )
|
||||
> if any(marker in token for marker in text_only_markers):
|
||||
> return False
|
||||
> multimodal_markers = (
|
||||
> "gpt-4o",
|
||||
> "gpt-4.1",
|
||||
> "vision",
|
||||
> "vl",
|
||||
> "gemini",
|
||||
> "claude-3",
|
||||
> "claude-sonnet-4",
|
||||
> "omni",
|
||||
> "multimodal",
|
||||
> "pixtral",
|
||||
> "llava",
|
||||
> "internvl",
|
||||
> "qwen-vl",
|
||||
> "qwen2-vl",
|
||||
> )
|
||||
> return any(marker in token for marker in multimodal_markers)
|
||||
# #endregion is_multimodal_model
|
||||
|
||||
|
||||
# #region resolve_bound_provider_id [C:3] [TYPE Function]
|
||||
# @ingroup Services
|
||||
# @BRIEF Resolve provider id configured for a task binding with fallback to default provider.
|
||||
# @PRE llm_settings is normalized or raw dict from config.
|
||||
# @POST Returns configured provider id or fallback id/empty string when not defined.
|
||||
# @RELATION DEPENDS_ON -> LLMProviderService
|
||||
> def resolve_bound_provider_id(llm_settings: Any, task_key: str) -> str:
|
||||
> normalized = normalize_llm_settings(llm_settings)
|
||||
> bindings = normalized.get("provider_bindings", {})
|
||||
> bound = bindings.get(task_key)
|
||||
> if isinstance(bound, str) and bound.strip():
|
||||
> return bound.strip()
|
||||
> default_provider = normalized.get("default_provider", "")
|
||||
> return default_provider.strip() if isinstance(default_provider, str) else ""
|
||||
# #endregion resolve_bound_provider_id
|
||||
|
||||
|
||||
# #region render_prompt [C:3] [TYPE Function]
|
||||
# @ingroup Services
|
||||
# @BRIEF Render prompt template using deterministic placeholder replacement with graceful fallback.
|
||||
# @PRE template is a string and variables values are already stringifiable.
|
||||
# @POST Returns rendered prompt text with known placeholders substituted. Warns about unfilled placeholders.
|
||||
# @RELATION DEPENDS_ON -> LLMProviderService
|
||||
> def render_prompt(template: str, variables: dict[str, Any]) -> str:
|
||||
> rendered = template
|
||||
> for key, value in variables.items():
|
||||
> rendered = rendered.replace("{" + key + "}", str(value))
|
||||
|
||||
# Warn about unfilled placeholders that would be sent to LLM
|
||||
> import re
|
||||
> unfilled = re.findall(r'\{(\w+)\}', rendered)
|
||||
> if unfilled:
|
||||
> logger.warning(
|
||||
> f"[render_prompt] Unfilled placeholders in rendered prompt: {unfilled}"
|
||||
> )
|
||||
|
||||
> return rendered
|
||||
# #endregion render_prompt
|
||||
|
||||
|
||||
# #endregion llm_prompt_templates
|
||||
267
backend/src/services/llm_provider.py,cover
Normal file
267
backend/src/services/llm_provider.py,cover
Normal file
@@ -0,0 +1,267 @@
|
||||
# #region llm_provider [C:3] [TYPE Module] [SEMANTICS sqlalchemy, llm, provider, encryption, config]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Service for managing LLM provider configurations with encrypted API keys.
|
||||
# @LAYER Domain
|
||||
# @RELATION DEPENDS_ON -> [LLMProvider]
|
||||
# @RELATION DEPENDS_ON -> [EncryptionManager]
|
||||
# @RELATION DEPENDS_ON -> [LLMProviderConfig]
|
||||
# @RATIONALE EncryptionManager imported from core/encryption.py (extracted from this module)
|
||||
# to decouple core encryption from LLM domain — see [SEC:C-1].
|
||||
> from typing import TYPE_CHECKING
|
||||
|
||||
> from cryptography.exceptions import InvalidTag
|
||||
> from sqlalchemy.orm import Session
|
||||
|
||||
> from ..core.encryption import EncryptionManager
|
||||
> from ..core.logger import belief_scope, logger
|
||||
> from ..models.llm import LLMProvider
|
||||
|
||||
- if TYPE_CHECKING:
|
||||
- from ..plugins.llm_analysis.models import LLMProviderConfig
|
||||
|
||||
> MASKED_API_KEY_PLACEHOLDER = "********"
|
||||
|
||||
|
||||
# #region mask_api_key [C:2] [TYPE Function]
|
||||
# @ingroup Services
|
||||
# @BRIEF Mask an API key for safe display, showing first 4 and last 4 characters.
|
||||
# @PRE api_key is a plaintext string or None.
|
||||
# @POST Returns "****" for very short keys; "{first 2}...{last 2}" for <=8 chars;
|
||||
# "{first 4}...{last 4}" for longer keys; "" for None/empty.
|
||||
> def mask_api_key(api_key: str | None) -> str:
|
||||
> if not api_key:
|
||||
> return ""
|
||||
> if len(api_key) <= 4:
|
||||
> return "****"
|
||||
> if len(api_key) <= 8:
|
||||
> return f"{api_key[:2]}...{api_key[-2:]}"
|
||||
> return f"{api_key[:4]}...{api_key[-4:]}"
|
||||
|
||||
|
||||
# #endregion mask_api_key
|
||||
|
||||
|
||||
# #region is_masked_or_placeholder [C:2] [TYPE Function]
|
||||
# @ingroup Services
|
||||
# @BRIEF Predicate: True when api_key is None, empty, "********", or contains "...".
|
||||
# @PRE api_key can be None.
|
||||
# @POST Returns True only for non-real-key values.
|
||||
> def is_masked_or_placeholder(api_key: str | None) -> bool:
|
||||
> if not api_key:
|
||||
> return True
|
||||
> return api_key == MASKED_API_KEY_PLACEHOLDER or "..." in api_key
|
||||
|
||||
|
||||
# #endregion is_masked_or_placeholder
|
||||
|
||||
|
||||
# NOTE: EncryptionManager is now imported from ..core.encryption
|
||||
|
||||
|
||||
# #region LLMProviderService [C:3] [TYPE Class]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Service to manage LLM provider lifecycle.
|
||||
# @RELATION DEPENDS_ON -> [LLMProvider]
|
||||
# @RELATION DEPENDS_ON -> [EncryptionManager]
|
||||
# @RELATION DEPENDS_ON -> [LLMProviderConfig]
|
||||
> class LLMProviderService:
|
||||
# region LLMProviderService_init [TYPE Function]
|
||||
# @PURPOSE: Initialize the service with database session.
|
||||
# @PRE db must be a valid SQLAlchemy Session.
|
||||
# @POST Service ready for provider operations.
|
||||
# @RELATION DEPENDS_ON ->[EncryptionManager]
|
||||
> def __init__(self, db: Session):
|
||||
> self.db = db
|
||||
> self.encryption = EncryptionManager()
|
||||
|
||||
# endregion LLMProviderService_init
|
||||
|
||||
# region get_all_providers [TYPE Function]
|
||||
# @PURPOSE: Returns all configured LLM providers.
|
||||
# @PRE Database connection must be active.
|
||||
# @POST Returns list of all LLMProvider records.
|
||||
# @RELATION DEPENDS_ON ->[LLMProvider]
|
||||
> def get_all_providers(self) -> list[LLMProvider]:
|
||||
> with belief_scope("get_all_providers"):
|
||||
> return self.db.query(LLMProvider).all()
|
||||
|
||||
# endregion get_all_providers
|
||||
|
||||
# region get_provider [TYPE Function]
|
||||
# @PURPOSE: Returns a single LLM provider by ID.
|
||||
# @PRE provider_id must be a valid string.
|
||||
# @POST Returns LLMProvider or None if not found.
|
||||
# @RELATION DEPENDS_ON ->[LLMProvider]
|
||||
> def get_provider(self, provider_id: str) -> LLMProvider | None:
|
||||
> with belief_scope("get_provider"):
|
||||
> return (
|
||||
> self.db.query(LLMProvider).filter(LLMProvider.id == provider_id).first()
|
||||
> )
|
||||
|
||||
# endregion get_provider
|
||||
|
||||
# region create_provider [TYPE Function]
|
||||
# @PURPOSE: Creates a new LLM provider with encrypted API key.
|
||||
# @PRE config must contain valid provider configuration.
|
||||
# @POST New provider created and persisted to database.
|
||||
# @RELATION DEPENDS_ON ->[LLMProviderConfig]
|
||||
# @RELATION DEPENDS_ON ->[LLMProvider]
|
||||
# @RELATION CALLS ->[EXT:method:encrypt]
|
||||
> def create_provider(self, config: "LLMProviderConfig") -> LLMProvider:
|
||||
> with belief_scope("create_provider"):
|
||||
> encrypted_key = self.encryption.encrypt(config.api_key)
|
||||
> db_provider = LLMProvider(
|
||||
> provider_type=config.provider_type.value,
|
||||
> name=config.name,
|
||||
> base_url=config.base_url,
|
||||
> api_key=encrypted_key,
|
||||
> default_model=config.default_model,
|
||||
> is_active=config.is_active,
|
||||
> is_multimodal=config.is_multimodal,
|
||||
> max_images=config.max_images,
|
||||
> context_window=config.context_window,
|
||||
> max_output_tokens=config.max_output_tokens,
|
||||
> )
|
||||
> self.db.add(db_provider)
|
||||
> self.db.commit()
|
||||
> self.db.refresh(db_provider)
|
||||
> return db_provider
|
||||
|
||||
# endregion create_provider
|
||||
|
||||
# region update_provider [TYPE Function]
|
||||
# @PURPOSE: Updates an existing LLM provider.
|
||||
# @PRE provider_id must exist, config must be valid.
|
||||
# @POST Provider updated and persisted to database.
|
||||
# @RELATION DEPENDS_ON ->[LLMProviderConfig]
|
||||
# @RELATION DEPENDS_ON ->[LLMProvider]
|
||||
# @RELATION CALLS ->[EXT:method:encrypt]
|
||||
> def update_provider(
|
||||
> self, provider_id: str, config: "LLMProviderConfig"
|
||||
> ) -> LLMProvider | None:
|
||||
> with belief_scope("update_provider"):
|
||||
> db_provider = self.get_provider(provider_id)
|
||||
> if not db_provider:
|
||||
! return None
|
||||
|
||||
> db_provider.provider_type = config.provider_type.value
|
||||
> db_provider.name = config.name
|
||||
> db_provider.base_url = config.base_url
|
||||
# Ignore masked/placeholder values; they are display-only and must not overwrite secrets.
|
||||
> if not is_masked_or_placeholder(config.api_key):
|
||||
> db_provider.api_key = self.encryption.encrypt(config.api_key)
|
||||
> db_provider.default_model = config.default_model
|
||||
> db_provider.is_active = config.is_active
|
||||
> db_provider.is_multimodal = config.is_multimodal
|
||||
> db_provider.max_images = config.max_images
|
||||
> db_provider.context_window = config.context_window
|
||||
> db_provider.max_output_tokens = config.max_output_tokens
|
||||
|
||||
> self.db.commit()
|
||||
> self.db.refresh(db_provider)
|
||||
> return db_provider
|
||||
|
||||
# endregion update_provider
|
||||
|
||||
# region set_max_images [TYPE Function]
|
||||
# @PURPOSE: Updates only the max_images field for a provider.
|
||||
# @PRE provider_id must exist.
|
||||
# @POST Returns updated provider or None if not found.
|
||||
# @RELATION DEPENDS_ON ->[LLMProvider]
|
||||
> def set_max_images(self, provider_id: str, max_images: int | None) -> LLMProvider | None:
|
||||
> with belief_scope("set_max_images"):
|
||||
> db_provider = self.get_provider(provider_id)
|
||||
> if not db_provider:
|
||||
> return None
|
||||
> db_provider.max_images = max_images
|
||||
> self.db.commit()
|
||||
> self.db.refresh(db_provider)
|
||||
> return db_provider
|
||||
|
||||
# endregion set_max_images
|
||||
|
||||
# region delete_provider [TYPE Function]
|
||||
# @PURPOSE: Deletes an LLM provider.
|
||||
# @PRE provider_id must exist.
|
||||
# @POST Provider removed from database.
|
||||
# @RELATION DEPENDS_ON ->[LLMProvider]
|
||||
> def delete_provider(self, provider_id: str) -> bool:
|
||||
> with belief_scope("delete_provider"):
|
||||
> db_provider = self.get_provider(provider_id)
|
||||
> if not db_provider:
|
||||
> return False
|
||||
> self.db.delete(db_provider)
|
||||
> self.db.commit()
|
||||
> return True
|
||||
|
||||
# endregion delete_provider
|
||||
|
||||
# region get_decrypted_api_key [TYPE Function]
|
||||
# @PURPOSE: Returns the decrypted API key for a provider.
|
||||
# @PRE provider_id must exist with valid encrypted key.
|
||||
# @POST Returns decrypted API key or None on failure.
|
||||
# @RELATION DEPENDS_ON ->[LLMProvider]
|
||||
# @RELATION CALLS ->[EXT:method:decrypt]
|
||||
> def get_decrypted_api_key(self, provider_id: str) -> str | None:
|
||||
> with belief_scope("get_decrypted_api_key"):
|
||||
> db_provider = self.get_provider(provider_id)
|
||||
> if not db_provider:
|
||||
> logger.warning(
|
||||
> f"[get_decrypted_api_key] Provider {provider_id} not found in database"
|
||||
> )
|
||||
> return None
|
||||
|
||||
> logger.info(f"[get_decrypted_api_key] Provider found: {db_provider.id}")
|
||||
> logger.info(
|
||||
> f"[get_decrypted_api_key] Encrypted API key length: {len(db_provider.api_key) if db_provider.api_key else 0}"
|
||||
> )
|
||||
|
||||
> try:
|
||||
> decrypted_key = self.encryption.decrypt(db_provider.api_key)
|
||||
> logger.info(
|
||||
> f"[get_decrypted_api_key] Decryption successful, key length: {len(decrypted_key) if decrypted_key else 0}"
|
||||
> )
|
||||
> return decrypted_key
|
||||
> except InvalidTag as e:
|
||||
! logger.error(
|
||||
! f"[get_decrypted_api_key] Integrity check failed (InvalidTag): {e!s}. "
|
||||
! "The encrypted API key may be corrupted or the ENCRYPTION_KEY has changed."
|
||||
! )
|
||||
! return None
|
||||
> except ValueError as e:
|
||||
! logger.error(
|
||||
! f"[get_decrypted_api_key] Decryption format error (ValueError): {e!s}. "
|
||||
! "The encrypted data may not be valid Fernet ciphertext."
|
||||
! )
|
||||
! return None
|
||||
> except Exception as e:
|
||||
> logger.error(
|
||||
> f"[get_decrypted_api_key] Decryption failed with unexpected error "
|
||||
> f"({type(e).__name__}): {e!s}"
|
||||
> )
|
||||
> return None
|
||||
|
||||
# endregion get_decrypted_api_key
|
||||
|
||||
# region get_provider_token_config [TYPE Function]
|
||||
# @PURPOSE: Returns provider token limits for batch sizing.
|
||||
# @PRE provider_id must be valid.
|
||||
# @POST Returns dict with model name, context_window, max_output_tokens.
|
||||
# Values from DB take priority; None means "use PROVIDER_DEFAULTS fallback".
|
||||
# @RATIONALE Centralised helper — both _batch_proc.py and _batch_sizer.py need
|
||||
# the same resolution logic. Avoids duplicating DB queries and defaults.
|
||||
> def get_provider_token_config(self, provider_id: str) -> dict:
|
||||
! provider = self.get_provider(provider_id)
|
||||
! if not provider:
|
||||
! return {"model": None, "context_window": None, "max_output_tokens": None}
|
||||
! return {
|
||||
! "model": provider.default_model or "gpt-4o-mini",
|
||||
! "context_window": provider.context_window,
|
||||
! "max_output_tokens": provider.max_output_tokens,
|
||||
! }
|
||||
# endregion get_provider_token_config
|
||||
|
||||
|
||||
# #endregion LLMProviderService
|
||||
|
||||
# #endregion llm_provider
|
||||
93
backend/src/services/mapping_service.py,cover
Normal file
93
backend/src/services/mapping_service.py,cover
Normal file
@@ -0,0 +1,93 @@
|
||||
# #region mapping_service [C:5] [TYPE Module] [SEMANTICS mapping, database, superset, fuzzy, suggestion]
|
||||
# @defgroup Services Module group.
|
||||
#
|
||||
# @BRIEF Orchestrates database fetching and fuzzy matching suggestions.
|
||||
# @LAYER Service
|
||||
# @PRE source/target environment identifiers are provided by caller.
|
||||
# @POST Exposes stateless mapping suggestion orchestration over configured environments.
|
||||
# @SIDE_EFFECT Performs remote metadata reads through Superset API clients.
|
||||
# @DATA_CONTRACT Input[source_env_id: str, target_env_id: str] -> Output[List[Dict]]
|
||||
# @RELATION DEPENDS_ON -> SupersetClient
|
||||
# @RELATION DEPENDS_ON -> suggest_mappings
|
||||
#
|
||||
# @INVARIANT Suggestions are based on database names.
|
||||
|
||||
|
||||
> from ..core.logger import belief_scope
|
||||
> from ..core.superset_client import SupersetClient
|
||||
> from ..core.utils.matching import suggest_mappings
|
||||
|
||||
|
||||
# #region MappingService [C:3] [TYPE Class]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Service for handling database mapping logic.
|
||||
# @PRE config_manager exposes get_environments() with environment objects containing id.
|
||||
# @POST Provides client resolution and mapping suggestion methods.
|
||||
# @SIDE_EFFECT Instantiates Superset clients and performs upstream metadata reads.
|
||||
# @DATA_CONTRACT Input[config_manager] -> Output[List[Dict]]
|
||||
# @RELATION DEPENDS_ON -> SupersetClient
|
||||
# @RELATION DEPENDS_ON -> suggest_mappings
|
||||
> class MappingService:
|
||||
# region init [TYPE Function]
|
||||
# @PURPOSE: Initializes the mapping service with a config manager.
|
||||
# @PRE config_manager is provided.
|
||||
# @PARAM config_manager (ConfigManager) - The configuration manager.
|
||||
# @POST Service is initialized.
|
||||
# @RELATION DEPENDS_ON -> MappingService
|
||||
> def __init__(self, config_manager):
|
||||
> with belief_scope("MappingService.__init__"):
|
||||
> self.config_manager = config_manager
|
||||
|
||||
# endregion init
|
||||
|
||||
# region _get_client [TYPE Function]
|
||||
# @PURPOSE: Helper to get an initialized SupersetClient for an environment.
|
||||
# @PARAM env_id (str) - The ID of the environment.
|
||||
# @PRE environment must exist in config.
|
||||
# @POST Returns an initialized SupersetClient.
|
||||
# @RETURN SupersetClient - Initialized client.
|
||||
# @RELATION CALLS -> SupersetClient
|
||||
> def _get_client(self, env_id: str) -> SupersetClient:
|
||||
> with belief_scope("MappingService._get_client", f"env_id={env_id}"):
|
||||
> envs = self.config_manager.get_environments()
|
||||
> env = next((e for e in envs if e.id == env_id), None)
|
||||
> if not env:
|
||||
> raise ValueError(f"Environment {env_id} not found")
|
||||
|
||||
> return SupersetClient(env)
|
||||
|
||||
# endregion _get_client
|
||||
|
||||
# region get_suggestions [TYPE Function]
|
||||
# @PURPOSE: Fetches databases from both environments and returns fuzzy matching suggestions.
|
||||
# @PARAM source_env_id (str) - Source environment ID.
|
||||
# @PARAM target_env_id (str) - Target environment ID.
|
||||
# @PRE Both environments must be accessible.
|
||||
# @POST Returns fuzzy-matched database suggestions.
|
||||
# @RETURN List[Dict] - Suggested mappings.
|
||||
# @RELATION CALLS -> _get_client
|
||||
# @RELATION CALLS -> suggest_mappings
|
||||
> async def get_suggestions(
|
||||
> self, source_env_id: str, target_env_id: str
|
||||
> ) -> list[dict]:
|
||||
> with belief_scope(
|
||||
> "MappingService.get_suggestions",
|
||||
> f"source={source_env_id}, target={target_env_id}",
|
||||
> ):
|
||||
> """
|
||||
> Get suggested mappings between two environments.
|
||||
> """
|
||||
> source_client = self._get_client(source_env_id)
|
||||
> target_client = self._get_client(target_env_id)
|
||||
|
||||
> source_dbs = source_client.get_databases_summary()
|
||||
> target_dbs = target_client.get_databases_summary()
|
||||
|
||||
> return suggest_mappings(source_dbs, target_dbs)
|
||||
|
||||
# endregion get_suggestions
|
||||
|
||||
|
||||
# #endregion MappingService
|
||||
|
||||
# #endregion mapping_service
|
||||
173
backend/src/services/profile_service.py,cover
Normal file
173
backend/src/services/profile_service.py,cover
Normal file
@@ -0,0 +1,173 @@
|
||||
# #region profile_service [C:5] [TYPE Module] [SEMANTICS sqlalchemy, profile, preference, superset, lookup]
|
||||
# @defgroup Services Module group.
|
||||
#
|
||||
# @BRIEF Composite facade orchestrating profile preference persistence, Superset account lookup,
|
||||
# security badges, and deterministic actor matching by delegating to focused sub-services.
|
||||
# @LAYER Domain
|
||||
# @RELATION DEPENDS_ON -> [UserDashboardPreference]
|
||||
# @RELATION DEPENDS_ON -> [ProfilePreferenceResponse]
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [AuthRepositoryModule]
|
||||
# @RELATION DEPENDS_ON -> [User]
|
||||
# @RELATION DEPENDS_ON -> [EXT:Library:sqlalchemy.orm.Session]
|
||||
# @RELATION DEPENDS_ON -> [ProfilePreferenceService]
|
||||
# @RELATION DEPENDS_ON -> [SupersetLookupService]
|
||||
# @RELATION DEPENDS_ON -> [SecurityBadgeService]
|
||||
# @RELATION DEPENDS_ON -> [ProfileUtils]
|
||||
#
|
||||
# @INVARIANT Profile ID needs to be unique per-user session
|
||||
#
|
||||
# @TEST_CONTRACT ProfilePreferenceUpdateRequest -> ProfilePreferenceResponse
|
||||
# @TEST_FIXTURE valid_profile_update -> {"user_id":"u-1","superset_username":"John_Doe","show_only_my_dashboards":true}
|
||||
# @TEST_EDGE enable_without_username -> toggle=true with empty username returns validation error
|
||||
# @TEST_EDGE cross_user_mutation -> attempt to update another user preference returns forbidden
|
||||
# @TEST_EDGE lookup_env_not_found -> unknown environment_id returns not found
|
||||
# @TEST_INVARIANT normalization_consistency -> VERIFIED_BY: [valid_profile_update, enable_without_username]
|
||||
# @DATA_CONTRACT Profile_id -> ProfileInfo; session_id -> valid UUID
|
||||
# @PRE Session is active and valid
|
||||
# @POST Profile with updated fields populated and
|
||||
# @SIDE_EFFECT Database read/write operations
|
||||
# @RATIONALE Decomposed from monolithic 770-line ProfileService into three focused services
|
||||
# (ProfilePreferenceService, SupersetLookupService, SecurityBadgeService) plus pure
|
||||
# utility functions (ProfileUtils) to satisfy INV_7. This module is a thin facade
|
||||
# that preserves the public API contract for all existing callers.
|
||||
# @REJECTED Keeping all three domains in a single ProfileService class was rejected — it violated
|
||||
# INV_7 (770 lines vs 150 max), mixed I/O patterns (DB writes + HTTP calls), and created
|
||||
# unnecessary coupling between preference persistence and Superset network operations.
|
||||
|
||||
> from collections.abc import Iterable
|
||||
> from typing import Any
|
||||
|
||||
> from sqlalchemy.orm import Session
|
||||
|
||||
> from ..models.auth import User
|
||||
> from ..schemas.profile import (
|
||||
> ProfilePreferenceResponse,
|
||||
> ProfilePreferenceUpdateRequest,
|
||||
> SupersetAccountLookupRequest,
|
||||
> SupersetAccountLookupResponse,
|
||||
> )
|
||||
> from .profile_preference_service import ProfilePreferenceService
|
||||
> from .profile_utils import (
|
||||
> EnvironmentNotFoundError,
|
||||
> ProfileAuthorizationError,
|
||||
> ProfileValidationError,
|
||||
> normalize_owner_tokens,
|
||||
> normalize_username,
|
||||
> )
|
||||
> from .security_badge_service import SecurityBadgeService
|
||||
> from .superset_lookup_service import SupersetLookupService
|
||||
|
||||
# Re-export exception classes for backward-compatible imports
|
||||
> __all__ = [
|
||||
> "EnvironmentNotFoundError",
|
||||
> "ProfileAuthorizationError",
|
||||
> "ProfilePreferenceService",
|
||||
> "ProfileService",
|
||||
> "ProfileValidationError",
|
||||
> "SecurityBadgeService",
|
||||
> "SupersetLookupService",
|
||||
> ]
|
||||
|
||||
|
||||
# #region ProfileService [C:4] [TYPE Class] [SEMANTICS profile,facade,delegate,coordinator]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Facade that composes ProfilePreferenceService, SupersetLookupService, and
|
||||
# SecurityBadgeService into a single interface for backward compatibility.
|
||||
# @RELATION DEPENDS_ON -> [ProfilePreferenceService]
|
||||
# @RELATION DEPENDS_ON -> [SupersetLookupService]
|
||||
# @RELATION DEPENDS_ON -> [SecurityBadgeService]
|
||||
# @RELATION DEPENDS_ON -> [ProfileUtils]
|
||||
# @PRE Caller provides authenticated User context for external service methods.
|
||||
# @POST Delegates to sub-services and returns normalized profile/lookup responses.
|
||||
# @SIDE_EFFECT Writes preference records and encrypted tokens; performs external account lookups when requested.
|
||||
# @DATA_CONTRACT Input[User,ProfilePreferenceUpdateRequest|SupersetAccountLookupRequest] -> Output[ProfilePreferenceResponse|SupersetAccountLookupResponse|bool]
|
||||
# @INVARIANT Profile data integrity maintained, cache consistency with database state
|
||||
# @RATIONALE Thin coordinator — all sub-10-line delegating methods. Actual business logic lives
|
||||
# in the injected sub-services.
|
||||
> class ProfileService:
|
||||
> """Facade composing preference, lookup, and security services."""
|
||||
|
||||
# region init [TYPE Function]
|
||||
# @BRIEF Initialize facade and create sub-services.
|
||||
# @PRE db session is active and config_manager supports get_environments().
|
||||
# @POST All sub-services are initialized and ready.
|
||||
> def __init__(self, db: Session, config_manager: Any, plugin_loader: Any = None):
|
||||
> self.preference_service = ProfilePreferenceService(db, config_manager, plugin_loader)
|
||||
> self.lookup_service = SupersetLookupService(config_manager)
|
||||
> self.security_badge_service = SecurityBadgeService(plugin_loader)
|
||||
> self.db = db
|
||||
> self.config_manager = config_manager
|
||||
> self.plugin_loader = plugin_loader
|
||||
|
||||
# endregion init
|
||||
|
||||
# region get_my_preference [TYPE Function]
|
||||
# @BRIEF Delegate to ProfilePreferenceService.
|
||||
> def get_my_preference(self, current_user: User) -> ProfilePreferenceResponse:
|
||||
> return self.preference_service.get_my_preference(current_user)
|
||||
|
||||
# endregion get_my_preference
|
||||
|
||||
# region get_dashboard_filter_binding [TYPE Function]
|
||||
# @BRIEF Delegate to ProfilePreferenceService.
|
||||
> def get_dashboard_filter_binding(self, current_user: User) -> dict:
|
||||
> return self.preference_service.get_dashboard_filter_binding(current_user)
|
||||
|
||||
# endregion get_dashboard_filter_binding
|
||||
|
||||
# region update_my_preference [TYPE Function]
|
||||
# @BRIEF Delegate to ProfilePreferenceService.
|
||||
> def update_my_preference(
|
||||
> self,
|
||||
> current_user: User,
|
||||
> payload: ProfilePreferenceUpdateRequest,
|
||||
> target_user_id: str | None = None,
|
||||
> ) -> ProfilePreferenceResponse:
|
||||
> return self.preference_service.update_my_preference(
|
||||
> current_user, payload, target_user_id
|
||||
> )
|
||||
|
||||
# endregion update_my_preference
|
||||
|
||||
# region lookup_superset_accounts [TYPE Function]
|
||||
# @BRIEF Delegate to SupersetLookupService.
|
||||
# @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API.
|
||||
> async def lookup_superset_accounts(
|
||||
> self,
|
||||
> current_user: User,
|
||||
> request: SupersetAccountLookupRequest,
|
||||
> ) -> SupersetAccountLookupResponse:
|
||||
> return await self.lookup_service.lookup_superset_accounts(current_user, request)
|
||||
|
||||
# endregion lookup_superset_accounts
|
||||
|
||||
# region matches_dashboard_actor [TYPE Function]
|
||||
# @BRIEF Apply trim+case-insensitive actor match across owners OR modified_by.
|
||||
# @PRE bound_username can be empty; owners may contain mixed payload.
|
||||
# @POST Returns True when normalized username matches owners or modified_by.
|
||||
> def matches_dashboard_actor(
|
||||
> self,
|
||||
> bound_username: str | None,
|
||||
> owners: Iterable[Any] | None,
|
||||
> modified_by: str | None,
|
||||
> ) -> bool:
|
||||
> normalized_actor = normalize_username(bound_username)
|
||||
> if not normalized_actor:
|
||||
> return False
|
||||
|
||||
> owner_tokens = normalize_owner_tokens(owners)
|
||||
> modified_token = normalize_username(modified_by)
|
||||
|
||||
> if normalized_actor in owner_tokens:
|
||||
> return True
|
||||
> if modified_token and normalized_actor == modified_token:
|
||||
> return True
|
||||
> return False
|
||||
|
||||
# endregion matches_dashboard_actor
|
||||
|
||||
|
||||
# #endregion ProfileService
|
||||
|
||||
# #endregion profile_service
|
||||
252
backend/src/services/profile_utils.py,cover
Normal file
252
backend/src/services/profile_utils.py,cover
Normal file
@@ -0,0 +1,252 @@
|
||||
# #region ProfileUtils [C:2] [TYPE Module] [SEMANTICS string,normalization,utility,sanitize]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Pure utility helpers for profile data sanitization, normalization, and secret masking.
|
||||
# Also contains shared exception classes to avoid circular imports between profile sub-modules.
|
||||
# @LAYER Domain
|
||||
# @RATIONALE Extracted from ProfileService to satisfy INV_7 — module length under 400 lines,
|
||||
# contract length under 150 lines. These are stateless pure functions that do not
|
||||
# require a class or service container.
|
||||
# @REJECTED Keeping these as private methods on ProfileService was rejected because they
|
||||
# inflate the class beyond 150 lines and mix pure computation with I/O-bound
|
||||
# service logic.
|
||||
|
||||
> from collections.abc import Iterable, Sequence
|
||||
> from datetime import UTC, datetime
|
||||
> from typing import Any
|
||||
|
||||
|
||||
# #region ProfileValidationError [C:2] [TYPE Class]
|
||||
# @defgroup Services Module group.
|
||||
# @RELATION INHERITS -> [EXT:Python:Exception]
|
||||
# @BRIEF Domain validation error for profile preference update requests.
|
||||
> class ProfileValidationError(Exception):
|
||||
> def __init__(self, errors: Sequence[str]):
|
||||
> self.errors = list(errors)
|
||||
> super().__init__("Profile preference validation failed")
|
||||
# #endregion ProfileValidationError
|
||||
|
||||
|
||||
# #region EnvironmentNotFoundError [C:2] [TYPE Class]
|
||||
# @defgroup Services Module group.
|
||||
# @RELATION INHERITS -> [EXT:Python:Exception]
|
||||
# @BRIEF Raised when environment_id from lookup request is unknown in app configuration.
|
||||
> class EnvironmentNotFoundError(Exception):
|
||||
> pass
|
||||
# #endregion EnvironmentNotFoundError
|
||||
|
||||
|
||||
# #region ProfileAuthorizationError [C:2] [TYPE Class]
|
||||
# @defgroup Services Module group.
|
||||
# @RELATION INHERITS -> [EXT:Python:Exception]
|
||||
# @BRIEF Raised when caller attempts cross-user preference mutation.
|
||||
> class ProfileAuthorizationError(Exception):
|
||||
> pass
|
||||
# #endregion ProfileAuthorizationError
|
||||
|
||||
> SUPPORTED_START_PAGES: set[str] = {"dashboards", "datasets", "reports"}
|
||||
> SUPPORTED_DENSITIES: set[str] = {"compact", "comfortable"}
|
||||
|
||||
|
||||
# #region sanitize_text [C:1] [TYPE Function]
|
||||
# @BRIEF Normalize optional text into trimmed form or None.
|
||||
> def sanitize_text(value: str | None) -> str | None:
|
||||
> normalized = str(value or "").strip()
|
||||
> if not normalized:
|
||||
> return None
|
||||
> return normalized
|
||||
# #endregion sanitize_text
|
||||
|
||||
|
||||
# #region sanitize_secret [C:1] [TYPE Function]
|
||||
# @BRIEF Normalize secret input into trimmed form or None.
|
||||
> def sanitize_secret(value: str | None) -> str | None:
|
||||
> if value is None:
|
||||
> return None
|
||||
> normalized = str(value).strip()
|
||||
> if not normalized:
|
||||
> return None
|
||||
> return normalized
|
||||
# #endregion sanitize_secret
|
||||
|
||||
|
||||
# #region sanitize_username [C:1] [TYPE Function]
|
||||
# @BRIEF Normalize raw username into trimmed form or None for empty input.
|
||||
> def sanitize_username(value: str | None) -> str | None:
|
||||
> return sanitize_text(value)
|
||||
# #endregion sanitize_username
|
||||
|
||||
|
||||
# #region normalize_username [C:1] [TYPE Function]
|
||||
# @BRIEF Apply deterministic trim+lower normalization for actor matching.
|
||||
> def normalize_username(value: str | None) -> str | None:
|
||||
> sanitized = sanitize_username(value)
|
||||
> if sanitized is None:
|
||||
> return None
|
||||
> return sanitized.lower()
|
||||
# #endregion normalize_username
|
||||
|
||||
|
||||
# #region normalize_start_page [C:1] [TYPE Function]
|
||||
# @BRIEF Normalize supported start page aliases to canonical values.
|
||||
> def normalize_start_page(value: str | None) -> str:
|
||||
> normalized = str(value or "").strip().lower()
|
||||
> if normalized == "reports-logs":
|
||||
> return "reports"
|
||||
> if normalized in SUPPORTED_START_PAGES:
|
||||
> return normalized
|
||||
> return "dashboards"
|
||||
# #endregion normalize_start_page
|
||||
|
||||
|
||||
# #region normalize_density [C:1] [TYPE Function]
|
||||
# @BRIEF Normalize supported density aliases to canonical values.
|
||||
> def normalize_density(value: str | None) -> str:
|
||||
> normalized = str(value or "").strip().lower()
|
||||
> if normalized == "free":
|
||||
> return "comfortable"
|
||||
> if normalized in SUPPORTED_DENSITIES:
|
||||
> return normalized
|
||||
> return "comfortable"
|
||||
# #endregion normalize_density
|
||||
|
||||
|
||||
# #region mask_secret_value [C:1] [TYPE Function]
|
||||
# @BRIEF Build a safe display value for sensitive secrets.
|
||||
> def mask_secret_value(secret: str | None) -> str | None:
|
||||
> sanitized = sanitize_secret(secret)
|
||||
> if sanitized is None:
|
||||
> return None
|
||||
> if len(sanitized) <= 4:
|
||||
> return "***"
|
||||
> return f"{sanitized[:2]}***{sanitized[-2:]}"
|
||||
# #endregion mask_secret_value
|
||||
|
||||
|
||||
# #region validate_update_payload [C:2] [TYPE Function]
|
||||
# @ingroup Services
|
||||
# @BRIEF Validate username/toggle constraints for preference mutation.
|
||||
# @RELATION CALLS -> [sanitize_username]
|
||||
# @RELATION CALLS -> [sanitize_text]
|
||||
# @RELATION DEPENDS_ON -> [ProfileUtils]
|
||||
# @RELATION DEPENDS_ON -> [ProfileUtils]
|
||||
# @POST Returns validation errors list; empty list means valid.
|
||||
> def validate_update_payload(
|
||||
> superset_username: str | None,
|
||||
> show_only_my_dashboards: bool,
|
||||
> git_email: str | None,
|
||||
> start_page: str,
|
||||
> dashboards_table_density: str,
|
||||
> email_address: str | None = None,
|
||||
> ) -> list[str]:
|
||||
> errors: list[str] = []
|
||||
> sanitized_username = sanitize_username(superset_username)
|
||||
|
||||
> if sanitized_username and any(ch.isspace() for ch in sanitized_username):
|
||||
> errors.append(
|
||||
> "Username should not contain spaces. Please enter a valid Apache Superset username."
|
||||
> )
|
||||
> if show_only_my_dashboards and not sanitized_username:
|
||||
> errors.append(
|
||||
> "Superset username is required when default filter is enabled."
|
||||
> )
|
||||
|
||||
> sanitized_git_email = sanitize_text(git_email)
|
||||
> if sanitized_git_email:
|
||||
> if (
|
||||
> " " in sanitized_git_email
|
||||
> or "@" not in sanitized_git_email
|
||||
> or sanitized_git_email.startswith("@")
|
||||
> or sanitized_git_email.endswith("@")
|
||||
> ):
|
||||
> errors.append("Git email should be a valid email address.")
|
||||
|
||||
> if start_page not in SUPPORTED_START_PAGES:
|
||||
> errors.append("Start page value is not supported.")
|
||||
|
||||
> if dashboards_table_density not in SUPPORTED_DENSITIES:
|
||||
> errors.append("Dashboards table density value is not supported.")
|
||||
|
||||
> sanitized_email = sanitize_text(email_address)
|
||||
> if sanitized_email:
|
||||
> if (
|
||||
> " " in sanitized_email
|
||||
> or "@" not in sanitized_email
|
||||
> or sanitized_email.startswith("@")
|
||||
> or sanitized_email.endswith("@")
|
||||
> ):
|
||||
> errors.append("Notification email should be a valid email address.")
|
||||
|
||||
> return errors
|
||||
# #endregion validate_update_payload
|
||||
|
||||
|
||||
# #region build_default_preference [C:2] [TYPE Function] [SEMANTICS preference,default,unconfigured]
|
||||
# @ingroup Services
|
||||
# @BRIEF Build non-persisted default preference DTO for unconfigured users.
|
||||
# @RELATION DEPENDS_ON -> [ProfilePreference]
|
||||
> def build_default_preference(user_id: str) -> Any:
|
||||
> """Build default ProfilePreference with disabled toggles and empty usernames."""
|
||||
> from ..schemas.profile import ProfilePreference
|
||||
> now = datetime.now(UTC)
|
||||
> return ProfilePreference(
|
||||
> user_id=str(user_id),
|
||||
> superset_username=None,
|
||||
> superset_username_normalized=None,
|
||||
> show_only_my_dashboards=False,
|
||||
> show_only_slug_dashboards=True,
|
||||
> git_username=None,
|
||||
> git_email=None,
|
||||
> has_git_personal_access_token=False,
|
||||
> git_personal_access_token_masked=None,
|
||||
> start_page="dashboards",
|
||||
> auto_open_task_drawer=True,
|
||||
> dashboards_table_density="comfortable",
|
||||
> telegram_id=None,
|
||||
> email_address=None,
|
||||
> notify_on_fail=True,
|
||||
> created_at=now,
|
||||
> updated_at=now,
|
||||
> )
|
||||
# #endregion build_default_preference
|
||||
|
||||
|
||||
# #region normalize_owner_tokens [C:2] [TYPE Function]
|
||||
# @ingroup Services
|
||||
# @BRIEF Normalize owners payload into deduplicated lower-cased tokens.
|
||||
# @RELATION CALLS -> [normalize_username]
|
||||
> def normalize_owner_tokens(owners: Iterable[Any] | None) -> list[str]:
|
||||
> if owners is None:
|
||||
> return []
|
||||
> normalized: list[str] = []
|
||||
> for owner in owners:
|
||||
> owner_candidates: list[Any]
|
||||
> if isinstance(owner, dict):
|
||||
> first_name = sanitize_username(str(owner.get("first_name") or ""))
|
||||
> last_name = sanitize_username(str(owner.get("last_name") or ""))
|
||||
> full_name = " ".join(
|
||||
> part for part in [first_name, last_name] if part
|
||||
> ).strip()
|
||||
> snake_name = "_".join(
|
||||
> part for part in [first_name, last_name] if part
|
||||
> ).strip("_")
|
||||
> owner_candidates = [
|
||||
> owner.get("username"),
|
||||
> owner.get("user_name"),
|
||||
> owner.get("name"),
|
||||
> owner.get("full_name"),
|
||||
> first_name,
|
||||
> last_name,
|
||||
> full_name or None,
|
||||
> snake_name or None,
|
||||
> owner.get("email"),
|
||||
> ]
|
||||
> else:
|
||||
> owner_candidates = [owner]
|
||||
|
||||
> for candidate in owner_candidates:
|
||||
> token = normalize_username(str(candidate or ""))
|
||||
> if token and token not in normalized:
|
||||
> normalized.append(token)
|
||||
> return normalized
|
||||
# #endregion normalize_owner_tokens
|
||||
# #endregion ProfileUtils
|
||||
181
backend/src/services/rbac_permission_catalog.py,cover
Normal file
181
backend/src/services/rbac_permission_catalog.py,cover
Normal file
@@ -0,0 +1,181 @@
|
||||
# #region rbac_permission_catalog [C:2] [TYPE Module] [SEMANTICS rbac, permission, discovery, route, sync]
|
||||
# @defgroup Services Module group.
|
||||
#
|
||||
# @BRIEF Discovers declared RBAC permissions from API routes/plugins and synchronizes them into auth database.
|
||||
|
||||
> from collections.abc import Iterable
|
||||
> from functools import lru_cache
|
||||
> from pathlib import Path
|
||||
> import re
|
||||
|
||||
> from sqlalchemy.orm import Session
|
||||
|
||||
> from ..core.logger import belief_scope, logger
|
||||
> from ..models.auth import Permission
|
||||
|
||||
# #region HAS_PERMISSION_PATTERN [TYPE Constant]
|
||||
# @ingroup Services
|
||||
# @BRIEF Regex pattern for extracting has_permission("resource", "ACTION") declarations.
|
||||
> HAS_PERMISSION_PATTERN = re.compile(
|
||||
> r"""has_permission\(\s*['"]([^'"]+)['"]\s*,\s*['"]([A-Z]+)['"]\s*\)"""
|
||||
> )
|
||||
# #endregion HAS_PERMISSION_PATTERN
|
||||
|
||||
# #region ROUTES_DIR [TYPE Constant]
|
||||
# @ingroup Services
|
||||
# @BRIEF Absolute directory path where API route RBAC declarations are defined.
|
||||
> ROUTES_DIR = Path(__file__).resolve().parent.parent / "api" / "routes"
|
||||
# #endregion ROUTES_DIR
|
||||
|
||||
|
||||
# #region _iter_route_files [C:4] [TYPE Function]
|
||||
# @BRIEF Iterates API route files that may contain RBAC declarations.
|
||||
# @PRE ROUTES_DIR points to backend/src/api/routes.
|
||||
# @POST Yields Python files excluding test and cache directories.
|
||||
> def _iter_route_files() -> Iterable[Path]:
|
||||
> with belief_scope("rbac_permission_catalog._iter_route_files"):
|
||||
> if not ROUTES_DIR.exists():
|
||||
> return []
|
||||
|
||||
> files = []
|
||||
> for file_path in ROUTES_DIR.rglob("*.py"):
|
||||
> path_parts = set(file_path.parts)
|
||||
> if "__tests__" in path_parts or "__pycache__" in path_parts:
|
||||
> continue
|
||||
> files.append(file_path)
|
||||
> return files
|
||||
# #endregion _iter_route_files
|
||||
|
||||
|
||||
# #region _discover_route_permissions [C:4] [TYPE Function]
|
||||
# @BRIEF Extracts explicit has_permission declarations from API route source code.
|
||||
# @PRE Route files are readable UTF-8 text files.
|
||||
# @POST Returns unique set of (resource, action) pairs declared in route guards.
|
||||
> def _discover_route_permissions() -> set[tuple[str, str]]:
|
||||
> with belief_scope("rbac_permission_catalog._discover_route_permissions"):
|
||||
> discovered: set[tuple[str, str]] = set()
|
||||
> for route_file in _iter_route_files():
|
||||
> try:
|
||||
> source = route_file.read_text(encoding="utf-8")
|
||||
! except OSError as read_error:
|
||||
! logger.explore(
|
||||
! "Failed to read route file",
|
||||
! extra={"src": "rbac_permission_catalog", "payload": {"file": str(route_file)}, "error": str(read_error)},
|
||||
! )
|
||||
! continue
|
||||
|
||||
> for resource, action in HAS_PERMISSION_PATTERN.findall(source):
|
||||
> normalized_resource = str(resource or "").strip()
|
||||
> normalized_action = str(action or "").strip().upper()
|
||||
> if normalized_resource and normalized_action:
|
||||
> discovered.add((normalized_resource, normalized_action))
|
||||
> return discovered
|
||||
# #endregion _discover_route_permissions
|
||||
|
||||
|
||||
# #region _discover_route_permissions_cached [C:4] [TYPE Function]
|
||||
# @BRIEF Cache route permission discovery because route source files are static during normal runtime.
|
||||
# @PRE None.
|
||||
# @POST Returns stable discovered route permission pairs without repeated filesystem scans.
|
||||
> @lru_cache(maxsize=1)
|
||||
> def _discover_route_permissions_cached() -> tuple[tuple[str, str], ...]:
|
||||
> with belief_scope("rbac_permission_catalog._discover_route_permissions_cached"):
|
||||
> return tuple(sorted(_discover_route_permissions()))
|
||||
# #endregion _discover_route_permissions_cached
|
||||
|
||||
|
||||
# #region _discover_plugin_execute_permissions [C:4] [TYPE Function]
|
||||
# @BRIEF Derives dynamic task permissions of form plugin:{plugin_id}:EXECUTE from plugin registry.
|
||||
# @PRE plugin_loader is optional and may expose get_all_plugin_configs.
|
||||
# @POST Returns unique plugin EXECUTE permissions if loader is available.
|
||||
> def _discover_plugin_execute_permissions(plugin_loader=None) -> set[tuple[str, str]]:
|
||||
! with belief_scope("rbac_permission_catalog._discover_plugin_execute_permissions"):
|
||||
! discovered: set[tuple[str, str]] = set()
|
||||
! if plugin_loader is None:
|
||||
! return discovered
|
||||
|
||||
! try:
|
||||
! plugin_configs = plugin_loader.get_all_plugin_configs()
|
||||
! except Exception as plugin_error:
|
||||
! logger.explore(
|
||||
! "Failed to read plugin configs for RBAC discovery",
|
||||
! extra={"src": "rbac_permission_catalog", "error": str(plugin_error)},
|
||||
! )
|
||||
! return discovered
|
||||
|
||||
! for plugin_config in plugin_configs:
|
||||
! plugin_id = str(getattr(plugin_config, "id", "") or "").strip()
|
||||
! if plugin_id:
|
||||
! discovered.add((f"plugin:{plugin_id}", "EXECUTE"))
|
||||
! return discovered
|
||||
# #endregion _discover_plugin_execute_permissions
|
||||
|
||||
|
||||
# #region _discover_plugin_execute_permissions_cached [C:4] [TYPE Function]
|
||||
# @BRIEF Cache dynamic plugin EXECUTE permission pairs by normalized plugin id tuple.
|
||||
# @PRE plugin_ids is a deterministic tuple of plugin ids.
|
||||
# @POST Returns stable permission tuple without repeated plugin catalog expansion.
|
||||
> @lru_cache(maxsize=8)
|
||||
> def _discover_plugin_execute_permissions_cached(
|
||||
> plugin_ids: tuple[str, ...],
|
||||
> ) -> tuple[tuple[str, str], ...]:
|
||||
> with belief_scope("rbac_permission_catalog._discover_plugin_execute_permissions_cached"):
|
||||
> return tuple((f"plugin:{plugin_id}", "EXECUTE") for plugin_id in plugin_ids)
|
||||
# #endregion _discover_plugin_execute_permissions_cached
|
||||
|
||||
|
||||
# #region discover_declared_permissions [C:4] [TYPE Function]
|
||||
# @ingroup Services
|
||||
# @BRIEF Builds canonical RBAC permission catalog from routes and plugin registry.
|
||||
# @PRE plugin_loader may be provided for dynamic task plugin permission discovery.
|
||||
# @POST Returns union of route-declared and dynamic plugin EXECUTE permissions.
|
||||
> def discover_declared_permissions(plugin_loader=None) -> set[tuple[str, str]]:
|
||||
> with belief_scope("rbac_permission_catalog.discover_declared_permissions"):
|
||||
> permissions = set(_discover_route_permissions_cached())
|
||||
> plugin_ids = tuple(
|
||||
> sorted(
|
||||
> {
|
||||
> str(getattr(plugin_config, "id", "") or "").strip()
|
||||
> for plugin_config in (plugin_loader.get_all_plugin_configs() if plugin_loader else [])
|
||||
> if str(getattr(plugin_config, "id", "") or "").strip()
|
||||
> }
|
||||
> )
|
||||
> )
|
||||
> permissions.update(_discover_plugin_execute_permissions_cached(plugin_ids))
|
||||
> return permissions
|
||||
# #endregion discover_declared_permissions
|
||||
|
||||
|
||||
# #region sync_permission_catalog [C:4] [TYPE Function]
|
||||
# @ingroup Services
|
||||
# @BRIEF Persists missing RBAC permission pairs into auth database.
|
||||
# @PRE db is a valid SQLAlchemy session bound to auth database.
|
||||
# @PRE declared_permissions is an iterable of (resource, action) tuples.
|
||||
# @POST Missing permissions are inserted; existing permissions remain untouched.
|
||||
# @SIDE_EFFECT Commits auth database transaction when new permissions are added.
|
||||
> def sync_permission_catalog(
|
||||
> db: Session,
|
||||
> declared_permissions: Iterable[tuple[str, str]],
|
||||
> ) -> int:
|
||||
> with belief_scope("rbac_permission_catalog.sync_permission_catalog"):
|
||||
> normalized_declared: set[tuple[str, str]] = set()
|
||||
> for resource, action in declared_permissions:
|
||||
> normalized_resource = str(resource or "").strip()
|
||||
> normalized_action = str(action or "").strip().upper()
|
||||
> if normalized_resource and normalized_action:
|
||||
> normalized_declared.add((normalized_resource, normalized_action))
|
||||
|
||||
> existing_permissions = db.query(Permission).all()
|
||||
> existing_pairs = {(perm.resource, perm.action.upper()) for perm in existing_permissions}
|
||||
|
||||
> missing_pairs = sorted(normalized_declared - existing_pairs)
|
||||
> for resource, action in missing_pairs:
|
||||
> db.add(Permission(resource=resource, action=action))
|
||||
|
||||
> if missing_pairs:
|
||||
> db.commit()
|
||||
|
||||
> return len(missing_pairs)
|
||||
# #endregion sync_permission_catalog
|
||||
|
||||
# #endregion rbac_permission_catalog
|
||||
518
backend/src/services/resource_service.py,cover
Normal file
518
backend/src/services/resource_service.py,cover
Normal file
@@ -0,0 +1,518 @@
|
||||
# #region ResourceServiceModule [C:5] [TYPE Module] [SEMANTICS resource, git, task, status, superset]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Shared service for fetching resource data with Git status and task status
|
||||
# @LAYER Service
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [TaskManagerPackage]
|
||||
# @RELATION DEPENDS_ON -> [TaskManagerModels]
|
||||
# @RELATION DEPENDS_ON -> [GitService]
|
||||
# @INVARIANT All resources include metadata about their current state
|
||||
# @SIDE_EFFECT Queries multiple backends for status
|
||||
# @DATA_CONTRACT ResourceQuery -> ResourceStatusSummary
|
||||
|
||||
> import asyncio
|
||||
> from datetime import UTC, datetime
|
||||
> from typing import Any
|
||||
|
||||
> from ..core.logger import belief_scope, logger
|
||||
> from ..core.task_manager.models import Task
|
||||
> from ..core.utils.client_registry import get_superset_client
|
||||
> from ..services.git_service import GitService
|
||||
|
||||
|
||||
# #region ResourceService [C:3] [TYPE Class]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Provides centralized access to resource data with enhanced metadata
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [GitService]
|
||||
> class ResourceService:
|
||||
|
||||
# region ResourceService_init [TYPE Function]
|
||||
# @PURPOSE: Initialize the resource service with dependencies
|
||||
# @PRE None
|
||||
# @POST ResourceService is ready to fetch resources
|
||||
> def __init__(self):
|
||||
> with belief_scope("ResourceService.__init__"):
|
||||
> self.git_service = GitService()
|
||||
> logger.reason("Initialized ResourceService", extra={"src": "ResourceService"})
|
||||
# endregion ResourceService_init
|
||||
|
||||
# region get_dashboards_with_status [TYPE Function]
|
||||
# @PURPOSE: Fetch dashboards from environment with Git status and last task status
|
||||
# @PRE env is a valid Environment object
|
||||
# @POST Returns list of dashboards with enhanced metadata
|
||||
# @PARAM env (Environment) - The environment to fetch from
|
||||
# @PARAM tasks (List[Task]) - List of tasks to check for status
|
||||
# @RETURN List[Dict] - Dashboards with git_status and last_task fields
|
||||
# @RELATION CALLS -> [SupersetClientGetDashboardsSummary]
|
||||
# @RELATION CALLS ->[EXT:method:_get_git_status_for_dashboard]
|
||||
# @RELATION CALLS ->[EXT:method:_get_last_llm_task_for_dashboard]
|
||||
> async def get_dashboards_with_status(
|
||||
> self,
|
||||
> env: Any,
|
||||
> tasks: list[Task] | None = None,
|
||||
> include_git_status: bool = True,
|
||||
> require_slug: bool = False,
|
||||
> ) -> list[dict[str, Any]]:
|
||||
> with belief_scope("get_dashboards_with_status", f"env={env.id}"):
|
||||
> client = await get_superset_client(env)
|
||||
> dashboards = await client.get_dashboards_summary(require_slug=require_slug)
|
||||
|
||||
# Enhance each dashboard with Git status and task status
|
||||
> result = []
|
||||
> for dashboard in dashboards:
|
||||
# dashboard is already a dict, no need to call .dict()
|
||||
> dashboard_dict = dashboard
|
||||
> dashboard_id = dashboard_dict.get('id')
|
||||
|
||||
# Git status can be skipped for list endpoints and loaded lazily on UI side.
|
||||
> if include_git_status:
|
||||
> git_status = self._get_git_status_for_dashboard(dashboard_id)
|
||||
> dashboard_dict['git_status'] = git_status
|
||||
> else:
|
||||
> dashboard_dict['git_status'] = None
|
||||
|
||||
# Show status of the latest LLM validation for this dashboard.
|
||||
> last_task = self._get_last_llm_task_for_dashboard(
|
||||
> dashboard_id,
|
||||
> env.id,
|
||||
> tasks,
|
||||
> )
|
||||
> dashboard_dict['last_task'] = last_task
|
||||
|
||||
> result.append(dashboard_dict)
|
||||
|
||||
> logger.info(f"[ResourceService][Coherence:OK] Fetched {len(result)} dashboards with status")
|
||||
> return result
|
||||
# endregion get_dashboards_with_status
|
||||
|
||||
# region get_dashboards_page_with_status [TYPE Function]
|
||||
# @PURPOSE: Fetch one dashboard page from environment and enrich only that page with status metadata.
|
||||
# @PRE env is valid; page >= 1; page_size > 0.
|
||||
# @POST Returns page items plus total counters without scanning all pages locally.
|
||||
# @PARAM env (Environment) - Source environment.
|
||||
# @PARAM tasks (Optional[List[Task]]) - Tasks for latest LLM status.
|
||||
# @PARAM page (int) - 1-based page number.
|
||||
# @PARAM page_size (int) - Page size.
|
||||
# @RETURN Dict[str, Any] - {"dashboards": List[Dict], "total": int, "total_pages": int}
|
||||
# @RELATION CALLS -> [SupersetClientGetDashboardsSummaryPage]
|
||||
# @RELATION CALLS ->[EXT:method:_get_git_status_for_dashboard]
|
||||
# @RELATION CALLS ->[EXT:method:_get_last_llm_task_for_dashboard]
|
||||
> async def get_dashboards_page_with_status(
|
||||
> self,
|
||||
> env: Any,
|
||||
> tasks: list[Task] | None = None,
|
||||
> page: int = 1,
|
||||
> page_size: int = 10,
|
||||
> search: str | None = None,
|
||||
> include_git_status: bool = True,
|
||||
> require_slug: bool = False,
|
||||
> ) -> dict[str, Any]:
|
||||
> with belief_scope(
|
||||
> "get_dashboards_page_with_status",
|
||||
> f"env={env.id}, page={page}, page_size={page_size}, search={search}",
|
||||
> ):
|
||||
> client = await get_superset_client(env)
|
||||
> total, dashboards_page = await client.get_dashboards_summary_page(
|
||||
> page=page,
|
||||
> page_size=page_size,
|
||||
> search=search,
|
||||
> require_slug=require_slug,
|
||||
> )
|
||||
|
||||
> result = []
|
||||
> for dashboard in dashboards_page:
|
||||
> dashboard_dict = dashboard
|
||||
> dashboard_id = dashboard_dict.get("id")
|
||||
|
||||
> if include_git_status:
|
||||
> dashboard_dict["git_status"] = self._get_git_status_for_dashboard(dashboard_id)
|
||||
> else:
|
||||
> dashboard_dict["git_status"] = None
|
||||
|
||||
> dashboard_dict["last_task"] = self._get_last_llm_task_for_dashboard(
|
||||
> dashboard_id,
|
||||
> env.id,
|
||||
> tasks,
|
||||
> )
|
||||
> result.append(dashboard_dict)
|
||||
|
||||
> total_pages = (total + page_size - 1) // page_size if total > 0 else 1
|
||||
> logger.info(
|
||||
> "[ResourceService][Coherence:OK] Fetched dashboards page %s/%s (%s items, total=%s)",
|
||||
> page,
|
||||
> total_pages,
|
||||
> len(result),
|
||||
> total,
|
||||
> )
|
||||
> return {
|
||||
> "dashboards": result,
|
||||
> "total": total,
|
||||
> "total_pages": total_pages,
|
||||
> }
|
||||
# endregion get_dashboards_page_with_status
|
||||
|
||||
# region _get_last_llm_task_for_dashboard [TYPE Function]
|
||||
# @PURPOSE: Get most recent LLM validation task for a dashboard in an environment
|
||||
# @PRE dashboard_id is a valid integer identifier
|
||||
# @POST Returns the newest llm_dashboard_validation task summary or None
|
||||
# @PARAM dashboard_id (int) - The dashboard ID
|
||||
# @PARAM env_id (Optional[str]) - Environment ID to match task params
|
||||
# @PARAM tasks (Optional[List[Task]]) - List of tasks to search
|
||||
# @RETURN Optional[Dict] - Task summary with task_id and status
|
||||
# @RELATION CALLS ->[EXT:method:_normalize_datetime_for_compare]
|
||||
# @RELATION CALLS ->[EXT:method:_normalize_validation_status]
|
||||
# @RELATION CALLS ->[EXT:method:_normalize_task_status]
|
||||
> def _get_last_llm_task_for_dashboard(
|
||||
> self,
|
||||
> dashboard_id: int,
|
||||
> env_id: str | None,
|
||||
> tasks: list[Task] | None = None,
|
||||
> ) -> dict[str, Any] | None:
|
||||
> if not tasks:
|
||||
> return None
|
||||
|
||||
> dashboard_id_str = str(dashboard_id)
|
||||
> matched_tasks = []
|
||||
|
||||
> for task in tasks:
|
||||
> if getattr(task, "plugin_id", None) != "llm_dashboard_validation":
|
||||
> continue
|
||||
|
||||
> params = getattr(task, "params", {}) or {}
|
||||
> if str(params.get("dashboard_id")) != dashboard_id_str:
|
||||
> continue
|
||||
|
||||
> if env_id is not None:
|
||||
> task_env = params.get("environment_id") or params.get("env")
|
||||
> if str(task_env) != str(env_id):
|
||||
> continue
|
||||
|
||||
> matched_tasks.append(task)
|
||||
|
||||
> if not matched_tasks:
|
||||
> return None
|
||||
|
||||
> def _task_time(task_obj: Any) -> datetime:
|
||||
> raw_time = (
|
||||
> getattr(task_obj, "started_at", None)
|
||||
> or getattr(task_obj, "finished_at", None)
|
||||
> or getattr(task_obj, "created_at", None)
|
||||
> )
|
||||
> return self._normalize_datetime_for_compare(raw_time)
|
||||
|
||||
> projected_tasks = []
|
||||
> for task in matched_tasks:
|
||||
> raw_result = getattr(task, "result", None)
|
||||
> validation_status = None
|
||||
> if isinstance(raw_result, dict):
|
||||
> validation_status = self._normalize_validation_status(raw_result.get("status"))
|
||||
> projected_tasks.append(
|
||||
> (
|
||||
> task,
|
||||
> validation_status,
|
||||
> _task_time(task),
|
||||
> )
|
||||
> )
|
||||
|
||||
> projected_tasks.sort(key=lambda item: item[2], reverse=True)
|
||||
> latest_task, latest_validation_status, _ = projected_tasks[0]
|
||||
> decisive_task = next(
|
||||
> (
|
||||
> item for item in projected_tasks
|
||||
> if item[1] in {"PASS", "WARN", "FAIL"}
|
||||
> ),
|
||||
> None,
|
||||
> )
|
||||
> validation_status = latest_validation_status
|
||||
> if validation_status == "UNKNOWN" and decisive_task is not None:
|
||||
> validation_status = decisive_task[1]
|
||||
|
||||
> return {
|
||||
> "task_id": str(getattr(latest_task, "id", "")),
|
||||
> "status": self._normalize_task_status(getattr(latest_task, "status", "")),
|
||||
> "validation_status": validation_status,
|
||||
> }
|
||||
# endregion _get_last_llm_task_for_dashboard
|
||||
|
||||
# region _normalize_task_status [TYPE Function]
|
||||
# @PURPOSE: Normalize task status to stable uppercase values for UI/API projections
|
||||
# @PRE raw_status can be enum or string
|
||||
# @POST Returns uppercase status without enum class prefix
|
||||
# @PARAM raw_status (Any) - Raw task status object/value
|
||||
# @RETURN str - Normalized status token
|
||||
# @RELATION CALLED_BY -> [EXT:method:_get_last_llm_task_for_dashboard]
|
||||
> def _normalize_task_status(self, raw_status: Any) -> str:
|
||||
> if raw_status is None:
|
||||
> return ""
|
||||
> value = getattr(raw_status, "value", raw_status)
|
||||
> status_text = str(value).strip()
|
||||
> if "." in status_text:
|
||||
> status_text = status_text.split(".")[-1]
|
||||
> return status_text.upper()
|
||||
# endregion _normalize_task_status
|
||||
|
||||
# region _normalize_validation_status [TYPE Function]
|
||||
# @PURPOSE: Normalize LLM validation status to PASS/FAIL/WARN/UNKNOWN
|
||||
# @PRE raw_status can be any scalar type
|
||||
# @POST Returns normalized validation status token or None
|
||||
# @PARAM raw_status (Any) - Raw validation status from task result
|
||||
# @RETURN Optional[str] - PASS|FAIL|WARN|UNKNOWN
|
||||
# @RELATION CALLED_BY -> [EXT:method:_get_last_llm_task_for_dashboard]
|
||||
> def _normalize_validation_status(self, raw_status: Any) -> str | None:
|
||||
> if raw_status is None:
|
||||
> return None
|
||||
> status_text = str(raw_status).strip().upper()
|
||||
> if status_text in {"PASS", "FAIL", "WARN"}:
|
||||
> return status_text
|
||||
> return "UNKNOWN"
|
||||
# endregion _normalize_validation_status
|
||||
|
||||
# region _normalize_datetime_for_compare [TYPE Function]
|
||||
# @PURPOSE: Normalize datetime values to UTC-aware values for safe comparisons.
|
||||
# @PRE value may be datetime or any scalar.
|
||||
# @POST Returns UTC-aware datetime; non-datetime values map to minimal UTC datetime.
|
||||
# @PARAM value (Any) - Candidate datetime-like value.
|
||||
# @RETURN datetime - UTC-aware comparable datetime.
|
||||
# @RELATION CALLED_BY -> [EXT:method:_get_last_llm_task_for_dashboard]
|
||||
# @RELATION CALLED_BY -> [EXT:method:_get_last_task_for_resource]
|
||||
> def _normalize_datetime_for_compare(self, value: Any) -> datetime:
|
||||
> if isinstance(value, datetime):
|
||||
> if value.tzinfo is None:
|
||||
> return value.replace(tzinfo=UTC)
|
||||
> return value.astimezone(UTC)
|
||||
> return datetime.min.replace(tzinfo=UTC)
|
||||
# endregion _normalize_datetime_for_compare
|
||||
|
||||
# region get_datasets_with_status [TYPE Function]
|
||||
# @PURPOSE: Fetch datasets from environment with mapping progress and last task status
|
||||
# @PRE env is a valid Environment object
|
||||
# @POST Returns list of datasets with enhanced metadata
|
||||
# @PARAM env (Environment) - The environment to fetch from
|
||||
# @PARAM tasks (List[Task]) - List of tasks to check for status
|
||||
# @RETURN List[Dict] - Datasets with mapped_fields, last_task and linked_dashboard_count fields
|
||||
# @RELATION CALLS -> [SupersetClientGetDatasetsSummary]
|
||||
# @RELATION CALLS -> [SupersetClientGetDatasetLinkedDashboardCount]
|
||||
# @RELATION CALLS ->[EXT:method:_get_last_task_for_resource]
|
||||
# @RATIONALE linked_dashboard_count was missing — get_datasets_summary() only returns id/table_name/schema/database
|
||||
# StatsBar showed linked_count=0 because the field was never populated in list endpoint.
|
||||
# Fix: fetch /dataset/{id}/related_objects for each dataset concurrently (semaphore=3, timeout=10s).
|
||||
# @REJECTED N+1 blocking calls rejected — would timeout at 26×1s. Semaphore + asyncio.to_thread + timeout prevents overload.
|
||||
> async def get_datasets_with_status(
|
||||
> self,
|
||||
> env: Any,
|
||||
> tasks: list[Task] | None = None
|
||||
> ) -> list[dict[str, Any]]:
|
||||
> with belief_scope("get_datasets_with_status", f"env={env.id}"):
|
||||
> client = await get_superset_client(env)
|
||||
> datasets = await client.get_datasets_summary()
|
||||
|
||||
# Enhance each dataset with task status and linked dashboard count
|
||||
> sem = asyncio.Semaphore(3) # limit concurrent Superset calls
|
||||
|
||||
> async def fetch_linked_count(ds_id: int) -> int:
|
||||
> async with sem:
|
||||
> try:
|
||||
> return await asyncio.wait_for(
|
||||
> client.get_dataset_linked_dashboard_count(ds_id),
|
||||
> timeout=10.0,
|
||||
> )
|
||||
> except (TimeoutError, Exception):
|
||||
> logger.warning(
|
||||
> f"[EXT:method:get_datasets_with_status][Warning] "
|
||||
> f"Failed to fetch linked dashboard count for dataset {ds_id}"
|
||||
> )
|
||||
> return 0
|
||||
|
||||
> result = []
|
||||
> linked_tasks = []
|
||||
> for dataset in datasets:
|
||||
> dataset_dict = dataset
|
||||
> dataset_id = dataset_dict.get('id')
|
||||
|
||||
# Get last task status
|
||||
> last_task = self._get_last_task_for_resource(
|
||||
> f"dataset-{dataset_id}",
|
||||
> tasks
|
||||
> )
|
||||
> dataset_dict['last_task'] = last_task
|
||||
|
||||
> result.append(dataset_dict)
|
||||
> linked_tasks.append(fetch_linked_count(dataset_id))
|
||||
|
||||
# Fetch linked dashboard counts concurrently — FR-022
|
||||
> if linked_tasks:
|
||||
> linked_counts = await asyncio.gather(*linked_tasks)
|
||||
> for ds, count in zip(result, linked_counts):
|
||||
> ds['linked_dashboard_count'] = count
|
||||
|
||||
> logger.info(
|
||||
> f"[ResourceService][Coherence:OK] "
|
||||
> f"Fetched {len(result)} datasets with status and linked counts"
|
||||
> )
|
||||
> return result
|
||||
# endregion get_datasets_with_status
|
||||
|
||||
# region get_activity_summary [TYPE Function]
|
||||
# @PURPOSE: Get summary of active and recent tasks for the activity indicator
|
||||
# @PRE tasks is a list of Task objects
|
||||
# @POST Returns summary with active_count and recent_tasks
|
||||
# @PARAM tasks (List[Task]) - List of tasks to summarize
|
||||
# @RETURN Dict - Activity summary
|
||||
# @RELATION CALLS ->[EXT:method:_extract_resource_name_from_task]
|
||||
# @RELATION CALLS ->[EXT:method:_extract_resource_type_from_task]
|
||||
> def get_activity_summary(self, tasks: list[Task]) -> dict[str, Any]:
|
||||
> with belief_scope("get_activity_summary"):
|
||||
# Count active (RUNNING, WAITING_INPUT) tasks
|
||||
> active_tasks = [
|
||||
> t for t in tasks
|
||||
> if t.status in ['RUNNING', 'WAITING_INPUT']
|
||||
> ]
|
||||
|
||||
# Get recent tasks (last 5)
|
||||
> recent_tasks = sorted(
|
||||
> tasks,
|
||||
> key=lambda t: t.created_at,
|
||||
> reverse=True
|
||||
> )[:5]
|
||||
|
||||
# Format recent tasks for frontend
|
||||
> recent_tasks_formatted = []
|
||||
> for task in recent_tasks:
|
||||
> resource_name = self._extract_resource_name_from_task(task)
|
||||
> recent_tasks_formatted.append({
|
||||
> 'task_id': str(task.id),
|
||||
> 'resource_name': resource_name,
|
||||
> 'resource_type': self._extract_resource_type_from_task(task),
|
||||
> 'status': task.status,
|
||||
> 'started_at': task.created_at.isoformat() if task.created_at else None
|
||||
> })
|
||||
|
||||
> return {
|
||||
> 'active_count': len(active_tasks),
|
||||
> 'recent_tasks': recent_tasks_formatted
|
||||
> }
|
||||
# endregion get_activity_summary
|
||||
|
||||
# region _get_git_status_for_dashboard [TYPE Function]
|
||||
# @PURPOSE: Get Git sync status for a dashboard
|
||||
# @PRE dashboard_id is a valid integer
|
||||
# @POST Returns git status or None if no repo exists
|
||||
# @PARAM dashboard_id (int) - The dashboard ID
|
||||
# @RETURN Optional[Dict] - Git status with branch and sync_status
|
||||
# @RELATION CALLS ->[EXT:method:get_repo]
|
||||
> def _get_git_status_for_dashboard(self, dashboard_id: int) -> dict[str, Any] | None:
|
||||
> try:
|
||||
> repo = self.git_service.get_repo(dashboard_id)
|
||||
> if not repo:
|
||||
> return {
|
||||
> 'branch': None,
|
||||
> 'sync_status': 'NO_REPO',
|
||||
> 'has_repo': False,
|
||||
> 'has_changes_for_commit': False
|
||||
> }
|
||||
|
||||
# Check if there are uncommitted changes
|
||||
> try:
|
||||
# Get current branch
|
||||
> branch = repo.active_branch.name
|
||||
|
||||
# Check for uncommitted changes
|
||||
> is_dirty = repo.is_dirty()
|
||||
> has_changes_for_commit = repo.is_dirty(untracked_files=True)
|
||||
|
||||
# Check for unpushed commits
|
||||
> unpushed = len(list(repo.iter_commits(f'{branch}@{{u}}..{branch}'))) if '@{u}' in str(repo.refs) else 0
|
||||
|
||||
> if is_dirty or unpushed > 0:
|
||||
> sync_status = 'DIFF'
|
||||
> else:
|
||||
> sync_status = 'OK'
|
||||
|
||||
> return {
|
||||
> 'branch': branch,
|
||||
> 'sync_status': sync_status,
|
||||
> 'has_repo': True,
|
||||
> 'has_changes_for_commit': has_changes_for_commit
|
||||
> }
|
||||
> except Exception:
|
||||
> logger.warning(f"[ResourceService][Warning] Failed to get git status for dashboard {dashboard_id}")
|
||||
> return {
|
||||
> 'branch': None,
|
||||
> 'sync_status': 'ERROR',
|
||||
> 'has_repo': True,
|
||||
> 'has_changes_for_commit': False
|
||||
> }
|
||||
> except Exception:
|
||||
# No repo exists for this dashboard
|
||||
> return {
|
||||
> 'branch': None,
|
||||
> 'sync_status': 'NO_REPO',
|
||||
> 'has_repo': False,
|
||||
> 'has_changes_for_commit': False
|
||||
> }
|
||||
# endregion _get_git_status_for_dashboard
|
||||
|
||||
# region _get_last_task_for_resource [TYPE Function]
|
||||
# @PURPOSE: Get the most recent task for a specific resource
|
||||
# @PRE resource_id is a valid string
|
||||
# @POST Returns task summary or None if no tasks found
|
||||
# @PARAM resource_id (str) - The resource identifier (e.g., "dashboard-123")
|
||||
# @PARAM tasks (Optional[List[Task]]) - List of tasks to search
|
||||
# @RETURN Optional[Dict] - Task summary with task_id and status
|
||||
# @RELATION CALLS ->[EXT:method:_normalize_datetime_for_compare]
|
||||
> def _get_last_task_for_resource(
|
||||
> self,
|
||||
> resource_id: str,
|
||||
> tasks: list[Task] | None = None
|
||||
> ) -> dict[str, Any] | None:
|
||||
> if not tasks:
|
||||
> return None
|
||||
|
||||
# Filter tasks for this resource
|
||||
> resource_tasks = []
|
||||
> for task in tasks:
|
||||
> params = task.params or {}
|
||||
> if params.get('resource_id') == resource_id:
|
||||
> resource_tasks.append(task)
|
||||
|
||||
> if not resource_tasks:
|
||||
> return None
|
||||
|
||||
# Get most recent task with timezone-safe comparison.
|
||||
> last_task = max(
|
||||
> resource_tasks,
|
||||
> key=lambda t: self._normalize_datetime_for_compare(getattr(t, "created_at", None)),
|
||||
> )
|
||||
|
||||
> return {
|
||||
> 'task_id': str(last_task.id),
|
||||
> 'status': last_task.status
|
||||
> }
|
||||
# endregion _get_last_task_for_resource
|
||||
|
||||
# region _extract_resource_name_from_task [TYPE Function]
|
||||
# @PURPOSE: Extract resource name from task params
|
||||
# @PRE task is a valid Task object
|
||||
# @POST Returns resource name or task ID
|
||||
# @PARAM task (Task) - The task to extract from
|
||||
# @RETURN str - Resource name or fallback
|
||||
# @RELATION CALLED_BY -> [EXT:method:get_activity_summary]
|
||||
> def _extract_resource_name_from_task(self, task: Task) -> str:
|
||||
> params = task.params or {}
|
||||
> return params.get('resource_name', f"Task {task.id}")
|
||||
# endregion _extract_resource_name_from_task
|
||||
|
||||
# region _extract_resource_type_from_task [TYPE Function]
|
||||
# @PURPOSE: Extract resource type from task params
|
||||
# @PRE task is a valid Task object
|
||||
# @POST Returns resource type or 'unknown'
|
||||
# @PARAM task (Task) - The task to extract from
|
||||
# @RETURN str - Resource type
|
||||
# @RELATION CALLED_BY -> [EXT:method:get_activity_summary]
|
||||
> def _extract_resource_type_from_task(self, task: Task) -> str:
|
||||
> params = task.params or {}
|
||||
> return params.get('resource_type', 'unknown')
|
||||
# endregion _extract_resource_type_from_task
|
||||
# #endregion ResourceService
|
||||
# #endregion ResourceServiceModule
|
||||
137
backend/src/services/security_badge_service.py,cover
Normal file
137
backend/src/services/security_badge_service.py,cover
Normal file
@@ -0,0 +1,137 @@
|
||||
# #region security_badge_service [C:4] [TYPE Module] [SEMANTICS profile,security,permissions,badges]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Builds security summary and permission badges for profile UI — role extraction,
|
||||
# permission pair collection, and catalog formatting.
|
||||
# @LAYER Domain
|
||||
# @RELATION DEPENDS_ON -> [User]
|
||||
# @RELATION DEPENDS_ON -> [discover_declared_permissions]
|
||||
# @RELATION DEPENDS_ON -> [ProfileUtils]
|
||||
# @RATIONALE Extracted from ProfileService to satisfy INV_7. Security badge construction
|
||||
# is a distinct concern with its own dependencies (discover_declared_permissions,
|
||||
# plugin_loader) and can be tested independently.
|
||||
# @REJECTED Keeping security badge logic inside ProfileService was rejected — it adds
|
||||
# plugin_loader coupling to preference operations that do not need it and
|
||||
# inflates the class past 150 lines.
|
||||
# @PRE Plugin loader may be None; user must have roles/permissions graph.
|
||||
# @POST Returns deterministic security projection for profile UI with sorted permissions.
|
||||
# @SIDE_EFFECT Reads plugin permissions via discover_declared_permissions when plugin_loader is set.
|
||||
|
||||
> from typing import Any
|
||||
|
||||
> from ..core.logger import belief_scope, logger
|
||||
> from ..models.auth import User
|
||||
> from ..schemas.profile import ProfilePermissionState, ProfileSecuritySummary
|
||||
> from .profile_utils import sanitize_text
|
||||
> from .rbac_permission_catalog import discover_declared_permissions
|
||||
|
||||
|
||||
# #region SecurityBadgeService [C:4] [TYPE Class] [SEMANTICS security,permission,badge,profile]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Builds security summary with role names and permission badges for profile UI display.
|
||||
# @RELATION DEPENDS_ON -> [User]
|
||||
# @RELATION CALLS -> [discover_declared_permissions]
|
||||
# @PRE plugin_loader may be None for degraded permission discovery.
|
||||
# @POST build_security_summary returns ProfileSecuritySummary with deterministic sorted permissions.
|
||||
> class SecurityBadgeService:
|
||||
> """Builds security summary with role names and permission badges."""
|
||||
|
||||
# #region __init__ [TYPE Function]
|
||||
# @BRIEF Initialize service with optional plugin loader for permission discovery.
|
||||
> def __init__(self, plugin_loader: Any = None):
|
||||
> self.plugin_loader = plugin_loader
|
||||
# #endregion __init__
|
||||
|
||||
# #region build_security_summary [TYPE Function]
|
||||
# @ingroup Services
|
||||
# @BRIEF Build read-only security snapshot with role and permission badges.
|
||||
# @PRE current_user is authenticated.
|
||||
# @POST Returns deterministic security projection for profile UI.
|
||||
> def build_security_summary(self, current_user: User) -> ProfileSecuritySummary:
|
||||
> with belief_scope("SecurityBadgeService.build_security_summary"):
|
||||
> role_names_set: set[str] = set()
|
||||
> roles = getattr(current_user, "roles", []) or []
|
||||
> for role in roles:
|
||||
> normalized_role_name = sanitize_text(getattr(role, "name", None))
|
||||
> if normalized_role_name:
|
||||
> role_names_set.add(normalized_role_name)
|
||||
> role_names = sorted(role_names_set)
|
||||
|
||||
> is_admin = any(str(role_name).lower() == "admin" for role_name in role_names)
|
||||
> user_permission_pairs = self._collect_user_permission_pairs(current_user)
|
||||
|
||||
> declared_permission_pairs: set[tuple[str, str]] = set()
|
||||
> try:
|
||||
> discovered_permissions = discover_declared_permissions(
|
||||
> plugin_loader=self.plugin_loader
|
||||
> )
|
||||
> for resource, action in discovered_permissions:
|
||||
> normalized_resource = sanitize_text(resource)
|
||||
> normalized_action = str(action or "").strip().upper()
|
||||
> if normalized_resource and normalized_action:
|
||||
> declared_permission_pairs.add(
|
||||
> (normalized_resource, normalized_action)
|
||||
> )
|
||||
> except Exception as discovery_error:
|
||||
> logger.explore(
|
||||
> "Failed to build declared permission catalog",
|
||||
> extra={"error": str(discovery_error), "src": "SecurityBadgeService.build_security_summary"},
|
||||
> )
|
||||
|
||||
> if not declared_permission_pairs:
|
||||
> declared_permission_pairs = set(user_permission_pairs)
|
||||
|
||||
> sorted_permission_pairs = sorted(
|
||||
> declared_permission_pairs,
|
||||
> key=lambda pair: (pair[0], pair[1]),
|
||||
> )
|
||||
> permission_states = [
|
||||
> ProfilePermissionState(
|
||||
> key=self._format_permission_key(resource, action),
|
||||
> allowed=bool(is_admin or (resource, action) in user_permission_pairs),
|
||||
> )
|
||||
> for resource, action in sorted_permission_pairs
|
||||
> ]
|
||||
|
||||
> auth_source = sanitize_text(getattr(current_user, "auth_source", None))
|
||||
> current_role = "Admin" if is_admin else (role_names[0] if role_names else None)
|
||||
|
||||
> return ProfileSecuritySummary(
|
||||
> read_only=True,
|
||||
> auth_source=auth_source,
|
||||
> current_role=current_role,
|
||||
> role_source=auth_source,
|
||||
> roles=role_names,
|
||||
> permissions=permission_states,
|
||||
> )
|
||||
# #endregion build_security_summary
|
||||
|
||||
# #region _collect_user_permission_pairs [TYPE Function]
|
||||
# @BRIEF Collect effective permission tuples from current user's roles.
|
||||
# @PRE current_user can include role/permission graph.
|
||||
# @POST Returns unique normalized (resource, ACTION) tuples.
|
||||
> def _collect_user_permission_pairs(
|
||||
> self, current_user: User
|
||||
> ) -> set[tuple[str, str]]:
|
||||
> collected: set[tuple[str, str]] = set()
|
||||
> roles = getattr(current_user, "roles", []) or []
|
||||
> for role in roles:
|
||||
> permissions = getattr(role, "permissions", []) or []
|
||||
> for permission in permissions:
|
||||
> resource = sanitize_text(getattr(permission, "resource", None))
|
||||
> action = str(getattr(permission, "action", "") or "").strip().upper()
|
||||
> if resource and action:
|
||||
> collected.add((resource, action))
|
||||
> return collected
|
||||
# #endregion _collect_user_permission_pairs
|
||||
|
||||
# #region _format_permission_key [C:1] [TYPE Function]
|
||||
# @BRIEF Convert normalized permission pair to compact UI key.
|
||||
> def _format_permission_key(self, resource: str, action: str) -> str:
|
||||
> normalized_resource = sanitize_text(resource) or ""
|
||||
> normalized_action = str(action or "").strip().upper()
|
||||
> if normalized_action == "READ":
|
||||
> return normalized_resource
|
||||
> return f"{normalized_resource}:{normalized_action.lower()}"
|
||||
# #endregion _format_permission_key
|
||||
# #endregion SecurityBadgeService
|
||||
# #endregion security_badge_service
|
||||
231
backend/src/services/sql_table_extractor.py,cover
Normal file
231
backend/src/services/sql_table_extractor.py,cover
Normal file
@@ -0,0 +1,231 @@
|
||||
# #region SqlTableExtractorModule [C:2] [TYPE Module] [SEMANTICS sql, jinja, table, extraction, parsing]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Three-phase SQL+Jinja table name extractor for virtual datasets as per FR-003.
|
||||
# Phase 1: Detect Jinja spans vs SQL spans
|
||||
# Phase 2: In Jinja spans, extract "schema.table" from string values
|
||||
# Phase 3: In SQL spans, regex pattern + sqlparse filter to reject string literal false positives
|
||||
# @LAYER Service
|
||||
# @RELATION DEPENDS_ON -> [EXT:Library:sqlparse]
|
||||
# @INVARIANT Only exact schema.table matches (case-insensitive); unqualified references are NOT matched.
|
||||
# @INVARIANT Returns a set[str] of fully-qualified table names (lowercased for case-insensitive matching).
|
||||
|
||||
> from collections.abc import Iterable
|
||||
> import re
|
||||
|
||||
> import sqlparse
|
||||
> from sqlparse.sql import Token, TokenList
|
||||
> from sqlparse.tokens import Literal
|
||||
|
||||
# ── Phase 1: Jinja span detection ───────────────────────────
|
||||
|
||||
# Minimal Jinja token detection: {% ... %}, {{ ... }}, {# ... #}
|
||||
> _JINJA_BLOCK_RE = re.compile(r"{%-?\s*.*?\s*-?%}", re.DOTALL)
|
||||
> _JINJA_EXPR_RE = re.compile(r"{{-?\s*.*?\s*-?}}", re.DOTALL)
|
||||
> _JINJA_COMMENT_RE = re.compile(r"{#.*?#}", re.DOTALL)
|
||||
|
||||
# ── Phase 3: schema.table regex (case-insensitive) ───────────
|
||||
> _SCHEMA_TABLE_RE = re.compile(
|
||||
> r"""
|
||||
> (?<!['"`\w]) # not preceded by string delimiter or word char
|
||||
> \b # word boundary
|
||||
> (?: # begin: non-capturing group for full match
|
||||
> (?:"([a-zA-Z_][\w]*)")? # optional quoted schema
|
||||
> (?:([a-zA-Z_][\w]*)) # schema (unquoted)
|
||||
> \.
|
||||
> (?:([a-zA-Z_][\w]*)) # table
|
||||
> )
|
||||
> \b # word boundary
|
||||
> (?!['"`]) # not followed by a string delimiter
|
||||
> """,
|
||||
> re.VERBOSE | re.IGNORECASE,
|
||||
> )
|
||||
|
||||
|
||||
# #region detect_jinja_spans [C:2] [TYPE Function]
|
||||
# @ingroup Services
|
||||
# @BRIEF Phase 1: Split raw SQL text into Jinja spans and SQL spans.
|
||||
# @PRE raw_sql is a string (possibly empty).
|
||||
# @POST Returns a list of (span_type: str, text: str) tuples.
|
||||
# span_type is "jinja" or "sql".
|
||||
> def detect_jinja_spans(raw_sql: str) -> list[tuple[str, str]]:
|
||||
> """Split raw SQL+Jinja text into alternating Jinja and SQL spans.
|
||||
|
||||
> Phase 1 detects Jinja blocks ({%%}, {{}}, {##}) and returns the
|
||||
> remaining text as SQL spans.
|
||||
> """
|
||||
> if not raw_sql or not raw_sql.strip():
|
||||
! return [("sql", raw_sql or "")]
|
||||
|
||||
> spans: list[tuple[str, str]] = []
|
||||
# Collect all Jinja match positions
|
||||
> jinja_matches: list[tuple[int, int]] = []
|
||||
> for pattern in (_JINJA_BLOCK_RE, _JINJA_EXPR_RE, _JINJA_COMMENT_RE):
|
||||
> for m in pattern.finditer(raw_sql):
|
||||
> jinja_matches.append((m.start(), m.end()))
|
||||
|
||||
# Merge overlapping Jinja spans
|
||||
> if jinja_matches:
|
||||
> jinja_matches.sort()
|
||||
> merged: list[tuple[int, int]] = [jinja_matches[0]]
|
||||
> for start, end in jinja_matches[1:]:
|
||||
! if start <= merged[-1][1]:
|
||||
! merged[-1] = (merged[-1][0], max(merged[-1][1], end))
|
||||
! else:
|
||||
! merged.append((start, end))
|
||||
|
||||
# Build alternating sql/jinja spans
|
||||
> cursor = 0
|
||||
> for start, end in merged:
|
||||
> if cursor < start:
|
||||
> sql_chunk = raw_sql[cursor:start].strip()
|
||||
> if sql_chunk:
|
||||
> spans.append(("sql", sql_chunk))
|
||||
> jinja_chunk = raw_sql[start:end].strip()
|
||||
> if jinja_chunk:
|
||||
> spans.append(("jinja", jinja_chunk))
|
||||
> cursor = end
|
||||
> if cursor < len(raw_sql):
|
||||
> remaining = raw_sql[cursor:].strip()
|
||||
> if remaining:
|
||||
> spans.append(("sql", remaining))
|
||||
> else:
|
||||
> spans.append(("sql", raw_sql.strip()))
|
||||
|
||||
> return spans
|
||||
# #endregion detect_jinja_spans
|
||||
|
||||
|
||||
# #region extract_tables_from_jinja [C:2] [TYPE Function]
|
||||
# @ingroup Services
|
||||
# @BRIEF Phase 2: Extract "schema.table" references from Jinja string values.
|
||||
# Looks for patterns like "raw.sales" inside Jinja text.
|
||||
# @PRE jinja_text is a string from a Jinja span.
|
||||
# @POST Returns a set of lowercased schema.table strings.
|
||||
> def extract_tables_from_jinja(jinja_text: str) -> set[str]:
|
||||
> """Extract ``schema.table`` references from within Jinja template blocks.
|
||||
|
||||
> Looks for double-quoted or single-quoted string values that match
|
||||
> the ``schema.table`` pattern, e.g. ``"raw.sales"`` or ``'raw.inventory'``.
|
||||
> """
|
||||
> tables: set[str] = set()
|
||||
# Match string values: "schema.table" or 'schema.table'
|
||||
> string_pattern = re.compile(
|
||||
> r"""["']([a-zA-Z_][\w]*\.[a-zA-Z_][\w]*)["']"""
|
||||
> )
|
||||
> for m in string_pattern.finditer(jinja_text):
|
||||
> tables.add(m.group(1).lower())
|
||||
> return tables
|
||||
# #endregion extract_tables_from_jinja
|
||||
|
||||
|
||||
# #region is_string_literal [C:1] [TYPE Function]
|
||||
# @BRIEF Check if a sqlparse Token is a string literal (not a schema.table identifier).
|
||||
# @PRE token is a sqlparse Token.
|
||||
# @POST Returns True if the token is a string/Single/Literal.String.
|
||||
> def is_string_literal(token: Token) -> bool:
|
||||
> """Return True if token is a string literal type in sqlparse."""
|
||||
> return token.ttype is Literal.String.Single or token.ttype is Literal.String
|
||||
# #endregion is_string_literal
|
||||
|
||||
|
||||
# #region extract_tables_from_sql_span [C:2] [TYPE Function]
|
||||
# @ingroup Services
|
||||
# @BRIEF Phase 3: Extract schema.table references from SQL text using regex + sqlparse filtering.
|
||||
# Uses regex to find all schema.table candidates, then sqlparse to filter out
|
||||
# false positives inside string literals.
|
||||
# @PRE sql_text is a string from a SQL span (non-Jinja).
|
||||
# @POST Returns a set of lowercased schema.table strings.
|
||||
> def extract_tables_from_sql_span(sql_text: str) -> set[str]:
|
||||
> """Extract ``schema.table`` references from plain SQL text.
|
||||
|
||||
> Uses regex to find all ``schema.table`` candidates, then sqlparse
|
||||
> to reject matches that fall inside string literals.
|
||||
> """
|
||||
> tables: set[str] = set()
|
||||
> raw_matches = _SCHEMA_TABLE_RE.findall(sql_text)
|
||||
> if not raw_matches:
|
||||
> return tables
|
||||
|
||||
# Use sqlparse to identify string literal positions
|
||||
> parsed = sqlparse.parse(sql_text)
|
||||
> string_literal_ranges: list[tuple[int, int]] = []
|
||||
|
||||
> def walk_tokens(tokens: Iterable[Token], base_offset: int = 0) -> None:
|
||||
> offset = base_offset
|
||||
> for token in tokens:
|
||||
> if isinstance(token, TokenList):
|
||||
! walk_tokens(token.flatten(), offset)
|
||||
> else:
|
||||
> ttype = token.ttype
|
||||
> val = token.value
|
||||
> if is_string_literal(token):
|
||||
> string_literal_ranges.append(
|
||||
> (offset, offset + len(val))
|
||||
> )
|
||||
> offset += len(val)
|
||||
|
||||
> for stmt in parsed:
|
||||
> if stmt is None:
|
||||
! continue
|
||||
> walk_tokens(stmt.flatten(), base_offset=0)
|
||||
|
||||
> def is_in_string(pos: int) -> bool:
|
||||
> for s_start, s_end in string_literal_ranges:
|
||||
> if s_start <= pos <= s_end:
|
||||
! return True
|
||||
> return False
|
||||
|
||||
# Re-scan with full match positions to filter
|
||||
> for m in _SCHEMA_TABLE_RE.finditer(sql_text):
|
||||
> if not is_in_string(m.start()):
|
||||
# Extract schema and table from match groups
|
||||
> schema = m.group(2) or m.group(1) or ""
|
||||
> table = m.group(3) or ""
|
||||
# Filter out column aliases (single-char schemas like "a.id", "o.id")
|
||||
> if len(schema) < 2 and schema.isalpha() and schema.islower():
|
||||
> continue
|
||||
> if schema and table:
|
||||
> tables.add(f"{schema.lower()}.{table.lower()}")
|
||||
|
||||
> return tables
|
||||
# #endregion extract_tables_from_sql_span
|
||||
|
||||
|
||||
# #region extract_tables_from_sql [C:2] [TYPE Function]
|
||||
# @ingroup Services
|
||||
# @BRIEF Three-phase entry point: extract all schema.table references from raw SQL+Jinja text.
|
||||
# @PRE raw_sql is a string (possibly empty).
|
||||
# @POST Returns a set of lowercased fully-qualified table names (schema.table format).
|
||||
# @RELATION CALLS -> [detect_jinja_spans]
|
||||
# @RELATION CALLS -> [extract_tables_from_jinja]
|
||||
# @RELATION CALLS -> [extract_tables_from_sql_span]
|
||||
> def extract_tables_from_sql(raw_sql: str) -> set[str]:
|
||||
> """Extract all ``schema.table`` references from raw SQL + Jinja text.
|
||||
|
||||
> Three-phase approach per FR-003:
|
||||
> 1. Detect Jinja spans vs SQL spans
|
||||
> 2. In Jinja spans, extract ``schema.table`` from quoted string values
|
||||
> 3. In SQL spans, regex + sqlparse filter to reject string literal false positives
|
||||
|
||||
> Returns a set of lowercased ``schema.table`` strings for case-insensitive matching.
|
||||
|
||||
> Example:
|
||||
> >>> extract_tables_from_sql("SELECT * FROM raw.sales JOIN raw.inventory ON ...")
|
||||
> {'raw.sales', 'raw.inventory'}
|
||||
> """
|
||||
> if not raw_sql or not raw_sql.strip():
|
||||
> return set()
|
||||
|
||||
> all_tables: set[str] = set()
|
||||
> spans = detect_jinja_spans(raw_sql)
|
||||
|
||||
> for span_type, text in spans:
|
||||
> if span_type == "jinja":
|
||||
> all_tables.update(extract_tables_from_jinja(text))
|
||||
> else:
|
||||
> all_tables.update(extract_tables_from_sql_span(text))
|
||||
|
||||
> return all_tables
|
||||
# #endregion extract_tables_from_sql
|
||||
|
||||
# #endregion SqlTableExtractorModule
|
||||
144
backend/src/services/superset_lookup_service.py,cover
Normal file
144
backend/src/services/superset_lookup_service.py,cover
Normal file
@@ -0,0 +1,144 @@
|
||||
# #region superset_lookup_service [C:4] [TYPE Module] [SEMANTICS superset,account,lookup,environment]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Environment-scoped Superset account lookup with degradation fallback for network failures.
|
||||
# @LAYER Domain
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [SupersetAccountLookupAdapter]
|
||||
# @RELATION DEPENDS_ON -> [ProfileUtils]
|
||||
# @RATIONALE Extracted from ProfileService to satisfy INV_7. Superset account lookup has distinct
|
||||
# I/O patterns (external HTTP calls, environment resolution) that isolate cleanly from
|
||||
# preference persistence and security badge logic.
|
||||
# @REJECTED Keeping lookup inside ProfileService was rejected — it adds network I/O and
|
||||
# environment coupling to preference operations that only need DB access, and inflates
|
||||
# the class past 150 lines.
|
||||
# @PRE current_user is authenticated; environment_id must be resolvable.
|
||||
# @POST Returns success or degraded SupersetAccountLookupResponse with stable shape.
|
||||
# @SIDE_EFFECT Performs external Superset HTTP calls via SupersetClient + adapter.
|
||||
# @RAISES EnvironmentNotFoundError when environment_id is unknown.
|
||||
|
||||
> from typing import Any
|
||||
|
||||
> from ..core.logger import belief_scope, logger
|
||||
> from ..core.async_superset_client import AsyncSupersetClient
|
||||
> from ..core.superset_profile_lookup import SupersetAccountLookupAdapter
|
||||
> from ..schemas.profile import (
|
||||
> SupersetAccountCandidate,
|
||||
> SupersetAccountLookupRequest,
|
||||
> SupersetAccountLookupResponse,
|
||||
> )
|
||||
> from .profile_utils import EnvironmentNotFoundError
|
||||
|
||||
|
||||
# #region SupersetLookupService [C:4] [TYPE Class] [SEMANTICS superset,lookup,environment,degradation]
|
||||
# @defgroup Services Module group.
|
||||
# @BRIEF Resolves environments and queries Superset users in selected environment,
|
||||
# returning canonical account candidates with degradation fallback.
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [SupersetAccountLookupAdapter]
|
||||
# @RELATION CALLS -> [SupersetClient]
|
||||
# @PRE config_manager supports get_environments().
|
||||
# @POST lookup_superset_accounts returns success payload or degraded payload with warning.
|
||||
> class SupersetLookupService:
|
||||
> """Environment-scoped Superset account lookup with degradation fallback."""
|
||||
|
||||
# #region __init__ [TYPE Function]
|
||||
# @BRIEF Initialize with config manager for environment resolution.
|
||||
> def __init__(self, config_manager: Any):
|
||||
> self.config_manager = config_manager
|
||||
# #endregion __init__
|
||||
|
||||
# #region lookup_superset_accounts [TYPE Function]
|
||||
# @ingroup Services
|
||||
# @BRIEF Query Superset users in selected environment and project canonical account candidates.
|
||||
# @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API.
|
||||
# @PRE current_user is authenticated and environment_id exists.
|
||||
# @POST Returns success payload or degraded payload with warning while preserving manual fallback.
|
||||
> async def lookup_superset_accounts(
|
||||
> self,
|
||||
> current_user: Any,
|
||||
> request: SupersetAccountLookupRequest,
|
||||
> ) -> SupersetAccountLookupResponse:
|
||||
> with belief_scope(
|
||||
> "SupersetLookupService.lookup_superset_accounts",
|
||||
> f"user_id={getattr(current_user, 'id', '?')}, environment_id={request.environment_id}",
|
||||
> ):
|
||||
> environment = self._resolve_environment(request.environment_id)
|
||||
> if environment is None:
|
||||
> logger.explore("Lookup aborted: environment not found")
|
||||
> raise EnvironmentNotFoundError(
|
||||
> f"Environment '{request.environment_id}' not found"
|
||||
> )
|
||||
|
||||
> sort_column = str(request.sort_column or "username").strip().lower()
|
||||
> sort_order = str(request.sort_order or "desc").strip().lower()
|
||||
> allowed_columns = {"username", "first_name", "last_name", "email"}
|
||||
> if sort_column not in allowed_columns:
|
||||
> sort_column = "username"
|
||||
> if sort_order not in {"asc", "desc"}:
|
||||
> sort_order = "desc"
|
||||
|
||||
> logger.reflect(
|
||||
> "Normalized lookup request "
|
||||
> f"(env={request.environment_id}, sort_column={sort_column}, sort_order={sort_order}, "
|
||||
> f"page_index={request.page_index}, page_size={request.page_size}, "
|
||||
> f"search={(request.search or '').strip()!r})"
|
||||
> )
|
||||
|
||||
> try:
|
||||
> logger.reason("Performing Superset account lookup")
|
||||
> superset_client = AsyncSupersetClient(environment)
|
||||
> adapter = SupersetAccountLookupAdapter(
|
||||
> network_client=superset_client.client,
|
||||
> environment_id=request.environment_id,
|
||||
> )
|
||||
> lookup_result = await adapter.get_users_page(
|
||||
> search=request.search,
|
||||
> page_index=request.page_index,
|
||||
> page_size=request.page_size,
|
||||
> sort_column=sort_column,
|
||||
> sort_order=sort_order,
|
||||
> )
|
||||
> items = [
|
||||
> SupersetAccountCandidate.model_validate(item)
|
||||
> for item in lookup_result.get("items", [])
|
||||
> ]
|
||||
> return SupersetAccountLookupResponse(
|
||||
> status="success",
|
||||
> environment_id=request.environment_id,
|
||||
> page_index=request.page_index,
|
||||
> page_size=request.page_size,
|
||||
> total=max(int(lookup_result.get("total", len(items))), 0),
|
||||
> warning=None,
|
||||
> items=items,
|
||||
> )
|
||||
> except Exception as exc:
|
||||
> logger.explore(
|
||||
> f"Lookup degraded due to upstream error: {exc}"
|
||||
> )
|
||||
> return SupersetAccountLookupResponse(
|
||||
> status="degraded",
|
||||
> environment_id=request.environment_id,
|
||||
> page_index=request.page_index,
|
||||
> page_size=request.page_size,
|
||||
> total=0,
|
||||
> warning=(
|
||||
> "Cannot load Superset accounts for this environment right now. "
|
||||
> "You can enter username manually."
|
||||
> ),
|
||||
> items=[],
|
||||
> )
|
||||
# #endregion lookup_superset_accounts
|
||||
|
||||
# #region _resolve_environment [C:2] [TYPE Function]
|
||||
# @BRIEF Resolve environment model from configured environments by id.
|
||||
# @PRE environment_id is provided.
|
||||
# @POST Returns environment object when found else None.
|
||||
> def _resolve_environment(self, environment_id: str):
|
||||
> environments = self.config_manager.get_environments()
|
||||
> for env in environments:
|
||||
> if str(getattr(env, "id", "")) == str(environment_id):
|
||||
> return env
|
||||
> return None
|
||||
# #endregion _resolve_environment
|
||||
# #endregion SupersetLookupService
|
||||
# #endregion superset_lookup_service
|
||||
Reference in New Issue
Block a user