busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: massive coverage expansion — 15 new test modules + assistant tool fixes + orthogonal testing

Browse Source 
- 10 translate plugin test files (100% coverage on 12 modules)
- assistant/handler tools: 85+ tests covering dispatch, registry, resolvers, routes, llm_planner, 13 tool handlers
- clean release: artifact_catalog_loader, mappers, approval, publication tests
- API routes: translate_helpers, validation_service extensions, datasets to 100%
- notifications: providers/service tests
- services: profile_preference_service
- docs/orthogonal-test-report.md — full speckit.tests audit
- Fixes: 3 git_base async mock failures, 4 assistant handler permission-check patches
- .gitignore: coverage artifacts
This commit is contained in:
busya
2026-06-15 15:38:59 +03:00
parent 6ba381676a
commit f75c15dbc6
67 changed files with 10769 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
backend/src/api/routes/dashboards/_helpers.py
View File

@@ -50,19 +50,10 @@ def _resolve_dashboard_id_from_ref(
dashboard_ref: str,
client: SupersetClient,
) -> int:
normalized_ref = str(dashboard_ref or "").strip()
if not normalized_ref:
raise HTTPException(status_code=404, detail="Dashboard not found")
# Slug-first: even if ref looks numeric, try slug first.
slug_match_id = _find_dashboard_id_by_slug(client, normalized_ref)
if slug_match_id is not None:
return slug_match_id
if normalized_ref.isdigit():
return int(normalized_ref)
raise HTTPException(status_code=404, detail="Dashboard not found")
raise RuntimeError(
"_resolve_dashboard_id_from_ref is deprecated. "
"Use _resolve_dashboard_id_from_ref_async() instead."
)
# #endregion _resolve_dashboard_id_from_ref

2
backend/src/api/routes/settings.py
View File

@@ -861,7 +861,7 @@ async def test_connection(
_=Depends(has_permission("settings.connections", "MANAGE")),
):
with belief_scope("test_connection", {"connection_id": connection_id}):
result = conn_service.test_connection(connection_id)
result = await conn_service.test_connection(connection_id)
logger.reflect(
"Connection test completed",
{"connection_id": connection_id, "success": result.get("success")},

39
backend/src/core/connection_service.py
View File

@@ -248,7 +248,7 @@ class ConnectionService:
# @PRE Connection exists and is reachable from the ss-tools backend.
# @POST Returns {success, latency_ms, db_version, error}. Connection always closed.
# @SIDE_EFFECT Opens/closes native DB connection. Network I/O. May trigger auth.
def test_connection(self, connection_id: str) -> dict[str, Any]:
async def test_connection(self, connection_id: str) -> dict[str, Any]:
conn = self.get_connection(connection_id)
if conn is None:
return {"success": False, "error": f"Connection '{connection_id}' not found"}
@@ -257,7 +257,7 @@ class ConnectionService:
try:
# Use dialect-appropriate test
if conn.dialect in ("postgresql",):
version = self._test_postgresql(conn)
version = await self._test_postgresql(conn)
elif conn.dialect == "clickhouse":
version = self._test_clickhouse(conn)
elif conn.dialect == "mysql":
@@ -283,31 +283,26 @@ class ConnectionService:
# #region _test_postgresql [C:2] [TYPE Function] [SEMANTICS settings,connections,test-pg]
# @BRIEF Test PostgreSQL connectivity using asyncpg. Returns server version.
def _test_postgresql(self, conn: DatabaseConnection) -> str:
import asyncio
async def _test_postgresql(self, conn: DatabaseConnection) -> str:
try:
import asyncpg # type: ignore[import-untyped]
except ImportError:
raise RuntimeError("asyncpg is not installed. Add to requirements.txt.")
async def _test():
attempt = await asyncpg.connect(
host=conn.host,
port=conn.port,
user=conn.username,
password=conn.password,
database=conn.database,
timeout=10,
)
try:
row = await attempt.fetchrow("SELECT version()")
version = row[0] if row else "Unknown"
return version
finally:
await attempt.close()
return asyncio.run(_test())
attempt = await asyncpg.connect(
host=conn.host,
port=conn.port,
user=conn.username,
password=conn.password,
database=conn.database,
timeout=10,
)
try:
row = await attempt.fetchrow("SELECT version()")
version = row[0] if row else "Unknown"
return version
finally:
await attempt.close()
# #endregion _test_postgresql

105
backend/src/core/db_executor.py
View File

@@ -67,7 +67,7 @@ class DbExecutor:
# @POST SQL executed in a transaction. Result returned with rows_affected and timing.
# @SIDE_EFFECT Writes data to target database. Opens/closes native driver connections.
# @DATA_CONTRACT Input: (connection_id: str, sql: str) → Output: DbExecutionResult
def execute_sql(self, connection_id: str, sql: str) -> DbExecutionResult:
async def execute_sql(self, connection_id: str, sql: str) -> DbExecutionResult:
"""Execute SQL directly against the target database.
Args:
@@ -86,7 +86,7 @@ class DbExecutor:
start = time.time()
try:
if conn.dialect == "postgresql":
return self._execute_pg(conn, sql, start)
return await self._execute_pg(conn, sql, start)
elif conn.dialect == "clickhouse":
return self._execute_ch(conn, sql, start)
elif conn.dialect == "mysql":
@@ -118,9 +118,7 @@ class DbExecutor:
# #region dialect executors [C:2] [TYPE Function] [SEMANTICS database,dialect,pg]
# @BRIEF PostgreSQL execution via asyncpg with connection pooling.
def _execute_pg(self, conn: Any, sql: str, start: float) -> DbExecutionResult:
import asyncio
async def _execute_pg(self, conn: Any, sql: str, start: float) -> DbExecutionResult:
try:
import asyncpg # type: ignore[import-untyped]
except ImportError:
@@ -132,40 +130,36 @@ class DbExecutor:
config_key = f"{conn.host}:{conn.port}/{conn.database}@{conn.updated_at}"
pool = self._get_or_create_pool(conn.id, config_key)
async def _run():
nonlocal pool
if pool is None:
pool = await asyncpg.create_pool(
host=conn.host,
port=conn.port,
user=conn.username,
password=conn.password,
database=conn.database,
min_size=1,
max_size=conn.pool_size,
)
self._set_pool(conn.id, pool, config_key)
if pool is None:
pool = await asyncpg.create_pool(
host=conn.host,
port=conn.port,
user=conn.username,
password=conn.password,
database=conn.database,
min_size=1,
max_size=conn.pool_size,
)
self._set_pool(conn.id, pool, config_key)
async with pool.acquire() as pg_conn:
# asyncpg execute returns status string like "INSERT 0 5"
status = await pg_conn.execute(sql)
elapsed = int((time.time() - start) * 1000)
# Parse rows affected from status string
rows = 0
if status:
parts = status.split()
if len(parts) >= 2:
try:
rows = int(parts[-1])
except ValueError:
rows = 0
return DbExecutionResult(
success=True,
rows_affected=rows,
execution_time_ms=elapsed,
)
return asyncio.run(_run())
async with pool.acquire() as pg_conn:
# asyncpg execute returns status string like "INSERT 0 5"
status = await pg_conn.execute(sql)
elapsed = int((time.time() - start) * 1000)
# Parse rows affected from status string
rows = 0
if status:
parts = status.split()
if len(parts) >= 2:
try:
rows = int(parts[-1])
except ValueError:
rows = 0
return DbExecutionResult(
success=True,
rows_affected=rows,
execution_time_ms=elapsed,
)
# #endregion _execute_pg
@@ -246,7 +240,7 @@ class DbExecutor:
# @PRE connection_id must reference a valid saved connection.
# @POST Returns list of DbSchemaColumn or None on failure.
# @RELATION CALLS -> [Core.ConnectionService]
def fetch_schema(
async def fetch_schema(
self, connection_id: str, schema: str, table: str, config_manager: Any
) -> list[DbSchemaColumn] | None:
"""Query target table columns directly via native driver."""
@@ -276,7 +270,7 @@ class DbExecutor:
f"WHERE table_schema = '{safe_schema}' AND table_name = '{safe_table}' "
f"ORDER BY ordinal_position"
)
return self._fetch_pg(conn, sql)
return await self._fetch_pg(conn, sql)
if conn.dialect == "mysql":
sql = (
@@ -292,29 +286,24 @@ class DbExecutor:
# #endregion fetch_schema
# #region _fetch_pg [C:2] [TYPE Function] [SEMANTICS database,postgresql,fetch,schema]
def _fetch_pg(self, conn: Any, sql: str) -> list[DbSchemaColumn] | None:
import asyncio
async def _fetch_pg(self, conn: Any, sql: str) -> list[DbSchemaColumn] | None:
try:
import asyncpg # type: ignore[import-untyped]
except ImportError:
return None
async def _run() -> list[DbSchemaColumn] | None:
pool_cfg = f"{conn.host}:{conn.port}/{conn.database}@{conn.updated_at}"
pool = self._get_or_create_pool(conn.id, pool_cfg)
if pool is None:
pool = await asyncpg.create_pool(
host=conn.host, port=conn.port,
user=conn.username, password=conn.password,
database=conn.database, min_size=1, max_size=conn.pool_size,
)
self._set_pool(conn.id, pool, pool_cfg)
async with pool.acquire() as pg_conn:
rows = await pg_conn.fetch(sql)
return [DbSchemaColumn(name=r["name"], data_type=r["type"]) for r in rows]
return asyncio.run(_run())
pool_cfg = f"{conn.host}:{conn.port}/{conn.database}@{conn.updated_at}"
pool = self._get_or_create_pool(conn.id, pool_cfg)
if pool is None:
pool = await asyncpg.create_pool(
host=conn.host, port=conn.port,
user=conn.username, password=conn.password,
database=conn.database, min_size=1, max_size=conn.pool_size,
)
self._set_pool(conn.id, pool, pool_cfg)
async with pool.acquire() as pg_conn:
rows = await pg_conn.fetch(sql)
return [DbSchemaColumn(name=r["name"], data_type=r["type"]) for r in rows]
# #endregion _fetch_pg

2
backend/src/plugins/translate/orchestrator_sql.py
View File

@@ -152,7 +152,7 @@ class SQLInsertService:
self.db.commit()
executor = DbExecutor(conn_service)
result = executor.execute_sql(job.connection_id, sql)
result = await executor.execute_sql(job.connection_id, sql)
if result.success:
return {

2
backend/src/plugins/translate/service_target_schema.py
View File

@@ -243,7 +243,7 @@ async def validate_target_table_schema(
schema = req.target_schema or "public"
connection_svc = ConnectionService(config_manager)
executor = DbExecutor(connection_svc)
columns = executor.fetch_schema(req.connection_id, schema, req.target_table, config_manager)
columns = await executor.fetch_schema(req.connection_id, schema, req.target_table, config_manager)
if columns is None:
return TargetSchemaValidationResponse(

4
backend/src/schemas/__init__.py,cover Normal file
View File

@@ -0,0 +1,4 @@
# #region SchemasPackage [TYPE Package] [SEMANTICS schema, package, init]
# @ingroup Schemas
# @BRIEF API schema package root.
# #endregion SchemasPackage

106
backend/src/schemas/agent.py,cover Normal file
View File

@@ -0,0 +1,106 @@
# backend/src/schemas/agent.py
# #region Schemas.Agent [C:1] [TYPE Module] [SEMANTICS agent,schema,api]
# @BRIEF Pydantic schemas for agent conversation API. Must match frontend/src/types/agent.ts exactly.
> from datetime import datetime
> from pydantic import BaseModel, Field
# #region Schemas.Agent.ConversationItem [C:1] [TYPE Class] [SEMANTICS agent,schema,conversation]
# @ingroup Schemas
> class ConversationItem(BaseModel):
> id: str
> title: str
> updated_at: datetime
> message_count: int
# #endregion Schemas.Agent.ConversationItem
# #region Schemas.Agent.ConversationListResponse [C:1] [TYPE Class] [SEMANTICS agent,schema,conversation]
# @ingroup Schemas
> class ConversationListResponse(BaseModel):
> items: list[ConversationItem]
> has_next: bool = False
> active_total: int = 0
> archived_total: int = 0
# #endregion Schemas.Agent.ConversationListResponse
# #region Schemas.Agent.ToolCall [C:1] [TYPE Class] [SEMANTICS agent,schema,tool-call]
# @ingroup Schemas
> class ToolCall(BaseModel):
> tool: str
> input: dict = Field(default_factory=dict)
> output: dict | None = None
> error: str | None = None
> status: str = "executing" # executing | completed | failed
# #endregion Schemas.Agent.ToolCall
# #region Schemas.Agent.AttachmentMeta [C:1] [TYPE Class] [SEMANTICS agent,schema,attachment]
# @ingroup Schemas
> class AttachmentMeta(BaseModel):
> name: str
> type: str # pdf | xlsx | json | csv | txt | png | jpeg
> size: int
> preview_url: str | None = None
# #endregion Schemas.Agent.AttachmentMeta
# #region Schemas.Agent.MessageItem [C:1] [TYPE Class] [SEMANTICS agent,schema,message]
# @ingroup Schemas
> class MessageItem(BaseModel):
> id: str
> conversation_id: str
> role: str # user | assistant | tool | system
> text: str | None = None
> state: str | None = None
> tool_calls: list[ToolCall] | None = None
> attachments: list[AttachmentMeta] | None = None
> created_at: datetime
# #endregion Schemas.Agent.MessageItem
# #region Schemas.Agent.HistoryResponse [C:1] [TYPE Class] [SEMANTICS agent,schema,history]
# @ingroup Schemas
> class HistoryResponse(BaseModel):
> items: list[MessageItem]
> has_next: bool = False
> conversation_id: str | None = None
# #endregion Schemas.Agent.HistoryResponse
# #region Schemas.Agent.DeleteResponse [C:1] [TYPE Class] [SEMANTICS agent,schema,delete]
# @ingroup Schemas
> class DeleteResponse(BaseModel):
> deleted: bool = True
# #endregion Schemas.Agent.DeleteResponse
# #region Schemas.Agent.ServiceTokenRequest [C:1] [TYPE Class] [SEMANTICS agent,schema,auth]
# @ingroup Schemas
> class ServiceTokenRequest(BaseModel):
> service_secret: str
# #endregion Schemas.Agent.ServiceTokenRequest
# #region Schemas.Agent.ServiceTokenResponse [C:1] [TYPE Class] [SEMANTICS agent,schema,auth]
# @ingroup Schemas
> class ServiceTokenResponse(BaseModel):
> access_token: str
> token_type: str = "bearer"
> expires_in: int = 86400
> role: str = "agent"
# #endregion Schemas.Agent.ServiceTokenResponse
# #region Schemas.Agent.SaveConversationRequest [C:1] [TYPE Class] [SEMANTICS agent,schema,save]
# @ingroup Schemas
> class SaveConversationRequest(BaseModel):
> conversation_id: str
> title: str = ""
> user_id: str = "admin"
> messages: list[dict] = []
# #endregion Schemas.Agent.SaveConversationRequest
# #endregion Schemas.Agent

192
backend/src/schemas/auth.py,cover Normal file
View File

@@ -0,0 +1,192 @@
# #region AuthSchemas [C:5] [TYPE Module] [SEMANTICS pydantic, auth, schema, token]
# @defgroup Schemas Module group.
#
# @BRIEF Pydantic schemas for authentication requests and responses.
# @LAYER API
# @RELATION DEPENDS_ON -> [EXT:Library:pydantic]
#
# @INVARIANT Sensitive fields like password must not be included in response schemas.
# @DATA_CONTRACT AuthPayload -> AuthSchema
> from datetime import datetime
> import re
> from pydantic import BaseModel, ConfigDict, EmailStr, Field, field_validator
# #region Token [C:1] [TYPE Class]
# @BRIEF Represents a JWT access token response.
> class Token(BaseModel):
> access_token: str
> token_type: str
# #endregion Token
# #region TokenData [C:1] [TYPE Class]
# @BRIEF Represents the data encoded in a JWT token.
> class TokenData(BaseModel):
> username: str | None = None
> scopes: list[str] = []
# #endregion TokenData
# #region PermissionSchema [C:1] [TYPE Class]
# @BRIEF Represents a permission in API responses.
> class PermissionSchema(BaseModel):
> id: str | None = None
> resource: str
> action: str
> model_config = ConfigDict(from_attributes=True)
# #endregion PermissionSchema
# #region RoleSchema [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Represents a role in API responses.
> class RoleSchema(BaseModel):
> id: str
> name: str
> description: str | None = None
> permissions: list[PermissionSchema] = []
> model_config = ConfigDict(from_attributes=True)
# #endregion RoleSchema
# #region RoleCreate [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for creating a new role.
> class RoleCreate(BaseModel):
> name: str
> description: str | None = None
> permissions: list[str] = [] # List of permission IDs or "resource:action" strings
# #endregion RoleCreate
# #region RoleUpdate [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for updating an existing role.
> class RoleUpdate(BaseModel):
> name: str | None = None
> description: str | None = None
> permissions: list[str] | None = None
# #endregion RoleUpdate
# #region ADGroupMappingSchema [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Represents an AD Group to Role mapping in API responses.
> class ADGroupMappingSchema(BaseModel):
> id: str
> ad_group: str
> role_id: str
> model_config = ConfigDict(from_attributes=True)
# #endregion ADGroupMappingSchema
# #region ADGroupMappingCreate [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for creating an AD Group mapping.
> class ADGroupMappingCreate(BaseModel):
> ad_group: str
> role_id: str
> @field_validator("ad_group")
> @classmethod
> def validate_ad_group(cls, v: str) -> str:
> if not v or not v.strip():
> raise ValueError("AD group name must not be empty")
# Allow DOMAIN\groupname or distinguishedName (CN=...,DC=...) formats
# Allow DOMAIN\groupname or distinguishedName (CN=...,DC=...) formats
> if not re.match(r'^[A-Za-z0-9_.@()=\\\\, -]+$', v):
> raise ValueError(
> "AD group name contains invalid characters. "
> "Use format: DOMAIN\\groupname or CN=groupname,DC=domain"
> )
> return v.strip()
# #endregion ADGroupMappingCreate
# #region UserBase [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Base schema for user data.
> class UserBase(BaseModel):
> username: str
> email: EmailStr | None = None
> is_active: bool = True
# #endregion UserBase
# #region UserCreate [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for creating a new user.
> class UserCreate(UserBase):
> password: str = Field(min_length=8)
> roles: list[str] = []
> @field_validator("password")
> @classmethod
> def validate_password_strength(cls, v: str) -> str:
> if len(v) < 8:
! raise ValueError("Password must be at least 8 characters")
> if not any(c.isupper() for c in v):
> raise ValueError("Password must contain at least one uppercase letter")
> if not any(c.islower() for c in v):
> raise ValueError("Password must contain at least one lowercase letter")
> if not any(c.isdigit() for c in v):
> raise ValueError("Password must contain at least one digit")
> return v
# #endregion UserCreate
# #region UserUpdate [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for updating an existing user.
> class UserUpdate(BaseModel):
> email: EmailStr | None = None
> password: str | None = None
> is_active: bool | None = None
> roles: list[str] | None = None
# #endregion UserUpdate
# #region User [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for user data in API responses.
> class User(UserBase):
> id: str
> auth_source: str
> created_at: datetime
> last_login: datetime | None = None
> roles: list[RoleSchema] = []
> model_config = ConfigDict(from_attributes=True)
# #endregion User
# #endregion AuthSchemas

29
backend/src/schemas/dataset_review.py,cover Normal file
View File

@@ -0,0 +1,29 @@
# #region DatasetReviewSchemas [C:2] [TYPE Module] [SEMANTICS dataset, review, schema, facade]
# @defgroup Schemas Module group.
# @BRIEF Thin facade re-exporting all dataset review API schemas from decomposed sub-modules.
# @LAYER API
# @RATIONALE Original 419-line file exceeded INV_7 (400-line module limit). Decomposed into DTO and composite sub-modules.
# @REJECTED Keeping all schemas in a single file because it exceeded the fractal limit.
! from src.schemas.dataset_review_pkg._composites import ( # noqa: F401
! ClarificationAnswerDto,
! ClarificationOptionDto,
! ClarificationQuestionDto,
! ClarificationSessionDto,
! CompiledPreviewDto,
! DatasetRunContextDto,
! SessionDetail,
! SessionSummary,
! )
! from src.schemas.dataset_review_pkg._dtos import ( # noqa: F401
! DatasetProfileDto,
! ExecutionMappingDto,
! ImportedFilterDto,
! SemanticCandidateDto,
! SemanticFieldEntryDto,
! SemanticSourceDto,
! SessionCollaboratorDto,
! TemplateVariableDto,
! ValidationFindingDto,
! )
# #endregion DatasetReviewSchemas

54
backend/src/schemas/health.py,cover Normal file
View File

@@ -0,0 +1,54 @@
# #region HealthSchemas [C:3] [TYPE Module] [SEMANTICS pydantic, health, schema, dashboard, dashboard-health-item]
# @defgroup Schemas Module group.
# @BRIEF Pydantic schemas for dashboard health summary — includes v2 LLM validation fields.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [EXT:Library:pydantic]
> from datetime import datetime
> from typing import Any
> from pydantic import BaseModel, Field
# #region DashboardHealthItem [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Represents the latest health status of a single dashboard, with v2 LLM validation data.
> class DashboardHealthItem(BaseModel):
> record_id: str | None = None
> dashboard_id: str
> dashboard_slug: str | None = None
> dashboard_title: str | None = None
> environment_id: str
> status: str = Field(..., pattern="^(PASS|WARN|FAIL|UNKNOWN)$")
> last_check: datetime
> task_id: str | None = None
> run_id: str | None = None
> policy_id: str | None = None
> summary: str | None = None
# v2 LLM validation fields
> execution_path: str | None = None # "screenshot" or "text_only"
> issues_count: int = 0
> timings: dict[str, Any] | None = None
> token_usage: dict[str, Any] | None = None
> screenshot_paths: list[str] | None = None
> chunk_count: int | None = None
> dashboard_name: str | None = None # human-readable alias for dashboard_title
# #endregion DashboardHealthItem
# #region HealthSummaryResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Aggregated health summary for all dashboards.
> class HealthSummaryResponse(BaseModel):
> items: list[DashboardHealthItem]
> pass_count: int
> warn_count: int
> fail_count: int
> unknown_count: int
# #endregion HealthSummaryResponse
# #endregion HealthSchemas

200
backend/src/schemas/profile.py,cover Normal file
View File

@@ -0,0 +1,200 @@
# #region ProfileSchemas [C:5] [TYPE Module] [SEMANTICS pydantic, profile, schema, superset, profile-permission-state]
# @defgroup Schemas Module group.
#
# @BRIEF Defines API schemas for profile preference persistence, security read-only snapshot, and Superset account lookup.
# @LAYER API
# @RELATION DEPENDS_ON -> [EXT:Library:pydantic]
#
# @INVARIANT Schema shapes stay stable for profile UI states and backend preference contracts.
# @DATA_CONTRACT ProfilePayload -> ProfileSchema
> from datetime import datetime
> from typing import Literal
> from pydantic import BaseModel, ConfigDict, Field
# #region ProfilePermissionState [C:3] [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Represents one permission badge state for profile read-only security view.
# @RELATION DEPENDS_ON -> ProfileSchemas
> class ProfilePermissionState(BaseModel):
> key: str
> allowed: bool
# #endregion ProfilePermissionState
# #region ProfileSecuritySummary [C:3] [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Read-only security and access snapshot for current user.
# @RELATION DEPENDS_ON -> ProfileSchemas
> class ProfileSecuritySummary(BaseModel):
> read_only: bool = True
> auth_source: str | None = None
> current_role: str | None = None
> role_source: str | None = None
> roles: list[str] = Field(default_factory=list)
> permissions: list[ProfilePermissionState] = Field(default_factory=list)
# #endregion ProfileSecuritySummary
# #region ProfilePreference [C:3] [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Represents persisted profile preference for a single authenticated user.
# @RELATION DEPENDS_ON -> ProfileSchemas
> class ProfilePreference(BaseModel):
> user_id: str
> superset_username: str | None = None
> superset_username_normalized: str | None = None
> show_only_my_dashboards: bool = False
> show_only_slug_dashboards: bool = True
> git_username: str | None = None
> git_email: str | None = None
> has_git_personal_access_token: bool = False
> git_personal_access_token_masked: str | None = None
> start_page: Literal["dashboards", "datasets", "reports"] = "dashboards"
> auto_open_task_drawer: bool = True
> dashboards_table_density: Literal["compact", "comfortable"] = "comfortable"
> telegram_id: str | None = None
> email_address: str | None = None
> notify_on_fail: bool = True
> created_at: datetime
> updated_at: datetime
> model_config = ConfigDict(from_attributes=True)
# #endregion ProfilePreference
# #region ProfilePreferenceUpdateRequest [C:3] [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Request payload for updating current user's profile settings.
# @RELATION DEPENDS_ON -> ProfileSchemas
> class ProfilePreferenceUpdateRequest(BaseModel):
> superset_username: str | None = Field(
> default=None,
> description="Apache Superset username bound to current user profile.",
> )
> show_only_my_dashboards: bool | None = Field(
> default=None,
> description='When true, "/dashboards" can auto-apply profile filter in main context.',
> )
> show_only_slug_dashboards: bool | None = Field(
> default=None,
> description='When true, "/dashboards" hides dashboards without slug by default.',
> )
> git_username: str | None = Field(
> default=None,
> description="Git author username used for commit signature.",
> )
> git_email: str | None = Field(
> default=None,
> description="Git author email used for commit signature.",
> )
> git_personal_access_token: str | None = Field(
> default=None,
> description="Personal Access Token value. Empty string clears existing token.",
> )
> start_page: Literal["dashboards", "datasets", "reports", "reports-logs"] | None = Field(
> default=None,
> description="Preferred start page after login.",
> )
> auto_open_task_drawer: bool | None = Field(
> default=None,
> description="Auto-open task drawer when long-running tasks start.",
> )
> dashboards_table_density: Literal["compact", "comfortable", "free"] | None = (
> Field(
> default=None,
> description="Preferred table density for dashboard listings.",
> )
> )
> telegram_id: str | None = Field(
> default=None,
> description="Telegram ID for notifications.",
> )
> email_address: str | None = Field(
> default=None,
> description="Email address for notifications (overrides system email).",
> )
> notify_on_fail: bool | None = Field(
> default=None,
> description="Whether to send notifications on validation failure.",
> )
# #endregion ProfilePreferenceUpdateRequest
# #region ProfilePreferenceResponse [C:3] [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Response envelope for profile preference read/update endpoints.
# @RELATION DEPENDS_ON -> ProfileSchemas
> class ProfilePreferenceResponse(BaseModel):
> status: Literal["success", "error"] = "success"
> message: str | None = None
> validation_errors: list[str] = Field(default_factory=list)
> preference: ProfilePreference
> security: ProfileSecuritySummary = Field(default_factory=ProfileSecuritySummary)
# #endregion ProfilePreferenceResponse
# #region SupersetAccountLookupRequest [C:3] [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Query contract for Superset account lookup by selected environment.
# @RELATION DEPENDS_ON -> ProfileSchemas
> class SupersetAccountLookupRequest(BaseModel):
> environment_id: str
> search: str | None = None
> page_index: int = Field(default=0, ge=0)
> page_size: int = Field(default=20, ge=1, le=100)
> sort_column: str = Field(default="username")
> sort_order: str = Field(default="desc")
# #endregion SupersetAccountLookupRequest
# #region SupersetAccountCandidate [C:3] [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Canonical account candidate projected from Superset users payload.
# @RELATION DEPENDS_ON -> ProfileSchemas
> class SupersetAccountCandidate(BaseModel):
> environment_id: str
> username: str
> display_name: str | None = None
> email: str | None = None
> is_active: bool | None = None
# #endregion SupersetAccountCandidate
# #region SupersetAccountLookupResponse [C:3] [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Response envelope for Superset account lookup (success or degraded mode).
# @RELATION DEPENDS_ON -> ProfileSchemas
> class SupersetAccountLookupResponse(BaseModel):
> status: Literal["success", "degraded"]
> environment_id: str
> page_index: int = Field(ge=0)
> page_size: int = Field(ge=1, le=100)
> total: int = Field(ge=0)
> warning: str | None = None
> items: list[SupersetAccountCandidate] = Field(default_factory=list)
# #endregion SupersetAccountLookupResponse
# #endregion ProfileSchemas

121
backend/src/schemas/settings.py,cover Normal file
View File

@@ -0,0 +1,121 @@
# #region SettingsSchemas [C:3] [TYPE Module] [SEMANTICS pydantic, schema, validate, notification-channel]
# @defgroup Schemas Module group.
# @BRIEF Pydantic schemas for application settings and automation policies.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [EXT:Library:pydantic]
> from datetime import datetime, time
> from pydantic import BaseModel, ConfigDict, Field
# #region NotificationChannel [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Structured notification channel definition for policy-level custom routing.
> class NotificationChannel(BaseModel):
> type: str = Field(
> ..., description="Notification channel type (e.g., SLACK, SMTP, TELEGRAM)"
> )
> target: str = Field(
> ..., description="Notification destination (e.g., #alerts, chat id, email)"
> )
# #endregion NotificationChannel
# #region ValidationPolicyBase [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Base schema for validation policy data.
> class ValidationPolicyBase(BaseModel):
> name: str = Field(..., description="Name of the policy")
> environment_id: str = Field(..., description="Target Superset environment ID")
> is_active: bool = Field(True, description="Whether the policy is currently active")
> dashboard_ids: list[str] = Field(
> ..., description="List of dashboard IDs to validate"
> )
> schedule_days: list[int] = Field(
> ..., description="Days of the week (0-6, 0=Sunday) to run"
> )
> window_start: time = Field(..., description="Start of the execution window")
> window_end: time = Field(..., description="End of the execution window")
> notify_owners: bool = Field(
> True, description="Whether to notify dashboard owners on failure"
> )
> custom_channels: list[NotificationChannel] | None = Field(
> None,
> description="List of additional structured notification channels",
> )
> alert_condition: str = Field(
> "FAIL_ONLY",
> description="Condition to trigger alerts: FAIL_ONLY, WARN_AND_FAIL, ALWAYS",
> )
# #endregion ValidationPolicyBase
# #region ValidationPolicyCreate [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for creating a new validation policy.
> class ValidationPolicyCreate(ValidationPolicyBase):
> pass
# #endregion ValidationPolicyCreate
# #region ValidationPolicyUpdate [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for updating an existing validation policy.
> class ValidationPolicyUpdate(BaseModel):
> name: str | None = None
> environment_id: str | None = None
> is_active: bool | None = None
> dashboard_ids: list[str] | None = None
> schedule_days: list[int] | None = None
> window_start: time | None = None
> window_end: time | None = None
> notify_owners: bool | None = None
> custom_channels: list[NotificationChannel] | None = None
> alert_condition: str | None = None
# #endregion ValidationPolicyUpdate
# #region ValidationPolicyResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for validation policy response data.
> class ValidationPolicyResponse(ValidationPolicyBase):
> id: str
> created_at: datetime
> updated_at: datetime
> model_config = ConfigDict(from_attributes=True)
# #endregion ValidationPolicyResponse
# #region TranslationScheduleItem [C:1] [TYPE Class]
# @BRIEF Response schema for a translation schedule item with joined job name.
> class TranslationScheduleItem(BaseModel):
> schedule_id: str
> job_id: str
> job_name: str | None = None
> cron_expression: str
> timezone: str
> is_active: bool
> execution_mode: str = "full"
> last_run_at: datetime | None = None
> next_run_at: datetime | None = None
> created_by: str | None = None
> created_at: datetime | None = None
> model_config = ConfigDict(from_attributes=True)
# #endregion TranslationScheduleItem
# #endregion SettingsSchemas

763
backend/src/schemas/translate.py,cover Normal file
View File

@@ -0,0 +1,763 @@
# #region TranslateSchemas [C:3] [TYPE Module] [SEMANTICS pydantic, translate, schema, translate-job-create]
# @defgroup Schemas Module group.
# @BRIEF Pydantic v2 schemas for translation API request/response serialization.
# @LAYER API
# @RELATION DEPENDS_ON -> [EXT:Library:pydantic]
! from datetime import datetime
! import re
! from typing import Any
! from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
! def _validate_bcp47_list(v: list[str] | None) -> list[str] | None:
! """Validate that each item in target_languages is a valid BCP-47 tag."""
! if not v:
! return v
! for tag in v:
! if not tag or not tag.strip():
! raise ValueError(f"Invalid BCP-47 tag: '{tag}' — must be a non-empty string")
! if not re.match(r'^[a-zA-Z]{2,8}(-[a-zA-Z0-9]{1,8})*$', tag.strip()):
! raise ValueError(
! f"Invalid BCP-47 tag: '{tag}'. "
! "Expected format like 'en', 'ru', 'zh-CN', 'pt-BR'."
! )
! return v
# #region TranslateJobCreate [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for creating a new translation job.
! class TranslateJobCreate(BaseModel):
! name: str
! description: str | None = None
! source_dialect: str = Field(..., description="Source database dialect (e.g. postgresql, clickhouse)")
! target_dialect: str = Field(..., description="Target database dialect (e.g. postgresql, clickhouse)")
! database_dialect: str | None = Field(None, description="Detected dialect from Superset connection at save time")
! source_datasource_id: str | None = Field(None, description="Superset datasource ID")
! source_table: str | None = Field(None, description="Source table name")
! target_schema: str | None = Field(None, description="Target table schema")
! target_table: str | None = Field(None, description="Target table name")
! source_key_cols: list[str] | None = Field(default_factory=list, description="Source key column names")
! target_key_cols: list[str] | None = Field(default_factory=list, description="Target key column names")
! translation_column: str | None = Field(None, description="Source column to translate")
! target_column: str | None = Field(None, description="Target column for translated output (defaults to translation_column)")
! target_language_column: str | None = Field(None, description="Target column for language code (e.g. 'ru', 'en')")
! target_source_column: str | None = Field(None, description="Target column for source/original text")
! target_source_language_column: str | None = Field(None, description="Target column for detected source language (BCP-47)")
! context_columns: list[str] | None = Field(default_factory=list, description="Context column names")
! target_language: str | None = Field(None, description="Target language code [DEPRECATED: use target_languages]")
! target_languages: list[str] | None = Field(default_factory=list, description="List of BCP-47 target language codes")
! provider_id: str | None = Field(None, description="LLM provider ID")
! _validate_target_languages = field_validator("target_languages")(_validate_bcp47_list)
! batch_size: int = Field(50, description="Records per batch")
! disable_reasoning: bool = Field(False, description="If true, pass reasoning_effort:none to LLM to save output tokens")
! upsert_strategy: str = Field("MERGE", description="UPSERT strategy: MERGE, INSERT, UPDATE")
! dictionary_ids: list[str] | None = Field(default_factory=list, description="Associated terminology dictionary IDs")
! environment_id: str | None = Field(None, description="Superset environment ID")
! target_database_id: str | None = Field(None, description="Superset database ID for SQL Lab insert target")
# Direct DB insert
! insert_method: str = Field("sqllab", description="Insert execution method: sqllab|direct_db")
! connection_id: str | None = Field(None, description="UUID referencing a DatabaseConnection in GlobalSettings.connections (required when insert_method=direct_db)")
! @model_validator(mode="after")
! def validate_insert_method(self):
! if self.insert_method == "direct_db" and not self.connection_id:
! raise ValueError("connection_id is required when insert_method='direct_db'")
! return self
# #endregion TranslateJobCreate
# #region TranslateJobUpdate [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for updating an existing translation job.
! class TranslateJobUpdate(BaseModel):
! name: str | None = None
! description: str | None = None
! source_dialect: str | None = None
! target_dialect: str | None = None
! database_dialect: str | None = None
! source_datasource_id: str | None = None
! source_table: str | None = None
! target_schema: str | None = None
! target_table: str | None = None
! source_key_cols: list[str] | None = None
! target_key_cols: list[str] | None = None
! translation_column: str | None = None
! target_column: str | None = None
! target_language_column: str | None = None
! target_source_column: str | None = None
! target_source_language_column: str | None = None
! context_columns: list[str] | None = None
! target_language: str | None = None
! target_languages: list[str] | None = None
! provider_id: str | None = None
! _validate_target_languages = field_validator("target_languages")(_validate_bcp47_list)
! batch_size: int | None = None
! disable_reasoning: bool | None = None
! upsert_strategy: str | None = None
! status: str | None = None
! dictionary_ids: list[str] | None = None
! environment_id: str | None = None
! target_database_id: str | None = None
! insert_method: str | None = None
! connection_id: str | None = None
# #endregion TranslateJobUpdate
# #region TranslateJobResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for translation job API responses.
! class TranslateJobResponse(BaseModel):
! id: str
! name: str
! description: str | None = None
! source_dialect: str
! target_dialect: str
! database_dialect: str | None = None
! source_datasource_id: str | None = None
! source_table: str | None = None
! target_schema: str | None = None
! target_table: str | None = None
! source_key_cols: list[str] | None = None
! target_key_cols: list[str] | None = None
! translation_column: str | None = None
! target_column: str | None = None
! target_language_column: str | None = None
! target_source_column: str | None = None
! target_source_language_column: str | None = None
! context_columns: list[str] | None = None
# source_language removed — deprecated, per-row auto-detected
! target_languages: list[str] | None = None
! provider_id: str | None = None
! batch_size: int = 50
! disable_reasoning: bool = False
! upsert_strategy: str = "MERGE"
! status: str
! created_by: str | None = None
! created_at: datetime
! updated_at: datetime | None = None
! dictionary_ids: list[str] | None = None
! environment_id: str | None = None
! target_database_id: str | None = None
! insert_method: str = "sqllab"
! connection_id: str | None = None
! model_config = ConfigDict(from_attributes=True)
# #endregion TranslateJobResponse
# #region DatasourceColumnResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for datasource column metadata response.
! class DatasourceColumnResponse(BaseModel):
! name: str
! type: str | None = None
! is_physical: bool = True
! is_dttm: bool = False
! description: str | None = None
# #endregion DatasourceColumnResponse
# #region DatasourceColumnsResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for datasource columns endpoint response.
! class DatasourceColumnsResponse(BaseModel):
! datasource_id: int
! datasource_name: str | None = None
! schema_name: str | None = None
! database_dialect: str
! columns: list[DatasourceColumnResponse] = []
! model_config = ConfigDict(from_attributes=True)
# #endregion DatasourceColumnsResponse
# #region DuplicateJobResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for duplicate job response.
! class DuplicateJobResponse(BaseModel):
! id: str
! name: str
! message: str = "Job duplicated successfully"
! model_config = ConfigDict(from_attributes=True)
# #endregion DuplicateJobResponse
# #region DictionaryCreate [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for creating a new terminology dictionary.
! class DictionaryCreate(BaseModel):
! name: str
! description: str | None = None
! is_active: bool = True
# #endregion DictionaryCreate
# #region DictionaryImport [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for importing entries into a terminology dictionary.
! class DictionaryImport(BaseModel):
! content: str = Field(..., description="CSV or TSV content as raw string")
! delimiter: str | None = Field(None, description="Detected or forced delimiter: ',' or '\\t'. Auto-detect if omitted.")
! on_conflict: str = Field("overwrite", description="'overwrite' or 'keep_existing' or 'cancel'")
! preview_only: bool = Field(False, description="If true, return preview without applying")
! default_source_language: str | None = Field(None, description="Default BCP-47 source language when column missing in data")
! default_target_language: str | None = Field(None, description="Default BCP-47 target language when column missing in data")
# #endregion DictionaryImport
# #region DictionaryResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for terminology dictionary API responses.
! class DictionaryResponse(BaseModel):
! id: str
! name: str
! description: str | None = None
! is_active: bool
! created_by: str | None = None
! created_at: datetime
! updated_at: datetime | None = None
! entry_count: int | None = None
! model_config = ConfigDict(from_attributes=True)
# #endregion DictionaryResponse
# #region DictionaryEntryCreate [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for adding/editing a dictionary entry with language pair support.
! class DictionaryEntryCreate(BaseModel):
! source_term: str = Field(..., description="Source term to translate")
! target_term: str = Field(..., description="Target/translated term")
! context_notes: str | None = Field(None, description="Optional context notes")
! source_language: str = Field("und", description="BCP-47 source language code (default: und)")
! target_language: str = Field("und", description="BCP-47 target language code (default: und)")
! is_regex: bool = Field(False, description="Whether source_term is a regex pattern")
! model_config = ConfigDict(from_attributes=True)
# #endregion DictionaryEntryCreate
# #region DictionaryEntryResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for dictionary entry API responses.
! class DictionaryEntryResponse(BaseModel):
! id: str
! dictionary_id: str
! source_term: str
! source_term_normalized: str
! target_term: str
! source_language: str | None = None
! target_language: str | None = None
! context_notes: str | None = None
! context_data: dict[str, Any] | None = None
! usage_notes: str | None = None
! has_context: bool = False
! is_regex: bool = False
! context_source: str | None = None
! origin_source_language: str | None = None
! created_at: datetime
! updated_at: datetime | None = None
! model_config = ConfigDict(from_attributes=True)
# #endregion DictionaryEntryResponse
# #region DictionaryImportResult [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for dictionary import result.
! class DictionaryImportResult(BaseModel):
! total: int = 0
! created: int = 0
! updated: int = 0
! skipped: int = 0
! errors: list[dict[str, Any]] = Field(default_factory=list)
! preview: list[dict[str, Any]] = Field(default_factory=list, description="Preview rows with conflict flags")
# #endregion DictionaryImportResult
# #region PreviewRequest [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for triggering a translation preview.
! class PreviewRequest(BaseModel):
! sample_size: int = Field(10, ge=1, le=100, description="Number of sample rows to preview")
! prompt_template: str | None = Field(None, description="Optional custom prompt template")
! env_id: str | None = Field(None, description="Superset environment ID for preview data fetch")
# #endregion PreviewRequest
# #region PreviewRowUpdate [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for approving/editing/rejecting a preview row.
! class PreviewRowUpdate(BaseModel):
! action: str = Field(..., description="'approve', 'reject', or 'edit'")
! translation: str | None = Field(None, description="Edited translation (required for 'edit' action)")
! feedback: str | None = Field(None, description="Optional feedback/comment")
! language_code: str | None = Field(None, description="BCP-47 language code for per-language actions (optional, applies to all if omitted)")
# #endregion PreviewRowUpdate
# #region PreviewAcceptResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for preview accept response.
! class PreviewAcceptResponse(BaseModel):
! id: str
! job_id: str
! status: str
! created_by: str | None = None
! created_at: datetime
! expires_at: datetime | None = None
! records: list['PreviewRow'] = []
! target_languages: list[str] = Field(default_factory=list, description="Target languages for this preview")
! model_config = ConfigDict(from_attributes=True)
# #endregion PreviewAcceptResponse
# #region CostEstimate [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for cost estimation in preview response.
! class CostEstimate(BaseModel):
! sample_size: int = 0
! num_languages: int = 1
! sample_prompt_tokens: int = 0
! sample_output_tokens: int = 0
! sample_total_tokens: int = 0
! sample_cost: float = 0.0
! estimated_total_rows: int = 0
! estimated_tokens: int = 0
! estimated_cost: float = 0.0
! warning: str | None = None
# #endregion CostEstimate
# #region TermCorrectionSubmit [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for submitting a term correction in a translation preview.
! class TermCorrectionSubmit(BaseModel):
! source_term: str
! incorrect_target_term: str
! corrected_target_term: str
! dictionary_id: str | None = Field(None, description="Target dictionary ID (language-filtered)")
! origin_run_id: str | None = Field(None, description="Run ID from which this correction originated")
! origin_row_key: str | None = Field(None, description="Row key within the run")
! context_data: dict[str, Any] | None = Field(None, description="Context data for the dictionary entry")
! usage_notes: str | None = Field(None, description="Usage notes for the dictionary entry")
! keep_context: bool = Field(True, description="If false, clear context_data and set has_context=False on dictionary entry")
# #endregion TermCorrectionSubmit
# #region TermCorrectionBulkSubmit [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for submitting multiple term corrections atomically.
! class TermCorrectionBulkSubmit(BaseModel):
! corrections: list[TermCorrectionSubmit]
! dictionary_id: str = Field(..., description="Target dictionary ID for all corrections")
# #endregion TermCorrectionBulkSubmit
# #region CorrectionConflictResult [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for reporting conflicts in correction submission.
! class CorrectionConflictResult(BaseModel):
! source_term: str
! existing_target_term: str
! submitted_target_term: str
! action: str = "keep_existing"
# #endregion CorrectionConflictResult
# #region CorrectionSubmitResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for correction submission response.
! class CorrectionSubmitResponse(BaseModel):
! entry_id: str | None = None
! action: str # "created", "updated", "conflict_detected", "skipped"
! source_term: str
! target_term: str
! conflict: CorrectionConflictResult | None = None
! message: str | None = None
# #endregion CorrectionSubmitResponse
# #region ScheduleConfig [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for configuring a recurring translation schedule.
! class ScheduleConfig(BaseModel):
! cron_expression: str = Field(..., description="Cron expression for scheduling (e.g. '0 2 * * *')")
! timezone: str = Field("UTC", description="Timezone for the cron schedule (e.g. 'UTC', 'Europe/Moscow')")
! is_active: bool = True
! execution_mode: str = Field("full", description="Translation execution mode: 'full' or 'new_key_only'")
# #endregion ScheduleConfig
# #region ScheduleResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for schedule API responses.
! class ScheduleResponse(BaseModel):
! id: str
! job_id: str
! cron_expression: str
! timezone: str
! is_active: bool
! execution_mode: str = "full"
! last_run_at: datetime | None = None
! next_run_at: datetime | None = None
! created_by: str | None = None
! created_at: datetime
! updated_at: datetime | None = None
! model_config = ConfigDict(from_attributes=True)
# #endregion ScheduleResponse
# #region NextExecutionResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for next execution preview.
! class NextExecutionResponse(BaseModel):
! job_id: str
! cron_expression: str
! timezone: str
! next_executions: list[str] = []
# #endregion NextExecutionResponse
# #region TranslationRunResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for translation run API responses.
! class TranslationRunResponse(BaseModel):
! id: str
! job_id: str
! status: str
! trigger_type: str | None = None
! started_at: datetime | None = None
! completed_at: datetime | None = None
! error_message: str | None = None
! total_records: int = 0
! successful_records: int = 0
! failed_records: int = 0
! skipped_records: int = 0
! cache_hits: int = 0
! insert_status: str | None = None
! insert_method: str | None = None
! connection_snapshot: dict[str, Any] | None = None
! superset_execution_id: str | None = None
! config_snapshot: dict[str, Any] | None = None
! key_hash: str | None = None
! config_hash: str | None = None
! dict_snapshot_hash: str | None = None
! created_by: str | None = None
! created_at: datetime
! language_stats: list['TranslationRunLanguageStatsResponse'] | None = None
! model_config = ConfigDict(from_attributes=True)
# #endregion TranslationRunResponse
# #region RunDetailResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for detailed run response including records, events, and config snapshot.
! class RunDetailResponse(BaseModel):
! run: TranslationRunResponse
! records: list[dict[str, Any]] = Field(default_factory=list, description="Paginated translation records")
! events: list[dict[str, Any]] = Field(default_factory=list, description="Run events")
! event_invariants: dict[str, Any] | None = None
! batch_count: int = 0
# #endregion RunDetailResponse
# #region RunHistoryFilter [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for filtering run history.
! class RunHistoryFilter(BaseModel):
! job_id: str | None = None
! status: str | None = None
! trigger_type: str | None = None
! created_by: str | None = None
! date_from: datetime | None = None
! date_to: datetime | None = None
! page: int = 1
! page_size: int = 20
# #endregion RunHistoryFilter
# #region RunListResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for paginated run list response.
! class RunListResponse(BaseModel):
! items: list[TranslationRunResponse] = []
! total: int = 0
! page: int = 1
! page_size: int = 20
# #endregion RunListResponse
# #region AggregatedMetricsResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for aggregated per-job metrics.
! class AggregatedMetricsResponse(BaseModel):
! job_id: str
! total_runs: int = 0
! successful_runs: int = 0
! failed_runs: int = 0
! cancelled_runs: int = 0
! total_records: int = 0
! successful_records: int = 0
! failed_records: int = 0
! skipped_records: int = 0
! cumulative_tokens: int | None = None
! cumulative_cost: float | None = None
! avg_duration_ms: float | None = None
! last_run_at: datetime | None = None
! next_scheduled_run: datetime | None = None
# #endregion AggregatedMetricsResponse
# #region TranslationPreviewLanguageResponse [C:1] [TYPE Class]
# @BRIEF Schema for per-language preview data in API responses.
! class TranslationPreviewLanguageResponse(BaseModel):
! language_code: str
! source_language_detected: str | None = None
! translated_value: str | None = None
! user_edit: str | None = None
! final_value: str | None = None
! status: str = "pending"
! needs_review: bool = False
! model_config = ConfigDict(from_attributes=True)
# #endregion TranslationPreviewLanguageResponse
# #region PreviewRow [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF A single row in a translation preview showing original and translated content side by side.
! class PreviewRow(BaseModel):
! id: str
! source_sql: str | None = None
! target_sql: str | None = None
! source_object_type: str | None = None
! source_object_id: str | None = None
! source_object_name: str | None = None
! status: str = "PENDING"
! feedback: str | None = None
! source_language_detected: str | None = None
! needs_review: bool = False
! languages: list[TranslationPreviewLanguageResponse] = Field(default_factory=list, description="Per-language translation data")
# #endregion PreviewRow
# #region TranslationPreviewResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for translation preview session API responses.
! class TranslationPreviewResponse(BaseModel):
! id: str
! job_id: str
! run_id: str | None = None
! status: str
! created_by: str | None = None
! created_at: datetime
! expires_at: datetime | None = None
! records: list[PreviewRow] = []
! target_languages: list[str] = Field(default_factory=list, description="Target languages for this preview")
! cost_estimate: CostEstimate | None = None
! model_config = ConfigDict(from_attributes=True)
# #endregion TranslationPreviewResponse
# #region TranslationBatchResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for translation batch API responses.
! class TranslationBatchResponse(BaseModel):
! id: str
! run_id: str
! batch_index: int
! status: str
! total_records: int = 0
! successful_records: int = 0
! failed_records: int = 0
! started_at: datetime | None = None
! completed_at: datetime | None = None
! created_at: datetime
! model_config = ConfigDict(from_attributes=True)
# #endregion TranslationBatchResponse
# #region MetricsResponse [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for translation metrics API responses.
! class MetricsResponse(BaseModel):
! job_id: str
! snapshot_date: datetime
! total_jobs: int = 0
! total_runs: int = 0
! total_records: int = 0
! successful_records: int = 0
! failed_records: int = 0
! skipped_records: int = 0
! avg_duration_ms: int | None = None
! p50_duration_ms: int | None = None
! p95_duration_ms: int | None = None
! p99_duration_ms: int | None = None
! model_config = ConfigDict(from_attributes=True)
# #endregion MetricsResponse
# #region TranslationLanguageResponse [C:1] [TYPE Class]
# @BRIEF Schema for per-language translation data in API responses.
! class TranslationLanguageResponse(BaseModel):
! language_code: str
! source_language_detected: str | None = None
! translated_value: str | None = None
! user_edit: str | None = None
! final_value: str | None = None
! status: str = "pending"
! needs_review: bool = False
! language_overridden: bool = False
! model_config = ConfigDict(from_attributes=True)
# #endregion TranslationLanguageResponse
# #region TranslationRunLanguageStatsResponse [C:1] [TYPE Class]
# @BRIEF Schema for per-language statistics in run API responses.
! class TranslationRunLanguageStatsResponse(BaseModel):
! language_code: str
! total_rows: int = 0
! translated_rows: int = 0
! failed_rows: int = 0
! skipped_rows: int = 0
! token_count: int = 0
! estimated_cost: float = 0.0
! model_config = ConfigDict(from_attributes=True)
# #endregion TranslationRunLanguageStatsResponse
# #region OverrideLanguageRequest [C:1] [TYPE Class]
# @BRIEF Schema for manually overriding the detected source language of a translation.
! class OverrideLanguageRequest(BaseModel):
! source_language: str = Field(..., description="BCP-47 language code to override with")
# #endregion OverrideLanguageRequest
# #region BulkFindReplaceRequest [C:1] [TYPE Class]
# @BRIEF Schema for bulk find-and-replace within translations for a target language.
! class BulkFindReplaceRequest(BaseModel):
! find_pattern: str = Field(..., description="Text or regex pattern to find", max_length=500)
! is_regex: bool = Field(False, description="Whether find_pattern is a regular expression")
! replacement_text: str = Field(..., description="Replacement text")
! target_language: str = Field(..., description="BCP-47 language code to target")
! preview: bool = Field(True, description="If true, return matches without applying replacements")
! submit_to_dictionary: bool = Field(False, description="If true, also submit corrections to dictionary")
! dictionary_id: str | None = Field(None, description="Target dictionary ID for submitting corrections")
! usage_notes: str | None = Field(None, description="Usage notes for dictionary entries")
! submit_to_dictionary_with_context: bool = Field(False, description="If true, auto-capture source row context when submitting to dictionary")
! @field_validator("find_pattern")
! @classmethod
! def validate_pattern_length(cls, v: str) -> str:
! """Reject patterns longer than 500 characters to prevent ReDoS."""
! if len(v) > 500:
! raise ValueError("Pattern too long (max 500 characters)")
! return v
# #endregion BulkFindReplaceRequest
# #region InlineEditRequest [C:1] [TYPE Class]
# @BRIEF Schema for inline editing a translated value on a completed run result.
! class InlineEditRequest(BaseModel):
! final_value: str = Field(..., description="Corrected/edited translation text")
! submit_to_dictionary: bool = Field(False, description="If true, also save correction to dictionary")
! dictionary_id: str | None = Field(None, description="Target dictionary ID for saving correction")
! context_data_override: dict[str, Any] | None = Field(None, description="Override auto-captured context data for dictionary entry")
! usage_notes: str | None = Field(None, description="Usage notes for the dictionary entry")
! keep_context: bool = Field(True, description="If false, clear context_data and set has_context=False on dictionary entry")
# #endregion InlineEditRequest
# #region BulkReplacePreviewItem [C:1] [TYPE Class]
# @BRIEF A single item in a bulk replace preview list.
! class BulkReplacePreviewItem(BaseModel):
! record_id: str
! language_code: str
! source_term: str
! current_value: str
! new_value: str
! source_language_detected: str | None = None
# #endregion BulkReplacePreviewItem
# #region BulkReplaceResponse [C:1] [TYPE Class]
# @BRIEF Response for bulk find-and-replace operation.
! class BulkReplaceResponse(BaseModel):
! rows_affected: int = 0
! corrections_submitted: int = 0
! preview: list[BulkReplacePreviewItem] = Field(default_factory=list)
# #endregion BulkReplaceResponse
# #region InlineCorrectionSubmit [C:1] [TYPE Class]
# @BRIEF Schema for submitting an inline correction for a specific translated record.
! class InlineCorrectionSubmit(BaseModel):
! record_id: str = Field(..., description="Translation record ID")
! language_code: str = Field(..., description="BCP-47 language code of the translated value")
! source_term: str = Field(..., description="Original source term that was incorrectly translated")
! corrected_term: str = Field(..., description="Corrected translation")
! dictionary_id: str = Field(..., description="Target dictionary ID to also save the correction")
! source_language: str = Field(..., description="BCP-47 source language code")
# #endregion InlineCorrectionSubmit
# #region TargetSchemaValidationRequest [C:1] [TYPE Class]
# @BRIEF Schema for requesting target table schema validation.
! class TargetSchemaValidationRequest(BaseModel):
! environment_id: str = Field(..., description="Superset environment ID")
! target_database_id: str = Field(..., description="Superset database ID for the target DB (SQL Lab)")
! target_schema: str = Field("public", description="Target table schema (default: public)")
! target_table: str = Field(..., description="Target table name")
# Column mapping from job config (everything that affects expected columns)
! target_key_cols: list[str] | None = Field(default_factory=list, description="Target key column names")
! target_column: str | None = Field(None, description="Target column for translated output")
! translation_column: str | None = Field(None, description="Source column name (fallback for target_column)")
! target_language_column: str | None = Field(None, description="Column for language code")
! target_source_column: str | None = Field(None, description="Column for source text")
! target_source_language_column: str | None = Field(None, description="Column for detected source language")
# Direct DB support — when insert_method=direct_db, use connection_id instead of target_database_id
! insert_method: str | None = Field(None, description="Insert method: 'sqllab' or 'direct_db'")
! connection_id: str | None = Field(None, description="Direct DB connection ID (required when insert_method=direct_db)")
# #endregion TargetSchemaValidationRequest
# #region TargetSchemaColumnInfo [C:1] [TYPE Class]
# @BRIEF Schema for a single column in the schema validation response.
! class TargetSchemaColumnInfo(BaseModel):
! name: str
! data_type: str | None = None
! is_nullable: bool = True
# #endregion TargetSchemaColumnInfo
# #region TargetSchemaValidationResponse [C:1] [TYPE Class]
# @BRIEF Schema for target table schema validation response.
! class TargetSchemaValidationResponse(BaseModel):
! table_exists: bool = False
! error: str | None = None
! expected_columns: list[TargetSchemaColumnInfo] = Field(default_factory=list, description="Columns that the INSERT expects")
! actual_columns: list[TargetSchemaColumnInfo] = Field(default_factory=list, description="Columns that exist in the target table")
! missing_columns: list[TargetSchemaColumnInfo] = Field(default_factory=list, description="Expected columns NOT found in target table")
! extra_columns: list[TargetSchemaColumnInfo] = Field(default_factory=list, description="Actual columns NOT in expected list")
! all_present: bool = False
! database_name: str | None = Field(default=None, description="Actual database name queried (from Superset)")
! database_backend: str | None = Field(default=None, description="Actual database backend dialect (from Superset)")
# #endregion TargetSchemaValidationResponse
# #endregion TranslateSchemas

241
backend/src/schemas/validation.py,cover Normal file
View File

@@ -0,0 +1,241 @@
# #region ValidationSchemas [C:3] [TYPE Module] [SEMANTICS pydantic, validation, schema, task, run]
# @defgroup Schemas Module group.
# @BRIEF Pydantic v2 schemas for validation task management and run history API serialization.
# @LAYER API
# @RELATION DEPENDS_ON -> [EXT:Library:pydantic]
> from datetime import datetime
> from typing import Any, Literal
> from pydantic import BaseModel, ConfigDict, Field
# #region SourceInput [C:1] [TYPE Class]
# @BRIEF Schema for creating a validation source from a dashboard ID or Superset URL.
# @RATIONALE v2 introduces typed source objects allowing both dashboard_id (existing numeric
# reference) and dashboard_url (full Superset link that gets parsed/extracted) inputs.
# @REJECTED Keeping only dashboard_id rejected — dashboard_url enables sharing Superset links
# directly without first resolving numeric IDs manually.
> class SourceInput(BaseModel):
> type: Literal["dashboard_id", "dashboard_url"] = Field(
> ..., description="Source type: 'dashboard_id' for numeric IDs, 'dashboard_url' for full Superset URLs"
> )
> value: str = Field(..., description="Source value: numeric dashboard ID or full Superset dashboard URL")
# #endregion SourceInput
# #region SourceResponse [C:1] [TYPE Class]
# @BRIEF Schema for displaying a validation source in API responses.
> class SourceResponse(BaseModel):
> id: str
> type: str
> value: str
> status: str = "valid"
> title: str | None = None
> resolved_dashboard_id: str | None = None
> last_error: str | None = None
> last_checked_at: datetime | None = None
> created_at: datetime | None = None
> model_config = ConfigDict(from_attributes=True)
# #endregion SourceResponse
# #region ValidationTaskCreate [C:2] [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for creating a new validation task (policy). v2 adds sources, screenshot_enabled,
# logs_enabled, execute_chart_data, llm_batch_size, concurrency_limit, and prompt_template.
# @RATIONALE All v2 fields are optional to support gradual migration. dashboard_ids is kept for
# backward compat — when non-empty and sources is empty, auto-create ValidationSource rows.
# @REJECTED Breaking dashboard_ids removal rejected — existing UI callers and API clients still
# send dashboard_ids; removing it would break migration without frontend changes.
> class ValidationTaskCreate(BaseModel):
> name: str = Field(..., description="Human-readable task name")
> description: str | None = Field(None, description="Optional task description")
> environment_id: str = Field(..., description="Target Superset environment ID")
> dashboard_ids: list[str] = Field(default_factory=list, description="List of dashboard IDs (v1 compat — auto-creates ValidationSource rows if non-empty)")
> provider_id: str = Field(..., description="LLM provider ID — must be multimodal when screenshot_enabled=True")
> schedule_days: list[int] | None = Field(None, description="Days of week (0-6, 0=Sunday). Omit or empty for manual-only.")
> window_start: str | None = Field(None, description="Window start time, HH:MM format")
> window_end: str | None = Field(None, description="Window end time, HH:MM format")
> notify_owners: bool = Field(True, description="Notify dashboard owners on failure")
> custom_channels: list[str] | None = Field(None, description="Additional notification channels")
> alert_condition: str = Field("FAIL_ONLY", description="FAIL_ONLY, WARN_AND_FAIL, or ALWAYS")
> is_active: bool = Field(True, description="Whether the task is active")
# v2 fields (all optional for gradual migration)
> screenshot_enabled: bool = Field(True, description="Enable screenshot-based validation (Path A)")
> logs_enabled: bool = Field(True, description="Enable log collection during validation")
> execute_chart_data: bool = Field(False, description="Execute chart data queries during validation")
> llm_batch_size: int = Field(1, description="Number of dashboards to send in a single LLM batch", ge=1)
> policy_dashboard_concurrency_limit: int = Field(3, description="Max concurrent dashboard validations for this policy", ge=1)
> prompt_template: str | None = Field(None, description="Custom prompt template override")
> sources: list[SourceInput] | None = Field(None, description="v2 source definitions (takes precedence over dashboard_ids)")
# #endregion ValidationTaskCreate
# #region ValidationTaskUpdate [C:2] [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for updating an existing validation task (policy). Includes v2 fields.
# @RATIONALE All fields optional (partial update). When sources is provided, replaces all existing
# ValidationSource rows for the policy.
> class ValidationTaskUpdate(BaseModel):
> name: str | None = None
> description: str | None = None
> environment_id: str | None = None
> dashboard_ids: list[str] | None = None
> provider_id: str | None = None
> schedule_days: list[int] | None = None
> window_start: str | None = None
> window_end: str | None = None
> notify_owners: bool | None = None
> custom_channels: list[str] | None = None
> alert_condition: str | None = None
> is_active: bool | None = None
# v2 fields
> screenshot_enabled: bool | None = None
> logs_enabled: bool | None = None
> execute_chart_data: bool | None = None
> llm_batch_size: int | None = None
> policy_dashboard_concurrency_limit: int | None = None
> prompt_template: str | None = None
> sources: list[SourceInput] | None = Field(None, description="Replace all sources when provided")
# #endregion ValidationTaskUpdate
# #region ValidationTaskResponse [C:2] [TYPE Class]
# @defgroup Schemas Module group.
# @BRIEF Schema for validation task API responses — includes last run info and v2 fields.
> class ValidationTaskResponse(BaseModel):
> id: str
> name: str
> description: str | None = Field(None, description="Optional task description")
> environment_id: str
> is_active: bool
> dashboard_ids: list[str]
> provider_id: str | None = None
> schedule_days: list[int] | None = None
> window_start: str | None = None
> window_end: str | None = None
> notify_owners: bool = True
> custom_channels: list[str] | None = None
> alert_condition: str = "FAIL_ONLY"
> created_at: datetime
> updated_at: datetime | None = None
> last_run_status: str | None = None
> last_run_at: datetime | None = None
> last_run_summary: dict | None = Field(None, description="Aggregated counts: {pass: N, warn: N, fail: N}")
# v2 fields
> screenshot_enabled: bool = True
> logs_enabled: bool = True
> execute_chart_data: bool = False
> llm_batch_size: int = 1
> policy_dashboard_concurrency_limit: int = 3
> prompt_template: str | None = None
> sources: list[SourceResponse] = Field(default_factory=list, description="Resolved validation sources")
> model_config = ConfigDict(from_attributes=True)
# #endregion ValidationTaskResponse
# #region ValidationTaskListResponse [C:1] [TYPE Class]
# @BRIEF Schema for paginated task list response.
> class ValidationTaskListResponse(BaseModel):
> items: list[ValidationTaskResponse] = []
> total: int = 0
> page: int = 1
> page_size: int = 20
# #endregion ValidationTaskListResponse
# #region TriggerRunResponse [C:1] [TYPE Class]
# @BRIEF Schema for trigger-run response — returns spawned task ID and run ID.
> class TriggerRunResponse(BaseModel):
> task_id: str
> spawned_task_id: str
> run_id: str | None = Field(None, description="ValidationRun ID (v2)")
> status: str = "PENDING"
> message: str = "Validation task spawned"
# #endregion TriggerRunResponse
# #region RunResponse [C:1] [TYPE Class]
# @BRIEF Schema for v2 ValidationRun display (aggregate run, not individual record).
> class RunResponse(BaseModel):
> id: str
> policy_id: str
> task_id: str | None = None
> status: str
> trigger: str = "manual"
> started_at: datetime | None = None
> finished_at: datetime | None = None
> dashboard_count: int = 0
> pass_count: int = 0
> warn_count: int = 0
> fail_count: int = 0
> unknown_count: int = 0
> created_at: datetime | None = None
> model_config = ConfigDict(from_attributes=True)
# #endregion RunResponse
# #region RunListResponse [C:1] [TYPE Class]
# @BRIEF Schema for paginated v2 run list response.
> class RunListResponse(BaseModel):
> items: list[RunResponse] = []
> total: int = 0
> page: int = 1
> page_size: int = 20
# #endregion RunListResponse
# #region RecordResponse [C:1] [TYPE Class]
# @BRIEF Schema for a single ValidationRecord in cross-task runs listing.
> class RecordResponse(BaseModel):
> id: str
> policy_id: str | None = None
> run_id: str | None = None
> dashboard_id: str
> dashboard_title: str | None = None
> environment_id: str | None = None
> status: str
> summary: str = ""
> timestamp: datetime | None = None
> execution_path: str | None = None
> screenshot_path: str | None = None
> screenshot_paths: list[str] = Field(default_factory=list)
> issues: list[dict[str, Any]] = Field(default_factory=list)
> raw_response: str | None = None
> dataset_health: dict[str, Any] | None = None
> chart_data_results: list[dict[str, Any]] = Field(default_factory=list)
> tab_screenshots: list[dict[str, Any]] = Field(default_factory=list)
> logs_sent_to_llm: list[str] = Field(default_factory=list)
> token_usage: dict[str, Any] | None = None
> timings: dict[str, Any] | None = None
> model_config = ConfigDict(from_attributes=True)
# #endregion RecordResponse
# #region RecordListResponse [C:1] [TYPE Class]
# @BRIEF Schema for paginated cross-task runs listing.
> class RecordListResponse(BaseModel):
> items: list[RecordResponse] = []
> total: int = 0
> page: int = 1
> page_size: int = 20
# #endregion RecordListResponse
# #region RunDetailResponse [C:1] [TYPE Class]
# @BRIEF Schema for v2 run detail — ValidationRun + all its ValidationRecords.
> class RunDetailResponse(BaseModel):
> run: RunResponse
> records: list[dict[str, Any]] = Field(default_factory=list)
# #endregion RunDetailResponse
# #endregion ValidationSchemas

145
backend/src/services/auth_service.py,cover Normal file
View File

@@ -0,0 +1,145 @@
# #region auth_service [C:5] [TYPE Module] [SEMANTICS sqlalchemy, auth, credential, session, jwt]
# @defgroup Services Module group.
# @BRIEF Orchestrates credential authentication and ADFS JIT user provisioning.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [AuthRepository]
# @RELATION DEPENDS_ON -> [verify_password]
# @RELATION DEPENDS_ON -> [create_access_token]
# @RELATION DEPENDS_ON -> [User]
# @RELATION DEPENDS_ON -> [Role]
# @INVARIANT Authentication succeeds only for active users with valid credentials; issued sessions encode subject and scopes from assigned roles.
# @PRE Core auth models and security utilities available.
# @POST User identity verified and session tokens issued according to role scopes.
# @SIDE_EFFECT Writes last login timestamps and JIT-provisions external users.
# @DATA_CONTRACT [Credentials | ADFSClaims] -> [UserEntity | SessionToken]
> from datetime import UTC, datetime
> from typing import Any
> from sqlalchemy.orm import Session
> from ..core.auth.jwt import create_access_token
> from ..core.auth.logger import log_security_event
> from ..core.auth.repository import AuthRepository
> from ..core.auth.security import verify_password
> from ..core.logger import belief_scope
> from ..models.auth import User
# #region AuthService [C:3] [TYPE Class]
# @defgroup Services Module group.
# @BRIEF Provides high-level authentication services.
# @RELATION DEPENDS_ON -> [AuthRepository]
# @RELATION DEPENDS_ON -> [User]
# @RELATION DEPENDS_ON -> [Role]
> class AuthService:
# region AuthService_init [TYPE Function]
# @PURPOSE: Initializes the authentication service with repository access over an active DB session.
# @PRE db is a valid SQLAlchemy Session instance bound to the auth persistence context.
# @POST self.repo is initialized and ready for auth user/role CRUD operations.
# @SIDE_EFFECT Allocates AuthRepository and binds it to the provided Session.
# @DATA_CONTRACT Input(Session) -> Model(AuthRepository)
# @PARAM db (Session) - SQLAlchemy session.
> def __init__(self, db: Session):
> self.db = db
> self.repo = AuthRepository(db)
# endregion AuthService_init
# region AuthService.authenticate_user [TYPE Function]
# @PURPOSE: Validates credentials and account state for local username/password authentication.
# @PRE username and password are non-empty credential inputs.
# @POST Returns User only when user exists, is active, and password hash verification succeeds; otherwise returns None.
# @SIDE_EFFECT Persists last_login update for successful authentications via repository.
# @DATA_CONTRACT Input(str username, str password) -> Output(User | None)
# @RELATION DEPENDS_ON ->[AuthRepository]
# @RELATION CALLS ->[verify_password]
# @RELATION DEPENDS_ON ->[User]
# @PARAM username (str) - The username.
# @PARAM password (str) - The plain password.
# @RETURN Optional[User] - The authenticated user or None.
> def authenticate_user(self, username: str, password: str) -> User | None:
> with belief_scope("auth.authenticate_user"):
> user = self.repo.get_user_by_username(username)
> if not user or not user.is_active:
> return None
> if not verify_password(password, user.password_hash):
> return None
# Update last login
> user.last_login = datetime.now(UTC)
> self.db.commit()
> self.db.refresh(user)
> return user
# endregion AuthService.authenticate_user
# region AuthService.create_session [TYPE Function]
# @PURPOSE: Issues an access token payload for an already authenticated user.
# @PRE user is a valid User entity containing username and iterable roles with role.name values.
# @POST Returns session dict with non-empty access_token and token_type='bearer'.
# @SIDE_EFFECT Generates signed JWT via auth JWT provider.
# @DATA_CONTRACT Input(User) -> Output(Dict[str, str]{access_token, token_type})
# @RELATION CALLS ->[create_access_token]
# @RELATION DEPENDS_ON ->[User]
# @RELATION DEPENDS_ON ->[Role]
# @PARAM user (User) - The authenticated user.
# @RETURN Dict[str, str] - Session data.
> def create_session(self, user: User) -> dict[str, str]:
> with belief_scope("auth.create_session"):
> roles = [role.name for role in user.roles]
> access_token = create_access_token(
> data={"sub": user.username, "scopes": roles}
> )
> return {"access_token": access_token, "token_type": "bearer"}
# endregion AuthService.create_session
# region AuthService.provision_adfs_user [TYPE Function]
# @PURPOSE: Performs ADFS Just-In-Time provisioning and role synchronization from AD group mappings.
# @PRE user_info contains identity claims where at least one of 'upn' or 'email' is present; 'groups' may be absent.
# @POST Returns persisted user entity with roles synchronized to mapped AD groups and refreshed state.
# @SIDE_EFFECT May insert new User, mutate user.roles, commit transaction, and refresh ORM state.
# @DATA_CONTRACT Input(Dict[str, Any]{upn|email, email, groups[]}) -> Output(User persisted)
# @RELATION DEPENDS_ON ->[AuthRepository]
# @RELATION DEPENDS_ON ->[User]
# @RELATION DEPENDS_ON ->[Role]
# @PARAM user_info (Dict[str, Any]) - Claims from ADFS token.
# @RETURN User - The provisioned user.
> def provision_adfs_user(self, user_info: dict[str, Any]) -> User:
> with belief_scope("auth.provision_adfs_user"):
> username = user_info.get("upn") or user_info.get("email")
> email = user_info.get("email")
> groups = user_info.get("groups", [])
> user = self.repo.get_user_by_username(username)
> if not user:
> user = User(
> username=username,
> email=email,
> full_name=user_info.get("name"),
> auth_source="ADFS",
> is_active=True,
> is_ad_user=True,
> )
> self.db.add(user)
> log_security_event("USER_PROVISIONED", username, {"source": "ADFS"})
# Sync roles from AD groups
> mapped_roles = self.repo.get_roles_by_ad_groups(groups)
> user.roles = mapped_roles
> user.last_login = datetime.now(UTC)
> self.db.commit()
> self.db.refresh(user)
> return user
# endregion AuthService.provision_adfs_user
# #endregion AuthService
# #endregion auth_service

10
backend/src/services/git_service.py,cover Normal file
View File

@@ -0,0 +1,10 @@
# #region git_service [C:1] [TYPE Module:Tombstone] [SEMANTICS git, service, shim, re-export, decomissioned]
# @BRIEF Re-export shim — GitService has been decomposed into services/git/ package.
# All consumers continue to import from this same path without changes.
# @RELATION CALLS -> [GitServiceModule]
# @RATIONALE Monolithic GitService (2101 lines) was decomposed into 8 domain-specific mixins
# under services/git/ to satisfy INV_7 (< 400 lines per module). This shim preserves
# the original import path for all 5 consumers.
# @REJECTED Breaking 5 consumer imports to remove this shim — unacceptable migration cost.
> from src.services.git import GitService # noqa: F401
# #endregion git_service

412
backend/src/services/health_service.py,cover Normal file
View File

@@ -0,0 +1,412 @@
# #region health_service [C:3] [TYPE Module] [SEMANTICS sqlalchemy, health, dashboard, validation, aggregate]
# @defgroup Services Module group.
# @BRIEF Business logic for aggregating dashboard health status from validation records.
# @LAYER Service
# @RELATION DEPENDS_ON -> [ValidationRecord]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [TaskCleanupService]
# @RELATION DEPENDS_ON -> [TaskManager]
> from datetime import UTC
> import json
> import os
> import time
> from typing import Any, cast
> from sqlalchemy import func
> from sqlalchemy.orm import Session
> from ..core.logger import logger
> from ..core.task_manager import TaskManager
> from ..core.utils.client_registry import get_superset_client
> from ..core.task_manager.cleanup import TaskCleanupService
> from ..models.llm import ValidationRecord
> from ..schemas.health import DashboardHealthItem, HealthSummaryResponse
> def _empty_dashboard_meta() -> dict[str, str | None]:
> return cast(dict[str, str | None], {"slug": None, "title": None})
# #region HealthService [C:4] [TYPE Class]
# @defgroup Services Module group.
# @BRIEF Aggregate latest dashboard validation state and manage persisted health report lifecycle.
# @PRE Service is constructed with a live SQLAlchemy session and optional config manager.
# @POST Exposes health summary aggregation and validation report deletion operations.
# @SIDE_EFFECT Maintains in-memory dashboard metadata caches and may coordinate cleanup through collaborators.
# @DATA_CONTRACT Input[Session, Optional[Any]] -> Output[HealthSummaryResponse|bool]
# @RELATION DEPENDS_ON -> [ValidationRecord]
# @RELATION DEPENDS_ON -> [DashboardHealthItem]
# @RELATION DEPENDS_ON -> [HealthSummaryResponse]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [TaskCleanupService]
# @RELATION DEPENDS_ON -> [TaskManager]
> class HealthService:
> _dashboard_summary_cache: dict[
> str, tuple[float, dict[str, dict[str, str | None]]]
> ] = {}
> _dashboard_summary_cache_ttl_seconds = 60.0
> """
> @PURPOSE: Service for managing and querying dashboard health data.
> """
# region HealthService_init [TYPE Function]
# @PURPOSE: Initialize health service with DB session and optional config access for dashboard metadata resolution.
# @PRE db is a valid SQLAlchemy session.
# @POST Service is ready to aggregate summaries and delete health reports.
# @SIDE_EFFECT Initializes per-instance dashboard metadata cache.
# @DATA_CONTRACT Input[db: Session, config_manager: Optional[Any]] -> Output[HealthService]
# @RELATION BINDS_TO ->[HealthService]
> def __init__(self, db: Session, config_manager=None):
> self.db = db
> self.config_manager = config_manager
> self._dashboard_meta_cache: dict[tuple[str, str], dict[str, str | None]] = {}
# endregion HealthService_init
# region _prime_dashboard_meta_cache [TYPE Function]
# @PURPOSE: Warm dashboard slug/title cache with one Superset list fetch per environment.
# @PRE records may contain mixed numeric and slug dashboard identifiers.
# @POST Numeric dashboard ids for known environments are cached when discoverable.
# @SIDE_EFFECT May call Superset dashboard list API once per referenced environment.
# @DATA_CONTRACT Input[records: List[ValidationRecord]] -> Output[None]
# @RELATION DEPENDS_ON ->[ValidationRecord]
# @RELATION DEPENDS_ON ->[ConfigManager]
# @RELATION DEPENDS_ON ->[SupersetClient]
> async def _prime_dashboard_meta_cache(self, records: list[ValidationRecord]) -> None:
> if not self.config_manager or not records:
! return
> numeric_ids_by_env: dict[str, set[str]] = {}
> for record in records:
> environment_id = str(record.environment_id or "").strip()
> dashboard_id = str(record.dashboard_id or "").strip()
> if not environment_id or not dashboard_id or not dashboard_id.isdigit():
! continue
> cache_key = (environment_id, dashboard_id)
> if cache_key in self._dashboard_meta_cache:
! continue
> numeric_ids_by_env.setdefault(environment_id, set()).add(dashboard_id)
> if not numeric_ids_by_env:
! return
> environments = {
> str(getattr(env, "id", "")).strip(): env
> for env in self.config_manager.get_environments()
> if str(getattr(env, "id", "")).strip()
> }
> for environment_id, dashboard_ids in numeric_ids_by_env.items():
> env = environments.get(environment_id)
> if not env:
! for dashboard_id in dashboard_ids:
! self._dashboard_meta_cache[(environment_id, dashboard_id)] = (
! _empty_dashboard_meta()
! )
! continue
> try:
> cached_meta = self.__class__._dashboard_summary_cache.get(
> environment_id
> )
> dashboard_meta_map: dict[str, dict[str, str | None]]
> if (
> cached_meta is not None
> and (time.monotonic() - cached_meta[0])
> < self.__class__._dashboard_summary_cache_ttl_seconds
> ):
! cached_meta_data = cast(
! tuple[float, dict[str, dict[str, str | None]]],
! cached_meta,
! )
! dashboard_meta_map = cached_meta_data[1]
> else:
> client = await get_superset_client(env)
> dashboards = await client.get_dashboards_summary()
> dashboard_meta_map = {
> str(item.get("id")): {
> "slug": item.get("slug"),
> "title": item.get("title"),
> }
> for item in dashboards
> if str(item.get("id") or "").strip()
> }
> self.__class__._dashboard_summary_cache[environment_id] = (
> time.monotonic(),
> dashboard_meta_map,
> )
> for dashboard_id in dashboard_ids:
> self._dashboard_meta_cache[(environment_id, dashboard_id)] = (
> dashboard_meta_map.get(
> dashboard_id,
> _empty_dashboard_meta(),
> )
> )
! except Exception as exc:
! logger.warning(
! "[HealthService][EXT:method:_prime_dashboard_meta_cache] Failed to preload dashboard metadata for env=%s: %s",
! environment_id,
! exc,
! )
! for dashboard_id in dashboard_ids:
! self._dashboard_meta_cache[(environment_id, dashboard_id)] = (
! _empty_dashboard_meta()
! )
# endregion _prime_dashboard_meta_cache
# region _resolve_dashboard_meta [TYPE Function]
# @PURPOSE: Resolve slug/title for a dashboard referenced by persisted validation record.
# @PRE dashboard_id may be numeric or slug-like; environment_id may be empty.
# @POST Returns dict with `slug` and `title` keys, using cache when possible.
# @SIDE_EFFECT Writes default cache entries for unresolved numeric dashboard ids.
> def _resolve_dashboard_meta(
> self, dashboard_id: str, environment_id: str | None
> ) -> dict[str, str | None]:
> normalized_dashboard_id = str(dashboard_id or "").strip()
> normalized_environment_id = str(environment_id or "").strip()
> if not normalized_dashboard_id:
! return _empty_dashboard_meta()
> if not normalized_dashboard_id.isdigit():
! return {"slug": normalized_dashboard_id, "title": None}
> if not self.config_manager or not normalized_environment_id:
! return _empty_dashboard_meta()
> cache_key = (normalized_environment_id, normalized_dashboard_id)
> cached = self._dashboard_meta_cache.get(cache_key)
> if cached is not None:
> return cached
! meta = _empty_dashboard_meta()
! self._dashboard_meta_cache[cache_key] = meta
! return meta
# endregion _resolve_dashboard_meta
# region get_health_summary [TYPE Function]
# @PURPOSE: Aggregate latest validation status per dashboard and enrich rows with dashboard slug/title.
# @PRE environment_id may be omitted to aggregate across all environments.
# @POST Returns HealthSummaryResponse with counts and latest record row per dashboard.
# @SIDE_EFFECT May call Superset API to resolve dashboard metadata.
# @DATA_CONTRACT Input[environment_id: Optional[str]] -> Output[HealthSummaryResponse]
# @RELATION CALLS ->[EXT:method:_prime_dashboard_meta_cache]
# @RELATION CALLS ->[EXT:method:_resolve_dashboard_meta]
> async def get_health_summary(
> self, environment_id: str = ""
> ) -> HealthSummaryResponse:
> """
> @PURPOSE: Aggregates the latest validation status for all dashboards.
> @PRE: environment_id (optional) to filter by environment.
> @POST: Returns a HealthSummaryResponse with aggregated status counts and items.
> """
# [REASON] We need the latest ValidationRecord for each unique dashboard_id.
# We use a subquery to find the max timestamp per dashboard_id.
> subquery = self.db.query(
> ValidationRecord.dashboard_id,
> func.max(ValidationRecord.timestamp).label("max_ts"),
> )
> if environment_id:
> subquery = subquery.filter(
> ValidationRecord.environment_id == environment_id
> )
> subquery = subquery.group_by(ValidationRecord.dashboard_id).subquery()
> query = self.db.query(ValidationRecord).join(
> subquery,
> (ValidationRecord.dashboard_id == subquery.c.dashboard_id)
> & (ValidationRecord.timestamp == subquery.c.max_ts),
> )
> records = query.all()
> await self._prime_dashboard_meta_cache(records)
> items = []
> pass_count = 0
> warn_count = 0
> fail_count = 0
> unknown_count = 0
> for rec in records:
> record = cast(Any, rec)
> status = str(record.status or "").upper()
> if status == "PASS":
> pass_count += 1
! elif status == "WARN":
! warn_count += 1
! elif status == "FAIL":
! fail_count += 1
! else:
! unknown_count += 1
! status = "UNKNOWN"
> record_id = str(record.id or "")
> dashboard_id = str(record.dashboard_id or "")
> resolved_environment_id = (
> str(record.environment_id)
> if record.environment_id is not None
> else None
> )
> response_environment_id = (
> resolved_environment_id
> if resolved_environment_id is not None
> else "unknown"
> )
> task_id = str(record.task_id) if record.task_id is not None else None
> summary = str(record.summary) if record.summary is not None else None
> timestamp = cast(Any, record.timestamp)
# Ensure timestamp is timezone-aware (DB stores naive UTC)
> if timestamp is not None and timestamp.tzinfo is None:
! timestamp = timestamp.replace(tzinfo=UTC)
# ── v2 fields ──────────────────────────────────────────────
> execution_path = str(record.execution_path) if record.execution_path else None
> issues_count = len(record.issues) if record.issues else 0
> timings = dict(record.timings) if record.timings else None
> token_usage = dict(record.token_usage) if record.token_usage else None
> screenshot_paths = list(record.screenshot_paths) if record.screenshot_paths else None
# Extract chunk_count from raw_response (stored as JSON in result_payload)
> chunk_count: int | None = None
> if record.raw_response:
! try:
! raw = json.loads(record.raw_response)
! chunk_count = int(raw["chunk_count"]) if raw.get("chunk_count") else None
! except (json.JSONDecodeError, KeyError, TypeError, ValueError):
! pass
> meta = self._resolve_dashboard_meta(dashboard_id, resolved_environment_id)
> dashboard_name = meta.get("title") or meta.get("slug") or dashboard_id
> items.append(
> DashboardHealthItem(
> record_id=record_id,
> dashboard_id=dashboard_id,
> dashboard_slug=meta.get("slug"),
> dashboard_title=meta.get("title"),
> environment_id=response_environment_id,
> status=status,
> last_check=timestamp,
> task_id=task_id,
> run_id=str(record.run_id) if record.run_id else None,
> policy_id=str(record.policy_id) if record.policy_id else None,
> summary=summary,
> execution_path=execution_path,
> issues_count=issues_count,
> timings=timings,
> token_usage=token_usage,
> screenshot_paths=screenshot_paths,
> chunk_count=chunk_count,
> dashboard_name=dashboard_name,
> )
> )
> logger.info(
> f"[HealthService][get_health_summary] Aggregated {len(items)} dashboard health records."
> )
> return HealthSummaryResponse(
> items=items,
> pass_count=pass_count,
> warn_count=warn_count,
> fail_count=fail_count,
> unknown_count=unknown_count,
> )
# endregion get_health_summary
# region delete_validation_report [TYPE Function]
# @PURPOSE: Delete one persisted health report and optionally clean linked task/log artifacts.
# @PRE record_id is a validation record identifier.
# @POST Returns True only when a matching record was deleted.
# @SIDE_EFFECT Deletes DB rows, optional screenshot file, and optional task/log persistence.
# @DATA_CONTRACT Input[record_id: str, task_manager: Optional[TaskManager]] -> Output[bool]
# @RELATION DEPENDS_ON ->[ValidationRecord]
# @RELATION DEPENDS_ON ->[TaskManager]
# @RELATION DEPENDS_ON ->[TaskCleanupService]
> def delete_validation_report(
> self, record_id: str, task_manager: TaskManager | None = None
> ) -> bool:
! record = (
! self.db.query(ValidationRecord)
! .filter(ValidationRecord.id == record_id)
! .first()
! )
! if not record:
! return False
! peer_query = self.db.query(ValidationRecord).filter(
! ValidationRecord.dashboard_id == record.dashboard_id
! )
! if record.environment_id is None:
! peer_query = peer_query.filter(ValidationRecord.environment_id.is_(None))
! else:
! peer_query = peer_query.filter(
! ValidationRecord.environment_id == record.environment_id
! )
! records_to_delete = peer_query.all()
! screenshot_paths = [
! str(item.screenshot_path or "").strip()
! for item in records_to_delete
! if str(item.screenshot_path or "").strip()
! ]
! task_ids = {
! str(item.task_id or "").strip()
! for item in records_to_delete
! if str(item.task_id or "").strip()
! }
! logger.info(
! "[HealthService][delete_validation_report] Removing %s validation record(s) for dashboard=%s environment=%s triggered_by_record=%s",
! len(records_to_delete),
! record.dashboard_id,
! record.environment_id,
! record_id,
! )
! for item in records_to_delete:
! self.db.delete(item)
! self.db.commit()
! for screenshot_path in screenshot_paths:
! try:
! if os.path.exists(screenshot_path):
! os.remove(screenshot_path)
! except OSError as exc:
! logger.warning(
! "[HealthService][delete_validation_report] Failed to remove screenshot %s: %s",
! screenshot_path,
! exc,
! )
! if task_ids and task_manager and self.config_manager:
! try:
! cleanup_service = TaskCleanupService(
! task_manager.persistence_service,
! task_manager.log_persistence_service,
! self.config_manager,
! )
! for task_id in task_ids:
! task_manager.tasks.pop(task_id, None)
! cleanup_service.delete_task_with_logs(task_id)
! except Exception as exc:
! logger.warning(
! "[HealthService][delete_validation_report] Failed to cleanup linked task/logs for dashboard=%s environment=%s: %s",
! record.dashboard_id,
! record.environment_id,
! exc,
! )
! return True
# endregion delete_validation_report
# #endregion HealthService
# #endregion health_service

267
backend/src/services/llm_prompt_templates.py,cover Normal file
View File

@@ -0,0 +1,267 @@
# #region llm_prompt_templates [C:2] [TYPE Module] [SEMANTICS llm, prompt, template, normalization]
# @defgroup Services Module group.
# @BRIEF Provide default LLM prompt templates and normalization helpers for runtime usage.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [ConfigManager]
# @INVARIANT All required prompt template keys are always present after normalization.
> from __future__ import annotations
> from copy import deepcopy
> from typing import Any
> import warnings
> from ..core.logger import logger
# #region DEFAULT_LLM_PROMPTS [C:2] [TYPE Constant]
# @ingroup Services
# @BRIEF Default prompt templates used by documentation, dashboard validation, and git commit generation.
> DEFAULT_LLM_PROMPTS: dict[str, str] = {
> "dashboard_validation_prompt": ( # Legacy v1 — deprecated for new tasks, kept for backward compat
> "Analyze the attached dashboard screenshot and the following execution logs for health and visual issues.\n\n"
> "Logs:\n"
> "{logs}\n\n"
> "Provide the analysis in JSON format with the following structure:\n"
> "{\n"
> ' "status": "PASS" | "WARN" | "FAIL",\n'
> ' "summary": "Short summary of findings",\n'
> ' "issues": [\n'
> " {\n"
> ' "severity": "WARN" | "FAIL",\n'
> ' "message": "Description of the issue",\n'
> ' "location": "Optional location info (e.g. chart name)"\n'
> " }\n"
> " ]\n"
> "}"
> ),
> "dashboard_validation_prompt_multimodal": ( # Path A — image-focused (v2)
> "Analyze the attached {total_chunks} dashboard tab screenshots and execution logs for visual and data health issues.\n\n"
> "Screenshots are attached IN ORDER — screenshot 0 is the first tab, screenshot 1 is the second, etc.\n"
> "Tab order:\n"
> "{tab_list}\n\n"
> "Logs:\n"
> "{logs}\n\n"
> "Provide the analysis in JSON format. Each issue MUST specify the tab_index field "
> "(0-based index of the screenshot/tab where the issue appears, or -1 if not tab-specific):\n"
> "{\n"
> ' "status": "PASS" | "WARN" | "FAIL",\n'
> ' "summary": "Short summary of findings",\n'
> ' "issues": [\n'
> " {\n"
> ' "severity": "WARN" | "FAIL",\n'
> ' "message": "Description of the issue",\n'
> ' "location": "Where the issue is located (tab name, chart name, screen region)",\n'
> ' "tab_index": "0-based tab index (-1 if unknown)"\n'
> " }\n"
> " ]\n"
> "}"
> ),
> "dashboard_validation_prompt_text": ( # Path B — topology-focused (v2)
> "Analyze the following dashboard topology, dataset health, and execution logs for "
> "data consistency and KXD connectivity issues.\n\n"
> "Dashboard structure (tabs are indicated in the topology below):\n"
> "{topology}\n\n"
> "Dataset health:\n"
> "{dataset_health}\n\n"
> "Logs:\n"
> "{logs}\n\n"
> "Provide the analysis in JSON format. Each issue MUST specify the tab_index field "
> "(0-based index of the tab where the issue appears, or -1 if not tab-specific):\n"
> "{\n"
> ' "status": "PASS" | "WARN" | "FAIL",\n'
> ' "summary": "Short summary of findings",\n'
> ' "issues": [\n'
> " {\n"
> ' "severity": "WARN" | "FAIL",\n'
> ' "message": "Description of the issue",\n'
> ' "location": "Where the issue is located (tab name, chart name, dataset name)",\n'
> ' "tab_index": "0-based tab index (-1 if unknown)"\n'
> " }\n"
> " ]\n"
> "}"
> ),
> "documentation_prompt": (
> "Generate professional documentation for the following dataset and its columns.\n"
> "Dataset: {dataset_name}\n"
> "Columns: {columns_json}\n\n"
> "Provide the documentation in JSON format:\n"
> "{\n"
> ' "dataset_description": "General description of the dataset",\n'
> ' "column_descriptions": [\n'
> " {\n"
> ' "name": "column_name",\n'
> ' "description": "Generated description"\n'
> " }\n"
> " ]\n"
> "}"
> ),
> "git_commit_prompt": (
> "Generate a concise and professional git commit message based on the following diff and recent history.\n"
> "Use Conventional Commits format (e.g., feat: ..., fix: ..., docs: ...).\n\n"
> "Recent History:\n"
> "{history}\n\n"
> "Diff:\n"
> "{diff}\n\n"
> "Commit Message:"
> ),
> }
# #endregion DEFAULT_LLM_PROMPTS
# #region DEFAULT_LLM_PROVIDER_BINDINGS [C:2] [TYPE Constant]
# @ingroup Services
# @BRIEF Default provider binding per task domain.
> DEFAULT_LLM_PROVIDER_BINDINGS: dict[str, str] = {
> "documentation": "",
> "git_commit": "",
> }
# #endregion DEFAULT_LLM_PROVIDER_BINDINGS
# #region DEFAULT_LLM_ASSISTANT_SETTINGS [C:2] [TYPE Constant]
# @ingroup Services
# @BRIEF Default planner settings for assistant chat intent model/provider resolution.
> DEFAULT_LLM_ASSISTANT_SETTINGS: dict[str, str] = {
> "assistant_planner_provider": "",
> "assistant_planner_model": "",
> }
# #endregion DEFAULT_LLM_ASSISTANT_SETTINGS
# #region normalize_llm_settings [C:3] [TYPE Function]
# @ingroup Services
# @BRIEF Ensure llm settings contain stable schema with prompts section and default templates.
# @PRE llm_settings is dictionary-like value or None.
# @POST Returned dict contains prompts with all required template keys.
# @RELATION DEPENDS_ON -> LLMProviderService
> def normalize_llm_settings(llm_settings: Any) -> dict[str, Any]:
> normalized: dict[str, Any] = {
> "providers": [],
> "default_provider": "",
> "prompts": {},
> "provider_bindings": {},
> **DEFAULT_LLM_ASSISTANT_SETTINGS,
> }
> if isinstance(llm_settings, dict):
> normalized.update(
> {
> k: v
> for k, v in llm_settings.items()
> if k
> in (
> "providers",
> "default_provider",
> "prompts",
> "provider_bindings",
> "assistant_planner_provider",
> "assistant_planner_model",
> )
> }
> )
> prompts = normalized.get("prompts") if isinstance(normalized.get("prompts"), dict) else {}
> merged_prompts = deepcopy(DEFAULT_LLM_PROMPTS)
> merged_prompts.update({k: v for k, v in prompts.items() if isinstance(v, str) and v.strip()})
> normalized["prompts"] = merged_prompts
> bindings = normalized.get("provider_bindings") if isinstance(normalized.get("provider_bindings"), dict) else {}
> merged_bindings = deepcopy(DEFAULT_LLM_PROVIDER_BINDINGS)
> merged_bindings.update({k: v for k, v in bindings.items() if isinstance(v, str)})
> normalized["provider_bindings"] = merged_bindings
> for key, default_value in DEFAULT_LLM_ASSISTANT_SETTINGS.items():
> value = normalized.get(key, default_value)
> normalized[key] = value.strip() if isinstance(value, str) else default_value
> return normalized
# #endregion normalize_llm_settings
# #region is_multimodal_model [C:3] [TYPE Function]
# @ingroup Services
# @BRIEF Heuristically determine whether model supports image input required for dashboard validation.
# @DEPRECATED Use the explicit `db_provider.is_multimodal` flag instead (see migration 9f8e7d6c5b4a).
# @RATIONALE Added import warnings + warnings.warn(DeprecationWarning) to is_multimodal_model as a deprecation shim.
# Replaced by an explicit boolean flag on `LLMProvider` that users control via checkbox in UI.
# This function is retained only as a backward-compatibility shim for the Alembic migration
# backfill and must not be imported in new production code.
# @REJECTED Keeping the function as a backward-compatibility shim; do not use for new validation.
# @PRE model_name may be empty or mixed-case.
# @POST Returns True when model likely supports multimodal input.
# @RELATION DEPENDS_ON -> LLMProviderService
> def is_multimodal_model(model_name: str, provider_type: str | None = None) -> bool:
> warnings.warn(
> "is_multimodal_model is deprecated; use LLMProvider.is_multimodal instead",
> DeprecationWarning,
> stacklevel=2,
> )
> token = (model_name or "").strip().lower()
> if not token:
> return False
> text_only_markers = (
> "text-only",
> "embedding",
> "rerank",
> "whisper",
> "tts",
> "transcribe",
> )
> if any(marker in token for marker in text_only_markers):
> return False
> multimodal_markers = (
> "gpt-4o",
> "gpt-4.1",
> "vision",
> "vl",
> "gemini",
> "claude-3",
> "claude-sonnet-4",
> "omni",
> "multimodal",
> "pixtral",
> "llava",
> "internvl",
> "qwen-vl",
> "qwen2-vl",
> )
> return any(marker in token for marker in multimodal_markers)
# #endregion is_multimodal_model
# #region resolve_bound_provider_id [C:3] [TYPE Function]
# @ingroup Services
# @BRIEF Resolve provider id configured for a task binding with fallback to default provider.
# @PRE llm_settings is normalized or raw dict from config.
# @POST Returns configured provider id or fallback id/empty string when not defined.
# @RELATION DEPENDS_ON -> LLMProviderService
> def resolve_bound_provider_id(llm_settings: Any, task_key: str) -> str:
> normalized = normalize_llm_settings(llm_settings)
> bindings = normalized.get("provider_bindings", {})
> bound = bindings.get(task_key)
> if isinstance(bound, str) and bound.strip():
> return bound.strip()
> default_provider = normalized.get("default_provider", "")
> return default_provider.strip() if isinstance(default_provider, str) else ""
# #endregion resolve_bound_provider_id
# #region render_prompt [C:3] [TYPE Function]
# @ingroup Services
# @BRIEF Render prompt template using deterministic placeholder replacement with graceful fallback.
# @PRE template is a string and variables values are already stringifiable.
# @POST Returns rendered prompt text with known placeholders substituted. Warns about unfilled placeholders.
# @RELATION DEPENDS_ON -> LLMProviderService
> def render_prompt(template: str, variables: dict[str, Any]) -> str:
> rendered = template
> for key, value in variables.items():
> rendered = rendered.replace("{" + key + "}", str(value))
# Warn about unfilled placeholders that would be sent to LLM
> import re
> unfilled = re.findall(r'\{(\w+)\}', rendered)
> if unfilled:
> logger.warning(
> f"[render_prompt] Unfilled placeholders in rendered prompt: {unfilled}"
> )
> return rendered
# #endregion render_prompt
# #endregion llm_prompt_templates

267
backend/src/services/llm_provider.py,cover Normal file
View File

@@ -0,0 +1,267 @@
# #region llm_provider [C:3] [TYPE Module] [SEMANTICS sqlalchemy, llm, provider, encryption, config]
# @defgroup Services Module group.
# @BRIEF Service for managing LLM provider configurations with encrypted API keys.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [LLMProvider]
# @RELATION DEPENDS_ON -> [EncryptionManager]
# @RELATION DEPENDS_ON -> [LLMProviderConfig]
# @RATIONALE EncryptionManager imported from core/encryption.py (extracted from this module)
# to decouple core encryption from LLM domain — see [SEC:C-1].
> from typing import TYPE_CHECKING
> from cryptography.exceptions import InvalidTag
> from sqlalchemy.orm import Session
> from ..core.encryption import EncryptionManager
> from ..core.logger import belief_scope, logger
> from ..models.llm import LLMProvider
- if TYPE_CHECKING:
- from ..plugins.llm_analysis.models import LLMProviderConfig
> MASKED_API_KEY_PLACEHOLDER = "********"
# #region mask_api_key [C:2] [TYPE Function]
# @ingroup Services
# @BRIEF Mask an API key for safe display, showing first 4 and last 4 characters.
# @PRE api_key is a plaintext string or None.
# @POST Returns "****" for very short keys; "{first 2}...{last 2}" for <=8 chars;
# "{first 4}...{last 4}" for longer keys; "" for None/empty.
> def mask_api_key(api_key: str | None) -> str:
> if not api_key:
> return ""
> if len(api_key) <= 4:
> return "****"
> if len(api_key) <= 8:
> return f"{api_key[:2]}...{api_key[-2:]}"
> return f"{api_key[:4]}...{api_key[-4:]}"
# #endregion mask_api_key
# #region is_masked_or_placeholder [C:2] [TYPE Function]
# @ingroup Services
# @BRIEF Predicate: True when api_key is None, empty, "********", or contains "...".
# @PRE api_key can be None.
# @POST Returns True only for non-real-key values.
> def is_masked_or_placeholder(api_key: str | None) -> bool:
> if not api_key:
> return True
> return api_key == MASKED_API_KEY_PLACEHOLDER or "..." in api_key
# #endregion is_masked_or_placeholder
# NOTE: EncryptionManager is now imported from ..core.encryption
# #region LLMProviderService [C:3] [TYPE Class]
# @defgroup Services Module group.
# @BRIEF Service to manage LLM provider lifecycle.
# @RELATION DEPENDS_ON -> [LLMProvider]
# @RELATION DEPENDS_ON -> [EncryptionManager]
# @RELATION DEPENDS_ON -> [LLMProviderConfig]
> class LLMProviderService:
# region LLMProviderService_init [TYPE Function]
# @PURPOSE: Initialize the service with database session.
# @PRE db must be a valid SQLAlchemy Session.
# @POST Service ready for provider operations.
# @RELATION DEPENDS_ON ->[EncryptionManager]
> def __init__(self, db: Session):
> self.db = db
> self.encryption = EncryptionManager()
# endregion LLMProviderService_init
# region get_all_providers [TYPE Function]
# @PURPOSE: Returns all configured LLM providers.
# @PRE Database connection must be active.
# @POST Returns list of all LLMProvider records.
# @RELATION DEPENDS_ON ->[LLMProvider]
> def get_all_providers(self) -> list[LLMProvider]:
> with belief_scope("get_all_providers"):
> return self.db.query(LLMProvider).all()
# endregion get_all_providers
# region get_provider [TYPE Function]
# @PURPOSE: Returns a single LLM provider by ID.
# @PRE provider_id must be a valid string.
# @POST Returns LLMProvider or None if not found.
# @RELATION DEPENDS_ON ->[LLMProvider]
> def get_provider(self, provider_id: str) -> LLMProvider | None:
> with belief_scope("get_provider"):
> return (
> self.db.query(LLMProvider).filter(LLMProvider.id == provider_id).first()
> )
# endregion get_provider
# region create_provider [TYPE Function]
# @PURPOSE: Creates a new LLM provider with encrypted API key.
# @PRE config must contain valid provider configuration.
# @POST New provider created and persisted to database.
# @RELATION DEPENDS_ON ->[LLMProviderConfig]
# @RELATION DEPENDS_ON ->[LLMProvider]
# @RELATION CALLS ->[EXT:method:encrypt]
> def create_provider(self, config: "LLMProviderConfig") -> LLMProvider:
> with belief_scope("create_provider"):
> encrypted_key = self.encryption.encrypt(config.api_key)
> db_provider = LLMProvider(
> provider_type=config.provider_type.value,
> name=config.name,
> base_url=config.base_url,
> api_key=encrypted_key,
> default_model=config.default_model,
> is_active=config.is_active,
> is_multimodal=config.is_multimodal,
> max_images=config.max_images,
> context_window=config.context_window,
> max_output_tokens=config.max_output_tokens,
> )
> self.db.add(db_provider)
> self.db.commit()
> self.db.refresh(db_provider)
> return db_provider
# endregion create_provider
# region update_provider [TYPE Function]
# @PURPOSE: Updates an existing LLM provider.
# @PRE provider_id must exist, config must be valid.
# @POST Provider updated and persisted to database.
# @RELATION DEPENDS_ON ->[LLMProviderConfig]
# @RELATION DEPENDS_ON ->[LLMProvider]
# @RELATION CALLS ->[EXT:method:encrypt]
> def update_provider(
> self, provider_id: str, config: "LLMProviderConfig"
> ) -> LLMProvider | None:
> with belief_scope("update_provider"):
> db_provider = self.get_provider(provider_id)
> if not db_provider:
! return None
> db_provider.provider_type = config.provider_type.value
> db_provider.name = config.name
> db_provider.base_url = config.base_url
# Ignore masked/placeholder values; they are display-only and must not overwrite secrets.
> if not is_masked_or_placeholder(config.api_key):
> db_provider.api_key = self.encryption.encrypt(config.api_key)
> db_provider.default_model = config.default_model
> db_provider.is_active = config.is_active
> db_provider.is_multimodal = config.is_multimodal
> db_provider.max_images = config.max_images
> db_provider.context_window = config.context_window
> db_provider.max_output_tokens = config.max_output_tokens
> self.db.commit()
> self.db.refresh(db_provider)
> return db_provider
# endregion update_provider
# region set_max_images [TYPE Function]
# @PURPOSE: Updates only the max_images field for a provider.
# @PRE provider_id must exist.
# @POST Returns updated provider or None if not found.
# @RELATION DEPENDS_ON ->[LLMProvider]
> def set_max_images(self, provider_id: str, max_images: int | None) -> LLMProvider | None:
> with belief_scope("set_max_images"):
> db_provider = self.get_provider(provider_id)
> if not db_provider:
> return None
> db_provider.max_images = max_images
> self.db.commit()
> self.db.refresh(db_provider)
> return db_provider
# endregion set_max_images
# region delete_provider [TYPE Function]
# @PURPOSE: Deletes an LLM provider.
# @PRE provider_id must exist.
# @POST Provider removed from database.
# @RELATION DEPENDS_ON ->[LLMProvider]
> def delete_provider(self, provider_id: str) -> bool:
> with belief_scope("delete_provider"):
> db_provider = self.get_provider(provider_id)
> if not db_provider:
> return False
> self.db.delete(db_provider)
> self.db.commit()
> return True
# endregion delete_provider
# region get_decrypted_api_key [TYPE Function]
# @PURPOSE: Returns the decrypted API key for a provider.
# @PRE provider_id must exist with valid encrypted key.
# @POST Returns decrypted API key or None on failure.
# @RELATION DEPENDS_ON ->[LLMProvider]
# @RELATION CALLS ->[EXT:method:decrypt]
> def get_decrypted_api_key(self, provider_id: str) -> str | None:
> with belief_scope("get_decrypted_api_key"):
> db_provider = self.get_provider(provider_id)
> if not db_provider:
> logger.warning(
> f"[get_decrypted_api_key] Provider {provider_id} not found in database"
> )
> return None
> logger.info(f"[get_decrypted_api_key] Provider found: {db_provider.id}")
> logger.info(
> f"[get_decrypted_api_key] Encrypted API key length: {len(db_provider.api_key) if db_provider.api_key else 0}"
> )
> try:
> decrypted_key = self.encryption.decrypt(db_provider.api_key)
> logger.info(
> f"[get_decrypted_api_key] Decryption successful, key length: {len(decrypted_key) if decrypted_key else 0}"
> )
> return decrypted_key
> except InvalidTag as e:
! logger.error(
! f"[get_decrypted_api_key] Integrity check failed (InvalidTag): {e!s}. "
! "The encrypted API key may be corrupted or the ENCRYPTION_KEY has changed."
! )
! return None
> except ValueError as e:
! logger.error(
! f"[get_decrypted_api_key] Decryption format error (ValueError): {e!s}. "
! "The encrypted data may not be valid Fernet ciphertext."
! )
! return None
> except Exception as e:
> logger.error(
> f"[get_decrypted_api_key] Decryption failed with unexpected error "
> f"({type(e).__name__}): {e!s}"
> )
> return None
# endregion get_decrypted_api_key
# region get_provider_token_config [TYPE Function]
# @PURPOSE: Returns provider token limits for batch sizing.
# @PRE provider_id must be valid.
# @POST Returns dict with model name, context_window, max_output_tokens.
# Values from DB take priority; None means "use PROVIDER_DEFAULTS fallback".
# @RATIONALE Centralised helper — both _batch_proc.py and _batch_sizer.py need
# the same resolution logic. Avoids duplicating DB queries and defaults.
> def get_provider_token_config(self, provider_id: str) -> dict:
! provider = self.get_provider(provider_id)
! if not provider:
! return {"model": None, "context_window": None, "max_output_tokens": None}
! return {
! "model": provider.default_model or "gpt-4o-mini",
! "context_window": provider.context_window,
! "max_output_tokens": provider.max_output_tokens,
! }
# endregion get_provider_token_config
# #endregion LLMProviderService
# #endregion llm_provider

93
backend/src/services/mapping_service.py,cover Normal file
View File

@@ -0,0 +1,93 @@
# #region mapping_service [C:5] [TYPE Module] [SEMANTICS mapping, database, superset, fuzzy, suggestion]
# @defgroup Services Module group.
#
# @BRIEF Orchestrates database fetching and fuzzy matching suggestions.
# @LAYER Service
# @PRE source/target environment identifiers are provided by caller.
# @POST Exposes stateless mapping suggestion orchestration over configured environments.
# @SIDE_EFFECT Performs remote metadata reads through Superset API clients.
# @DATA_CONTRACT Input[source_env_id: str, target_env_id: str] -> Output[List[Dict]]
# @RELATION DEPENDS_ON -> SupersetClient
# @RELATION DEPENDS_ON -> suggest_mappings
#
# @INVARIANT Suggestions are based on database names.
> from ..core.logger import belief_scope
> from ..core.superset_client import SupersetClient
> from ..core.utils.matching import suggest_mappings
# #region MappingService [C:3] [TYPE Class]
# @defgroup Services Module group.
# @BRIEF Service for handling database mapping logic.
# @PRE config_manager exposes get_environments() with environment objects containing id.
# @POST Provides client resolution and mapping suggestion methods.
# @SIDE_EFFECT Instantiates Superset clients and performs upstream metadata reads.
# @DATA_CONTRACT Input[config_manager] -> Output[List[Dict]]
# @RELATION DEPENDS_ON -> SupersetClient
# @RELATION DEPENDS_ON -> suggest_mappings
> class MappingService:
# region init [TYPE Function]
# @PURPOSE: Initializes the mapping service with a config manager.
# @PRE config_manager is provided.
# @PARAM config_manager (ConfigManager) - The configuration manager.
# @POST Service is initialized.
# @RELATION DEPENDS_ON -> MappingService
> def __init__(self, config_manager):
> with belief_scope("MappingService.__init__"):
> self.config_manager = config_manager
# endregion init
# region _get_client [TYPE Function]
# @PURPOSE: Helper to get an initialized SupersetClient for an environment.
# @PARAM env_id (str) - The ID of the environment.
# @PRE environment must exist in config.
# @POST Returns an initialized SupersetClient.
# @RETURN SupersetClient - Initialized client.
# @RELATION CALLS -> SupersetClient
> def _get_client(self, env_id: str) -> SupersetClient:
> with belief_scope("MappingService._get_client", f"env_id={env_id}"):
> envs = self.config_manager.get_environments()
> env = next((e for e in envs if e.id == env_id), None)
> if not env:
> raise ValueError(f"Environment {env_id} not found")
> return SupersetClient(env)
# endregion _get_client
# region get_suggestions [TYPE Function]
# @PURPOSE: Fetches databases from both environments and returns fuzzy matching suggestions.
# @PARAM source_env_id (str) - Source environment ID.
# @PARAM target_env_id (str) - Target environment ID.
# @PRE Both environments must be accessible.
# @POST Returns fuzzy-matched database suggestions.
# @RETURN List[Dict] - Suggested mappings.
# @RELATION CALLS -> _get_client
# @RELATION CALLS -> suggest_mappings
> async def get_suggestions(
> self, source_env_id: str, target_env_id: str
> ) -> list[dict]:
> with belief_scope(
> "MappingService.get_suggestions",
> f"source={source_env_id}, target={target_env_id}",
> ):
> """
> Get suggested mappings between two environments.
> """
> source_client = self._get_client(source_env_id)
> target_client = self._get_client(target_env_id)
> source_dbs = source_client.get_databases_summary()
> target_dbs = target_client.get_databases_summary()
> return suggest_mappings(source_dbs, target_dbs)
# endregion get_suggestions
# #endregion MappingService
# #endregion mapping_service

173
backend/src/services/profile_service.py,cover Normal file
View File

@@ -0,0 +1,173 @@
# #region profile_service [C:5] [TYPE Module] [SEMANTICS sqlalchemy, profile, preference, superset, lookup]
# @defgroup Services Module group.
#
# @BRIEF Composite facade orchestrating profile preference persistence, Superset account lookup,
# security badges, and deterministic actor matching by delegating to focused sub-services.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [UserDashboardPreference]
# @RELATION DEPENDS_ON -> [ProfilePreferenceResponse]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [AuthRepositoryModule]
# @RELATION DEPENDS_ON -> [User]
# @RELATION DEPENDS_ON -> [EXT:Library:sqlalchemy.orm.Session]
# @RELATION DEPENDS_ON -> [ProfilePreferenceService]
# @RELATION DEPENDS_ON -> [SupersetLookupService]
# @RELATION DEPENDS_ON -> [SecurityBadgeService]
# @RELATION DEPENDS_ON -> [ProfileUtils]
#
# @INVARIANT Profile ID needs to be unique per-user session
#
# @TEST_CONTRACT ProfilePreferenceUpdateRequest -> ProfilePreferenceResponse
# @TEST_FIXTURE valid_profile_update -> {"user_id":"u-1","superset_username":"John_Doe","show_only_my_dashboards":true}
# @TEST_EDGE enable_without_username -> toggle=true with empty username returns validation error
# @TEST_EDGE cross_user_mutation -> attempt to update another user preference returns forbidden
# @TEST_EDGE lookup_env_not_found -> unknown environment_id returns not found
# @TEST_INVARIANT normalization_consistency -> VERIFIED_BY: [valid_profile_update, enable_without_username]
# @DATA_CONTRACT Profile_id -> ProfileInfo; session_id -> valid UUID
# @PRE Session is active and valid
# @POST Profile with updated fields populated and
# @SIDE_EFFECT Database read/write operations
# @RATIONALE Decomposed from monolithic 770-line ProfileService into three focused services
# (ProfilePreferenceService, SupersetLookupService, SecurityBadgeService) plus pure
# utility functions (ProfileUtils) to satisfy INV_7. This module is a thin facade
# that preserves the public API contract for all existing callers.
# @REJECTED Keeping all three domains in a single ProfileService class was rejected — it violated
# INV_7 (770 lines vs 150 max), mixed I/O patterns (DB writes + HTTP calls), and created
# unnecessary coupling between preference persistence and Superset network operations.
> from collections.abc import Iterable
> from typing import Any
> from sqlalchemy.orm import Session
> from ..models.auth import User
> from ..schemas.profile import (
> ProfilePreferenceResponse,
> ProfilePreferenceUpdateRequest,
> SupersetAccountLookupRequest,
> SupersetAccountLookupResponse,
> )
> from .profile_preference_service import ProfilePreferenceService
> from .profile_utils import (
> EnvironmentNotFoundError,
> ProfileAuthorizationError,
> ProfileValidationError,
> normalize_owner_tokens,
> normalize_username,
> )
> from .security_badge_service import SecurityBadgeService
> from .superset_lookup_service import SupersetLookupService
# Re-export exception classes for backward-compatible imports
> __all__ = [
> "EnvironmentNotFoundError",
> "ProfileAuthorizationError",
> "ProfilePreferenceService",
> "ProfileService",
> "ProfileValidationError",
> "SecurityBadgeService",
> "SupersetLookupService",
> ]
# #region ProfileService [C:4] [TYPE Class] [SEMANTICS profile,facade,delegate,coordinator]
# @defgroup Services Module group.
# @BRIEF Facade that composes ProfilePreferenceService, SupersetLookupService, and
# SecurityBadgeService into a single interface for backward compatibility.
# @RELATION DEPENDS_ON -> [ProfilePreferenceService]
# @RELATION DEPENDS_ON -> [SupersetLookupService]
# @RELATION DEPENDS_ON -> [SecurityBadgeService]
# @RELATION DEPENDS_ON -> [ProfileUtils]
# @PRE Caller provides authenticated User context for external service methods.
# @POST Delegates to sub-services and returns normalized profile/lookup responses.
# @SIDE_EFFECT Writes preference records and encrypted tokens; performs external account lookups when requested.
# @DATA_CONTRACT Input[User,ProfilePreferenceUpdateRequest|SupersetAccountLookupRequest] -> Output[ProfilePreferenceResponse|SupersetAccountLookupResponse|bool]
# @INVARIANT Profile data integrity maintained, cache consistency with database state
# @RATIONALE Thin coordinator — all sub-10-line delegating methods. Actual business logic lives
# in the injected sub-services.
> class ProfileService:
> """Facade composing preference, lookup, and security services."""
# region init [TYPE Function]
# @BRIEF Initialize facade and create sub-services.
# @PRE db session is active and config_manager supports get_environments().
# @POST All sub-services are initialized and ready.
> def __init__(self, db: Session, config_manager: Any, plugin_loader: Any = None):
> self.preference_service = ProfilePreferenceService(db, config_manager, plugin_loader)
> self.lookup_service = SupersetLookupService(config_manager)
> self.security_badge_service = SecurityBadgeService(plugin_loader)
> self.db = db
> self.config_manager = config_manager
> self.plugin_loader = plugin_loader
# endregion init
# region get_my_preference [TYPE Function]
# @BRIEF Delegate to ProfilePreferenceService.
> def get_my_preference(self, current_user: User) -> ProfilePreferenceResponse:
> return self.preference_service.get_my_preference(current_user)
# endregion get_my_preference
# region get_dashboard_filter_binding [TYPE Function]
# @BRIEF Delegate to ProfilePreferenceService.
> def get_dashboard_filter_binding(self, current_user: User) -> dict:
> return self.preference_service.get_dashboard_filter_binding(current_user)
# endregion get_dashboard_filter_binding
# region update_my_preference [TYPE Function]
# @BRIEF Delegate to ProfilePreferenceService.
> def update_my_preference(
> self,
> current_user: User,
> payload: ProfilePreferenceUpdateRequest,
> target_user_id: str | None = None,
> ) -> ProfilePreferenceResponse:
> return self.preference_service.update_my_preference(
> current_user, payload, target_user_id
> )
# endregion update_my_preference
# region lookup_superset_accounts [TYPE Function]
# @BRIEF Delegate to SupersetLookupService.
# @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API.
> async def lookup_superset_accounts(
> self,
> current_user: User,
> request: SupersetAccountLookupRequest,
> ) -> SupersetAccountLookupResponse:
> return await self.lookup_service.lookup_superset_accounts(current_user, request)
# endregion lookup_superset_accounts
# region matches_dashboard_actor [TYPE Function]
# @BRIEF Apply trim+case-insensitive actor match across owners OR modified_by.
# @PRE bound_username can be empty; owners may contain mixed payload.
# @POST Returns True when normalized username matches owners or modified_by.
> def matches_dashboard_actor(
> self,
> bound_username: str | None,
> owners: Iterable[Any] | None,
> modified_by: str | None,
> ) -> bool:
> normalized_actor = normalize_username(bound_username)
> if not normalized_actor:
> return False
> owner_tokens = normalize_owner_tokens(owners)
> modified_token = normalize_username(modified_by)
> if normalized_actor in owner_tokens:
> return True
> if modified_token and normalized_actor == modified_token:
> return True
> return False
# endregion matches_dashboard_actor
# #endregion ProfileService
# #endregion profile_service

252
backend/src/services/profile_utils.py,cover Normal file
View File

@@ -0,0 +1,252 @@
# #region ProfileUtils [C:2] [TYPE Module] [SEMANTICS string,normalization,utility,sanitize]
# @defgroup Services Module group.
# @BRIEF Pure utility helpers for profile data sanitization, normalization, and secret masking.
# Also contains shared exception classes to avoid circular imports between profile sub-modules.
# @LAYER Domain
# @RATIONALE Extracted from ProfileService to satisfy INV_7 — module length under 400 lines,
# contract length under 150 lines. These are stateless pure functions that do not
# require a class or service container.
# @REJECTED Keeping these as private methods on ProfileService was rejected because they
# inflate the class beyond 150 lines and mix pure computation with I/O-bound
# service logic.
> from collections.abc import Iterable, Sequence
> from datetime import UTC, datetime
> from typing import Any
# #region ProfileValidationError [C:2] [TYPE Class]
# @defgroup Services Module group.
# @RELATION INHERITS -> [EXT:Python:Exception]
# @BRIEF Domain validation error for profile preference update requests.
> class ProfileValidationError(Exception):
> def __init__(self, errors: Sequence[str]):
> self.errors = list(errors)
> super().__init__("Profile preference validation failed")
# #endregion ProfileValidationError
# #region EnvironmentNotFoundError [C:2] [TYPE Class]
# @defgroup Services Module group.
# @RELATION INHERITS -> [EXT:Python:Exception]
# @BRIEF Raised when environment_id from lookup request is unknown in app configuration.
> class EnvironmentNotFoundError(Exception):
> pass
# #endregion EnvironmentNotFoundError
# #region ProfileAuthorizationError [C:2] [TYPE Class]
# @defgroup Services Module group.
# @RELATION INHERITS -> [EXT:Python:Exception]
# @BRIEF Raised when caller attempts cross-user preference mutation.
> class ProfileAuthorizationError(Exception):
> pass
# #endregion ProfileAuthorizationError
> SUPPORTED_START_PAGES: set[str] = {"dashboards", "datasets", "reports"}
> SUPPORTED_DENSITIES: set[str] = {"compact", "comfortable"}
# #region sanitize_text [C:1] [TYPE Function]
# @BRIEF Normalize optional text into trimmed form or None.
> def sanitize_text(value: str | None) -> str | None:
> normalized = str(value or "").strip()
> if not normalized:
> return None
> return normalized
# #endregion sanitize_text
# #region sanitize_secret [C:1] [TYPE Function]
# @BRIEF Normalize secret input into trimmed form or None.
> def sanitize_secret(value: str | None) -> str | None:
> if value is None:
> return None
> normalized = str(value).strip()
> if not normalized:
> return None
> return normalized
# #endregion sanitize_secret
# #region sanitize_username [C:1] [TYPE Function]
# @BRIEF Normalize raw username into trimmed form or None for empty input.
> def sanitize_username(value: str | None) -> str | None:
> return sanitize_text(value)
# #endregion sanitize_username
# #region normalize_username [C:1] [TYPE Function]
# @BRIEF Apply deterministic trim+lower normalization for actor matching.
> def normalize_username(value: str | None) -> str | None:
> sanitized = sanitize_username(value)
> if sanitized is None:
> return None
> return sanitized.lower()
# #endregion normalize_username
# #region normalize_start_page [C:1] [TYPE Function]
# @BRIEF Normalize supported start page aliases to canonical values.
> def normalize_start_page(value: str | None) -> str:
> normalized = str(value or "").strip().lower()
> if normalized == "reports-logs":
> return "reports"
> if normalized in SUPPORTED_START_PAGES:
> return normalized
> return "dashboards"
# #endregion normalize_start_page
# #region normalize_density [C:1] [TYPE Function]
# @BRIEF Normalize supported density aliases to canonical values.
> def normalize_density(value: str | None) -> str:
> normalized = str(value or "").strip().lower()
> if normalized == "free":
> return "comfortable"
> if normalized in SUPPORTED_DENSITIES:
> return normalized
> return "comfortable"
# #endregion normalize_density
# #region mask_secret_value [C:1] [TYPE Function]
# @BRIEF Build a safe display value for sensitive secrets.
> def mask_secret_value(secret: str | None) -> str | None:
> sanitized = sanitize_secret(secret)
> if sanitized is None:
> return None
> if len(sanitized) <= 4:
> return "***"
> return f"{sanitized[:2]}***{sanitized[-2:]}"
# #endregion mask_secret_value
# #region validate_update_payload [C:2] [TYPE Function]
# @ingroup Services
# @BRIEF Validate username/toggle constraints for preference mutation.
# @RELATION CALLS -> [sanitize_username]
# @RELATION CALLS -> [sanitize_text]
# @RELATION DEPENDS_ON -> [ProfileUtils]
# @RELATION DEPENDS_ON -> [ProfileUtils]
# @POST Returns validation errors list; empty list means valid.
> def validate_update_payload(
> superset_username: str | None,
> show_only_my_dashboards: bool,
> git_email: str | None,
> start_page: str,
> dashboards_table_density: str,
> email_address: str | None = None,
> ) -> list[str]:
> errors: list[str] = []
> sanitized_username = sanitize_username(superset_username)
> if sanitized_username and any(ch.isspace() for ch in sanitized_username):
> errors.append(
> "Username should not contain spaces. Please enter a valid Apache Superset username."
> )
> if show_only_my_dashboards and not sanitized_username:
> errors.append(
> "Superset username is required when default filter is enabled."
> )
> sanitized_git_email = sanitize_text(git_email)
> if sanitized_git_email:
> if (
> " " in sanitized_git_email
> or "@" not in sanitized_git_email
> or sanitized_git_email.startswith("@")
> or sanitized_git_email.endswith("@")
> ):
> errors.append("Git email should be a valid email address.")
> if start_page not in SUPPORTED_START_PAGES:
> errors.append("Start page value is not supported.")
> if dashboards_table_density not in SUPPORTED_DENSITIES:
> errors.append("Dashboards table density value is not supported.")
> sanitized_email = sanitize_text(email_address)
> if sanitized_email:
> if (
> " " in sanitized_email
> or "@" not in sanitized_email
> or sanitized_email.startswith("@")
> or sanitized_email.endswith("@")
> ):
> errors.append("Notification email should be a valid email address.")
> return errors
# #endregion validate_update_payload
# #region build_default_preference [C:2] [TYPE Function] [SEMANTICS preference,default,unconfigured]
# @ingroup Services
# @BRIEF Build non-persisted default preference DTO for unconfigured users.
# @RELATION DEPENDS_ON -> [ProfilePreference]
> def build_default_preference(user_id: str) -> Any:
> """Build default ProfilePreference with disabled toggles and empty usernames."""
> from ..schemas.profile import ProfilePreference
> now = datetime.now(UTC)
> return ProfilePreference(
> user_id=str(user_id),
> superset_username=None,
> superset_username_normalized=None,
> show_only_my_dashboards=False,
> show_only_slug_dashboards=True,
> git_username=None,
> git_email=None,
> has_git_personal_access_token=False,
> git_personal_access_token_masked=None,
> start_page="dashboards",
> auto_open_task_drawer=True,
> dashboards_table_density="comfortable",
> telegram_id=None,
> email_address=None,
> notify_on_fail=True,
> created_at=now,
> updated_at=now,
> )
# #endregion build_default_preference
# #region normalize_owner_tokens [C:2] [TYPE Function]
# @ingroup Services
# @BRIEF Normalize owners payload into deduplicated lower-cased tokens.
# @RELATION CALLS -> [normalize_username]
> def normalize_owner_tokens(owners: Iterable[Any] | None) -> list[str]:
> if owners is None:
> return []
> normalized: list[str] = []
> for owner in owners:
> owner_candidates: list[Any]
> if isinstance(owner, dict):
> first_name = sanitize_username(str(owner.get("first_name") or ""))
> last_name = sanitize_username(str(owner.get("last_name") or ""))
> full_name = " ".join(
> part for part in [first_name, last_name] if part
> ).strip()
> snake_name = "_".join(
> part for part in [first_name, last_name] if part
> ).strip("_")
> owner_candidates = [
> owner.get("username"),
> owner.get("user_name"),
> owner.get("name"),
> owner.get("full_name"),
> first_name,
> last_name,
> full_name or None,
> snake_name or None,
> owner.get("email"),
> ]
> else:
> owner_candidates = [owner]
> for candidate in owner_candidates:
> token = normalize_username(str(candidate or ""))
> if token and token not in normalized:
> normalized.append(token)
> return normalized
# #endregion normalize_owner_tokens
# #endregion ProfileUtils

181
backend/src/services/rbac_permission_catalog.py,cover Normal file
View File

@@ -0,0 +1,181 @@
# #region rbac_permission_catalog [C:2] [TYPE Module] [SEMANTICS rbac, permission, discovery, route, sync]
# @defgroup Services Module group.
#
# @BRIEF Discovers declared RBAC permissions from API routes/plugins and synchronizes them into auth database.
> from collections.abc import Iterable
> from functools import lru_cache
> from pathlib import Path
> import re
> from sqlalchemy.orm import Session
> from ..core.logger import belief_scope, logger
> from ..models.auth import Permission
# #region HAS_PERMISSION_PATTERN [TYPE Constant]
# @ingroup Services
# @BRIEF Regex pattern for extracting has_permission("resource", "ACTION") declarations.
> HAS_PERMISSION_PATTERN = re.compile(
> r"""has_permission\(\s*['"]([^'"]+)['"]\s*,\s*['"]([A-Z]+)['"]\s*\)"""
> )
# #endregion HAS_PERMISSION_PATTERN
# #region ROUTES_DIR [TYPE Constant]
# @ingroup Services
# @BRIEF Absolute directory path where API route RBAC declarations are defined.
> ROUTES_DIR = Path(__file__).resolve().parent.parent / "api" / "routes"
# #endregion ROUTES_DIR
# #region _iter_route_files [C:4] [TYPE Function]
# @BRIEF Iterates API route files that may contain RBAC declarations.
# @PRE ROUTES_DIR points to backend/src/api/routes.
# @POST Yields Python files excluding test and cache directories.
> def _iter_route_files() -> Iterable[Path]:
> with belief_scope("rbac_permission_catalog._iter_route_files"):
> if not ROUTES_DIR.exists():
> return []
> files = []
> for file_path in ROUTES_DIR.rglob("*.py"):
> path_parts = set(file_path.parts)
> if "__tests__" in path_parts or "__pycache__" in path_parts:
> continue
> files.append(file_path)
> return files
# #endregion _iter_route_files
# #region _discover_route_permissions [C:4] [TYPE Function]
# @BRIEF Extracts explicit has_permission declarations from API route source code.
# @PRE Route files are readable UTF-8 text files.
# @POST Returns unique set of (resource, action) pairs declared in route guards.
> def _discover_route_permissions() -> set[tuple[str, str]]:
> with belief_scope("rbac_permission_catalog._discover_route_permissions"):
> discovered: set[tuple[str, str]] = set()
> for route_file in _iter_route_files():
> try:
> source = route_file.read_text(encoding="utf-8")
! except OSError as read_error:
! logger.explore(
! "Failed to read route file",
! extra={"src": "rbac_permission_catalog", "payload": {"file": str(route_file)}, "error": str(read_error)},
! )
! continue
> for resource, action in HAS_PERMISSION_PATTERN.findall(source):
> normalized_resource = str(resource or "").strip()
> normalized_action = str(action or "").strip().upper()
> if normalized_resource and normalized_action:
> discovered.add((normalized_resource, normalized_action))
> return discovered
# #endregion _discover_route_permissions
# #region _discover_route_permissions_cached [C:4] [TYPE Function]
# @BRIEF Cache route permission discovery because route source files are static during normal runtime.
# @PRE None.
# @POST Returns stable discovered route permission pairs without repeated filesystem scans.
> @lru_cache(maxsize=1)
> def _discover_route_permissions_cached() -> tuple[tuple[str, str], ...]:
> with belief_scope("rbac_permission_catalog._discover_route_permissions_cached"):
> return tuple(sorted(_discover_route_permissions()))
# #endregion _discover_route_permissions_cached
# #region _discover_plugin_execute_permissions [C:4] [TYPE Function]
# @BRIEF Derives dynamic task permissions of form plugin:{plugin_id}:EXECUTE from plugin registry.
# @PRE plugin_loader is optional and may expose get_all_plugin_configs.
# @POST Returns unique plugin EXECUTE permissions if loader is available.
> def _discover_plugin_execute_permissions(plugin_loader=None) -> set[tuple[str, str]]:
! with belief_scope("rbac_permission_catalog._discover_plugin_execute_permissions"):
! discovered: set[tuple[str, str]] = set()
! if plugin_loader is None:
! return discovered
! try:
! plugin_configs = plugin_loader.get_all_plugin_configs()
! except Exception as plugin_error:
! logger.explore(
! "Failed to read plugin configs for RBAC discovery",
! extra={"src": "rbac_permission_catalog", "error": str(plugin_error)},
! )
! return discovered
! for plugin_config in plugin_configs:
! plugin_id = str(getattr(plugin_config, "id", "") or "").strip()
! if plugin_id:
! discovered.add((f"plugin:{plugin_id}", "EXECUTE"))
! return discovered
# #endregion _discover_plugin_execute_permissions
# #region _discover_plugin_execute_permissions_cached [C:4] [TYPE Function]
# @BRIEF Cache dynamic plugin EXECUTE permission pairs by normalized plugin id tuple.
# @PRE plugin_ids is a deterministic tuple of plugin ids.
# @POST Returns stable permission tuple without repeated plugin catalog expansion.
> @lru_cache(maxsize=8)
> def _discover_plugin_execute_permissions_cached(
> plugin_ids: tuple[str, ...],
> ) -> tuple[tuple[str, str], ...]:
> with belief_scope("rbac_permission_catalog._discover_plugin_execute_permissions_cached"):
> return tuple((f"plugin:{plugin_id}", "EXECUTE") for plugin_id in plugin_ids)
# #endregion _discover_plugin_execute_permissions_cached
# #region discover_declared_permissions [C:4] [TYPE Function]
# @ingroup Services
# @BRIEF Builds canonical RBAC permission catalog from routes and plugin registry.
# @PRE plugin_loader may be provided for dynamic task plugin permission discovery.
# @POST Returns union of route-declared and dynamic plugin EXECUTE permissions.
> def discover_declared_permissions(plugin_loader=None) -> set[tuple[str, str]]:
> with belief_scope("rbac_permission_catalog.discover_declared_permissions"):
> permissions = set(_discover_route_permissions_cached())
> plugin_ids = tuple(
> sorted(
> {
> str(getattr(plugin_config, "id", "") or "").strip()
> for plugin_config in (plugin_loader.get_all_plugin_configs() if plugin_loader else [])
> if str(getattr(plugin_config, "id", "") or "").strip()
> }
> )
> )
> permissions.update(_discover_plugin_execute_permissions_cached(plugin_ids))
> return permissions
# #endregion discover_declared_permissions
# #region sync_permission_catalog [C:4] [TYPE Function]
# @ingroup Services
# @BRIEF Persists missing RBAC permission pairs into auth database.
# @PRE db is a valid SQLAlchemy session bound to auth database.
# @PRE declared_permissions is an iterable of (resource, action) tuples.
# @POST Missing permissions are inserted; existing permissions remain untouched.
# @SIDE_EFFECT Commits auth database transaction when new permissions are added.
> def sync_permission_catalog(
> db: Session,
> declared_permissions: Iterable[tuple[str, str]],
> ) -> int:
> with belief_scope("rbac_permission_catalog.sync_permission_catalog"):
> normalized_declared: set[tuple[str, str]] = set()
> for resource, action in declared_permissions:
> normalized_resource = str(resource or "").strip()
> normalized_action = str(action or "").strip().upper()
> if normalized_resource and normalized_action:
> normalized_declared.add((normalized_resource, normalized_action))
> existing_permissions = db.query(Permission).all()
> existing_pairs = {(perm.resource, perm.action.upper()) for perm in existing_permissions}
> missing_pairs = sorted(normalized_declared - existing_pairs)
> for resource, action in missing_pairs:
> db.add(Permission(resource=resource, action=action))
> if missing_pairs:
> db.commit()
> return len(missing_pairs)
# #endregion sync_permission_catalog
# #endregion rbac_permission_catalog

518
backend/src/services/resource_service.py,cover Normal file
View File

@@ -0,0 +1,518 @@
# #region ResourceServiceModule [C:5] [TYPE Module] [SEMANTICS resource, git, task, status, superset]
# @defgroup Services Module group.
# @BRIEF Shared service for fetching resource data with Git status and task status
# @LAYER Service
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [TaskManagerPackage]
# @RELATION DEPENDS_ON -> [TaskManagerModels]
# @RELATION DEPENDS_ON -> [GitService]
# @INVARIANT All resources include metadata about their current state
# @SIDE_EFFECT Queries multiple backends for status
# @DATA_CONTRACT ResourceQuery -> ResourceStatusSummary
> import asyncio
> from datetime import UTC, datetime
> from typing import Any
> from ..core.logger import belief_scope, logger
> from ..core.task_manager.models import Task
> from ..core.utils.client_registry import get_superset_client
> from ..services.git_service import GitService
# #region ResourceService [C:3] [TYPE Class]
# @defgroup Services Module group.
# @BRIEF Provides centralized access to resource data with enhanced metadata
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [GitService]
> class ResourceService:
# region ResourceService_init [TYPE Function]
# @PURPOSE: Initialize the resource service with dependencies
# @PRE None
# @POST ResourceService is ready to fetch resources
> def __init__(self):
> with belief_scope("ResourceService.__init__"):
> self.git_service = GitService()
> logger.reason("Initialized ResourceService", extra={"src": "ResourceService"})
# endregion ResourceService_init
# region get_dashboards_with_status [TYPE Function]
# @PURPOSE: Fetch dashboards from environment with Git status and last task status
# @PRE env is a valid Environment object
# @POST Returns list of dashboards with enhanced metadata
# @PARAM env (Environment) - The environment to fetch from
# @PARAM tasks (List[Task]) - List of tasks to check for status
# @RETURN List[Dict] - Dashboards with git_status and last_task fields
# @RELATION CALLS -> [SupersetClientGetDashboardsSummary]
# @RELATION CALLS ->[EXT:method:_get_git_status_for_dashboard]
# @RELATION CALLS ->[EXT:method:_get_last_llm_task_for_dashboard]
> async def get_dashboards_with_status(
> self,
> env: Any,
> tasks: list[Task] | None = None,
> include_git_status: bool = True,
> require_slug: bool = False,
> ) -> list[dict[str, Any]]:
> with belief_scope("get_dashboards_with_status", f"env={env.id}"):
> client = await get_superset_client(env)
> dashboards = await client.get_dashboards_summary(require_slug=require_slug)
# Enhance each dashboard with Git status and task status
> result = []
> for dashboard in dashboards:
# dashboard is already a dict, no need to call .dict()
> dashboard_dict = dashboard
> dashboard_id = dashboard_dict.get('id')
# Git status can be skipped for list endpoints and loaded lazily on UI side.
> if include_git_status:
> git_status = self._get_git_status_for_dashboard(dashboard_id)
> dashboard_dict['git_status'] = git_status
> else:
> dashboard_dict['git_status'] = None
# Show status of the latest LLM validation for this dashboard.
> last_task = self._get_last_llm_task_for_dashboard(
> dashboard_id,
> env.id,
> tasks,
> )
> dashboard_dict['last_task'] = last_task
> result.append(dashboard_dict)
> logger.info(f"[ResourceService][Coherence:OK] Fetched {len(result)} dashboards with status")
> return result
# endregion get_dashboards_with_status
# region get_dashboards_page_with_status [TYPE Function]
# @PURPOSE: Fetch one dashboard page from environment and enrich only that page with status metadata.
# @PRE env is valid; page >= 1; page_size > 0.
# @POST Returns page items plus total counters without scanning all pages locally.
# @PARAM env (Environment) - Source environment.
# @PARAM tasks (Optional[List[Task]]) - Tasks for latest LLM status.
# @PARAM page (int) - 1-based page number.
# @PARAM page_size (int) - Page size.
# @RETURN Dict[str, Any] - {"dashboards": List[Dict], "total": int, "total_pages": int}
# @RELATION CALLS -> [SupersetClientGetDashboardsSummaryPage]
# @RELATION CALLS ->[EXT:method:_get_git_status_for_dashboard]
# @RELATION CALLS ->[EXT:method:_get_last_llm_task_for_dashboard]
> async def get_dashboards_page_with_status(
> self,
> env: Any,
> tasks: list[Task] | None = None,
> page: int = 1,
> page_size: int = 10,
> search: str | None = None,
> include_git_status: bool = True,
> require_slug: bool = False,
> ) -> dict[str, Any]:
> with belief_scope(
> "get_dashboards_page_with_status",
> f"env={env.id}, page={page}, page_size={page_size}, search={search}",
> ):
> client = await get_superset_client(env)
> total, dashboards_page = await client.get_dashboards_summary_page(
> page=page,
> page_size=page_size,
> search=search,
> require_slug=require_slug,
> )
> result = []
> for dashboard in dashboards_page:
> dashboard_dict = dashboard
> dashboard_id = dashboard_dict.get("id")
> if include_git_status:
> dashboard_dict["git_status"] = self._get_git_status_for_dashboard(dashboard_id)
> else:
> dashboard_dict["git_status"] = None
> dashboard_dict["last_task"] = self._get_last_llm_task_for_dashboard(
> dashboard_id,
> env.id,
> tasks,
> )
> result.append(dashboard_dict)
> total_pages = (total + page_size - 1) // page_size if total > 0 else 1
> logger.info(
> "[ResourceService][Coherence:OK] Fetched dashboards page %s/%s (%s items, total=%s)",
> page,
> total_pages,
> len(result),
> total,
> )
> return {
> "dashboards": result,
> "total": total,
> "total_pages": total_pages,
> }
# endregion get_dashboards_page_with_status
# region _get_last_llm_task_for_dashboard [TYPE Function]
# @PURPOSE: Get most recent LLM validation task for a dashboard in an environment
# @PRE dashboard_id is a valid integer identifier
# @POST Returns the newest llm_dashboard_validation task summary or None
# @PARAM dashboard_id (int) - The dashboard ID
# @PARAM env_id (Optional[str]) - Environment ID to match task params
# @PARAM tasks (Optional[List[Task]]) - List of tasks to search
# @RETURN Optional[Dict] - Task summary with task_id and status
# @RELATION CALLS ->[EXT:method:_normalize_datetime_for_compare]
# @RELATION CALLS ->[EXT:method:_normalize_validation_status]
# @RELATION CALLS ->[EXT:method:_normalize_task_status]
> def _get_last_llm_task_for_dashboard(
> self,
> dashboard_id: int,
> env_id: str | None,
> tasks: list[Task] | None = None,
> ) -> dict[str, Any] | None:
> if not tasks:
> return None
> dashboard_id_str = str(dashboard_id)
> matched_tasks = []
> for task in tasks:
> if getattr(task, "plugin_id", None) != "llm_dashboard_validation":
> continue
> params = getattr(task, "params", {}) or {}
> if str(params.get("dashboard_id")) != dashboard_id_str:
> continue
> if env_id is not None:
> task_env = params.get("environment_id") or params.get("env")
> if str(task_env) != str(env_id):
> continue
> matched_tasks.append(task)
> if not matched_tasks:
> return None
> def _task_time(task_obj: Any) -> datetime:
> raw_time = (
> getattr(task_obj, "started_at", None)
> or getattr(task_obj, "finished_at", None)
> or getattr(task_obj, "created_at", None)
> )
> return self._normalize_datetime_for_compare(raw_time)
> projected_tasks = []
> for task in matched_tasks:
> raw_result = getattr(task, "result", None)
> validation_status = None
> if isinstance(raw_result, dict):
> validation_status = self._normalize_validation_status(raw_result.get("status"))
> projected_tasks.append(
> (
> task,
> validation_status,
> _task_time(task),
> )
> )
> projected_tasks.sort(key=lambda item: item[2], reverse=True)
> latest_task, latest_validation_status, _ = projected_tasks[0]
> decisive_task = next(
> (
> item for item in projected_tasks
> if item[1] in {"PASS", "WARN", "FAIL"}
> ),
> None,
> )
> validation_status = latest_validation_status
> if validation_status == "UNKNOWN" and decisive_task is not None:
> validation_status = decisive_task[1]
> return {
> "task_id": str(getattr(latest_task, "id", "")),
> "status": self._normalize_task_status(getattr(latest_task, "status", "")),
> "validation_status": validation_status,
> }
# endregion _get_last_llm_task_for_dashboard
# region _normalize_task_status [TYPE Function]
# @PURPOSE: Normalize task status to stable uppercase values for UI/API projections
# @PRE raw_status can be enum or string
# @POST Returns uppercase status without enum class prefix
# @PARAM raw_status (Any) - Raw task status object/value
# @RETURN str - Normalized status token
# @RELATION CALLED_BY -> [EXT:method:_get_last_llm_task_for_dashboard]
> def _normalize_task_status(self, raw_status: Any) -> str:
> if raw_status is None:
> return ""
> value = getattr(raw_status, "value", raw_status)
> status_text = str(value).strip()
> if "." in status_text:
> status_text = status_text.split(".")[-1]
> return status_text.upper()
# endregion _normalize_task_status
# region _normalize_validation_status [TYPE Function]
# @PURPOSE: Normalize LLM validation status to PASS/FAIL/WARN/UNKNOWN
# @PRE raw_status can be any scalar type
# @POST Returns normalized validation status token or None
# @PARAM raw_status (Any) - Raw validation status from task result
# @RETURN Optional[str] - PASS|FAIL|WARN|UNKNOWN
# @RELATION CALLED_BY -> [EXT:method:_get_last_llm_task_for_dashboard]
> def _normalize_validation_status(self, raw_status: Any) -> str | None:
> if raw_status is None:
> return None
> status_text = str(raw_status).strip().upper()
> if status_text in {"PASS", "FAIL", "WARN"}:
> return status_text
> return "UNKNOWN"
# endregion _normalize_validation_status
# region _normalize_datetime_for_compare [TYPE Function]
# @PURPOSE: Normalize datetime values to UTC-aware values for safe comparisons.
# @PRE value may be datetime or any scalar.
# @POST Returns UTC-aware datetime; non-datetime values map to minimal UTC datetime.
# @PARAM value (Any) - Candidate datetime-like value.
# @RETURN datetime - UTC-aware comparable datetime.
# @RELATION CALLED_BY -> [EXT:method:_get_last_llm_task_for_dashboard]
# @RELATION CALLED_BY -> [EXT:method:_get_last_task_for_resource]
> def _normalize_datetime_for_compare(self, value: Any) -> datetime:
> if isinstance(value, datetime):
> if value.tzinfo is None:
> return value.replace(tzinfo=UTC)
> return value.astimezone(UTC)
> return datetime.min.replace(tzinfo=UTC)
# endregion _normalize_datetime_for_compare
# region get_datasets_with_status [TYPE Function]
# @PURPOSE: Fetch datasets from environment with mapping progress and last task status
# @PRE env is a valid Environment object
# @POST Returns list of datasets with enhanced metadata
# @PARAM env (Environment) - The environment to fetch from
# @PARAM tasks (List[Task]) - List of tasks to check for status
# @RETURN List[Dict] - Datasets with mapped_fields, last_task and linked_dashboard_count fields
# @RELATION CALLS -> [SupersetClientGetDatasetsSummary]
# @RELATION CALLS -> [SupersetClientGetDatasetLinkedDashboardCount]
# @RELATION CALLS ->[EXT:method:_get_last_task_for_resource]
# @RATIONALE linked_dashboard_count was missing — get_datasets_summary() only returns id/table_name/schema/database
# StatsBar showed linked_count=0 because the field was never populated in list endpoint.
# Fix: fetch /dataset/{id}/related_objects for each dataset concurrently (semaphore=3, timeout=10s).
# @REJECTED N+1 blocking calls rejected — would timeout at 26×1s. Semaphore + asyncio.to_thread + timeout prevents overload.
> async def get_datasets_with_status(
> self,
> env: Any,
> tasks: list[Task] | None = None
> ) -> list[dict[str, Any]]:
> with belief_scope("get_datasets_with_status", f"env={env.id}"):
> client = await get_superset_client(env)
> datasets = await client.get_datasets_summary()
# Enhance each dataset with task status and linked dashboard count
> sem = asyncio.Semaphore(3) # limit concurrent Superset calls
> async def fetch_linked_count(ds_id: int) -> int:
> async with sem:
> try:
> return await asyncio.wait_for(
> client.get_dataset_linked_dashboard_count(ds_id),
> timeout=10.0,
> )
> except (TimeoutError, Exception):
> logger.warning(
> f"[EXT:method:get_datasets_with_status][Warning] "
> f"Failed to fetch linked dashboard count for dataset {ds_id}"
> )
> return 0
> result = []
> linked_tasks = []
> for dataset in datasets:
> dataset_dict = dataset
> dataset_id = dataset_dict.get('id')
# Get last task status
> last_task = self._get_last_task_for_resource(
> f"dataset-{dataset_id}",
> tasks
> )
> dataset_dict['last_task'] = last_task
> result.append(dataset_dict)
> linked_tasks.append(fetch_linked_count(dataset_id))
# Fetch linked dashboard counts concurrently — FR-022
> if linked_tasks:
> linked_counts = await asyncio.gather(*linked_tasks)
> for ds, count in zip(result, linked_counts):
> ds['linked_dashboard_count'] = count
> logger.info(
> f"[ResourceService][Coherence:OK] "
> f"Fetched {len(result)} datasets with status and linked counts"
> )
> return result
# endregion get_datasets_with_status
# region get_activity_summary [TYPE Function]
# @PURPOSE: Get summary of active and recent tasks for the activity indicator
# @PRE tasks is a list of Task objects
# @POST Returns summary with active_count and recent_tasks
# @PARAM tasks (List[Task]) - List of tasks to summarize
# @RETURN Dict - Activity summary
# @RELATION CALLS ->[EXT:method:_extract_resource_name_from_task]
# @RELATION CALLS ->[EXT:method:_extract_resource_type_from_task]
> def get_activity_summary(self, tasks: list[Task]) -> dict[str, Any]:
> with belief_scope("get_activity_summary"):
# Count active (RUNNING, WAITING_INPUT) tasks
> active_tasks = [
> t for t in tasks
> if t.status in ['RUNNING', 'WAITING_INPUT']
> ]
# Get recent tasks (last 5)
> recent_tasks = sorted(
> tasks,
> key=lambda t: t.created_at,
> reverse=True
> )[:5]
# Format recent tasks for frontend
> recent_tasks_formatted = []
> for task in recent_tasks:
> resource_name = self._extract_resource_name_from_task(task)
> recent_tasks_formatted.append({
> 'task_id': str(task.id),
> 'resource_name': resource_name,
> 'resource_type': self._extract_resource_type_from_task(task),
> 'status': task.status,
> 'started_at': task.created_at.isoformat() if task.created_at else None
> })
> return {
> 'active_count': len(active_tasks),
> 'recent_tasks': recent_tasks_formatted
> }
# endregion get_activity_summary
# region _get_git_status_for_dashboard [TYPE Function]
# @PURPOSE: Get Git sync status for a dashboard
# @PRE dashboard_id is a valid integer
# @POST Returns git status or None if no repo exists
# @PARAM dashboard_id (int) - The dashboard ID
# @RETURN Optional[Dict] - Git status with branch and sync_status
# @RELATION CALLS ->[EXT:method:get_repo]
> def _get_git_status_for_dashboard(self, dashboard_id: int) -> dict[str, Any] | None:
> try:
> repo = self.git_service.get_repo(dashboard_id)
> if not repo:
> return {
> 'branch': None,
> 'sync_status': 'NO_REPO',
> 'has_repo': False,
> 'has_changes_for_commit': False
> }
# Check if there are uncommitted changes
> try:
# Get current branch
> branch = repo.active_branch.name
# Check for uncommitted changes
> is_dirty = repo.is_dirty()
> has_changes_for_commit = repo.is_dirty(untracked_files=True)
# Check for unpushed commits
> unpushed = len(list(repo.iter_commits(f'{branch}@{{u}}..{branch}'))) if '@{u}' in str(repo.refs) else 0
> if is_dirty or unpushed > 0:
> sync_status = 'DIFF'
> else:
> sync_status = 'OK'
> return {
> 'branch': branch,
> 'sync_status': sync_status,
> 'has_repo': True,
> 'has_changes_for_commit': has_changes_for_commit
> }
> except Exception:
> logger.warning(f"[ResourceService][Warning] Failed to get git status for dashboard {dashboard_id}")
> return {
> 'branch': None,
> 'sync_status': 'ERROR',
> 'has_repo': True,
> 'has_changes_for_commit': False
> }
> except Exception:
# No repo exists for this dashboard
> return {
> 'branch': None,
> 'sync_status': 'NO_REPO',
> 'has_repo': False,
> 'has_changes_for_commit': False
> }
# endregion _get_git_status_for_dashboard
# region _get_last_task_for_resource [TYPE Function]
# @PURPOSE: Get the most recent task for a specific resource
# @PRE resource_id is a valid string
# @POST Returns task summary or None if no tasks found
# @PARAM resource_id (str) - The resource identifier (e.g., "dashboard-123")
# @PARAM tasks (Optional[List[Task]]) - List of tasks to search
# @RETURN Optional[Dict] - Task summary with task_id and status
# @RELATION CALLS ->[EXT:method:_normalize_datetime_for_compare]
> def _get_last_task_for_resource(
> self,
> resource_id: str,
> tasks: list[Task] | None = None
> ) -> dict[str, Any] | None:
> if not tasks:
> return None
# Filter tasks for this resource
> resource_tasks = []
> for task in tasks:
> params = task.params or {}
> if params.get('resource_id') == resource_id:
> resource_tasks.append(task)
> if not resource_tasks:
> return None
# Get most recent task with timezone-safe comparison.
> last_task = max(
> resource_tasks,
> key=lambda t: self._normalize_datetime_for_compare(getattr(t, "created_at", None)),
> )
> return {
> 'task_id': str(last_task.id),
> 'status': last_task.status
> }
# endregion _get_last_task_for_resource
# region _extract_resource_name_from_task [TYPE Function]
# @PURPOSE: Extract resource name from task params
# @PRE task is a valid Task object
# @POST Returns resource name or task ID
# @PARAM task (Task) - The task to extract from
# @RETURN str - Resource name or fallback
# @RELATION CALLED_BY -> [EXT:method:get_activity_summary]
> def _extract_resource_name_from_task(self, task: Task) -> str:
> params = task.params or {}
> return params.get('resource_name', f"Task {task.id}")
# endregion _extract_resource_name_from_task
# region _extract_resource_type_from_task [TYPE Function]
# @PURPOSE: Extract resource type from task params
# @PRE task is a valid Task object
# @POST Returns resource type or 'unknown'
# @PARAM task (Task) - The task to extract from
# @RETURN str - Resource type
# @RELATION CALLED_BY -> [EXT:method:get_activity_summary]
> def _extract_resource_type_from_task(self, task: Task) -> str:
> params = task.params or {}
> return params.get('resource_type', 'unknown')
# endregion _extract_resource_type_from_task
# #endregion ResourceService
# #endregion ResourceServiceModule

137
backend/src/services/security_badge_service.py,cover Normal file
View File

@@ -0,0 +1,137 @@
# #region security_badge_service [C:4] [TYPE Module] [SEMANTICS profile,security,permissions,badges]
# @defgroup Services Module group.
# @BRIEF Builds security summary and permission badges for profile UI — role extraction,
# permission pair collection, and catalog formatting.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [User]
# @RELATION DEPENDS_ON -> [discover_declared_permissions]
# @RELATION DEPENDS_ON -> [ProfileUtils]
# @RATIONALE Extracted from ProfileService to satisfy INV_7. Security badge construction
# is a distinct concern with its own dependencies (discover_declared_permissions,
# plugin_loader) and can be tested independently.
# @REJECTED Keeping security badge logic inside ProfileService was rejected — it adds
# plugin_loader coupling to preference operations that do not need it and
# inflates the class past 150 lines.
# @PRE Plugin loader may be None; user must have roles/permissions graph.
# @POST Returns deterministic security projection for profile UI with sorted permissions.
# @SIDE_EFFECT Reads plugin permissions via discover_declared_permissions when plugin_loader is set.
> from typing import Any
> from ..core.logger import belief_scope, logger
> from ..models.auth import User
> from ..schemas.profile import ProfilePermissionState, ProfileSecuritySummary
> from .profile_utils import sanitize_text
> from .rbac_permission_catalog import discover_declared_permissions
# #region SecurityBadgeService [C:4] [TYPE Class] [SEMANTICS security,permission,badge,profile]
# @defgroup Services Module group.
# @BRIEF Builds security summary with role names and permission badges for profile UI display.
# @RELATION DEPENDS_ON -> [User]
# @RELATION CALLS -> [discover_declared_permissions]
# @PRE plugin_loader may be None for degraded permission discovery.
# @POST build_security_summary returns ProfileSecuritySummary with deterministic sorted permissions.
> class SecurityBadgeService:
> """Builds security summary with role names and permission badges."""
# #region __init__ [TYPE Function]
# @BRIEF Initialize service with optional plugin loader for permission discovery.
> def __init__(self, plugin_loader: Any = None):
> self.plugin_loader = plugin_loader
# #endregion __init__
# #region build_security_summary [TYPE Function]
# @ingroup Services
# @BRIEF Build read-only security snapshot with role and permission badges.
# @PRE current_user is authenticated.
# @POST Returns deterministic security projection for profile UI.
> def build_security_summary(self, current_user: User) -> ProfileSecuritySummary:
> with belief_scope("SecurityBadgeService.build_security_summary"):
> role_names_set: set[str] = set()
> roles = getattr(current_user, "roles", []) or []
> for role in roles:
> normalized_role_name = sanitize_text(getattr(role, "name", None))
> if normalized_role_name:
> role_names_set.add(normalized_role_name)
> role_names = sorted(role_names_set)
> is_admin = any(str(role_name).lower() == "admin" for role_name in role_names)
> user_permission_pairs = self._collect_user_permission_pairs(current_user)
> declared_permission_pairs: set[tuple[str, str]] = set()
> try:
> discovered_permissions = discover_declared_permissions(
> plugin_loader=self.plugin_loader
> )
> for resource, action in discovered_permissions:
> normalized_resource = sanitize_text(resource)
> normalized_action = str(action or "").strip().upper()
> if normalized_resource and normalized_action:
> declared_permission_pairs.add(
> (normalized_resource, normalized_action)
> )
> except Exception as discovery_error:
> logger.explore(
> "Failed to build declared permission catalog",
> extra={"error": str(discovery_error), "src": "SecurityBadgeService.build_security_summary"},
> )
> if not declared_permission_pairs:
> declared_permission_pairs = set(user_permission_pairs)
> sorted_permission_pairs = sorted(
> declared_permission_pairs,
> key=lambda pair: (pair[0], pair[1]),
> )
> permission_states = [
> ProfilePermissionState(
> key=self._format_permission_key(resource, action),
> allowed=bool(is_admin or (resource, action) in user_permission_pairs),
> )
> for resource, action in sorted_permission_pairs
> ]
> auth_source = sanitize_text(getattr(current_user, "auth_source", None))
> current_role = "Admin" if is_admin else (role_names[0] if role_names else None)
> return ProfileSecuritySummary(
> read_only=True,
> auth_source=auth_source,
> current_role=current_role,
> role_source=auth_source,
> roles=role_names,
> permissions=permission_states,
> )
# #endregion build_security_summary
# #region _collect_user_permission_pairs [TYPE Function]
# @BRIEF Collect effective permission tuples from current user's roles.
# @PRE current_user can include role/permission graph.
# @POST Returns unique normalized (resource, ACTION) tuples.
> def _collect_user_permission_pairs(
> self, current_user: User
> ) -> set[tuple[str, str]]:
> collected: set[tuple[str, str]] = set()
> roles = getattr(current_user, "roles", []) or []
> for role in roles:
> permissions = getattr(role, "permissions", []) or []
> for permission in permissions:
> resource = sanitize_text(getattr(permission, "resource", None))
> action = str(getattr(permission, "action", "") or "").strip().upper()
> if resource and action:
> collected.add((resource, action))
> return collected
# #endregion _collect_user_permission_pairs
# #region _format_permission_key [C:1] [TYPE Function]
# @BRIEF Convert normalized permission pair to compact UI key.
> def _format_permission_key(self, resource: str, action: str) -> str:
> normalized_resource = sanitize_text(resource) or ""
> normalized_action = str(action or "").strip().upper()
> if normalized_action == "READ":
> return normalized_resource
> return f"{normalized_resource}:{normalized_action.lower()}"
# #endregion _format_permission_key
# #endregion SecurityBadgeService
# #endregion security_badge_service

231
backend/src/services/sql_table_extractor.py,cover Normal file
View File

@@ -0,0 +1,231 @@
# #region SqlTableExtractorModule [C:2] [TYPE Module] [SEMANTICS sql, jinja, table, extraction, parsing]
# @defgroup Services Module group.
# @BRIEF Three-phase SQL+Jinja table name extractor for virtual datasets as per FR-003.
# Phase 1: Detect Jinja spans vs SQL spans
# Phase 2: In Jinja spans, extract "schema.table" from string values
# Phase 3: In SQL spans, regex pattern + sqlparse filter to reject string literal false positives
# @LAYER Service
# @RELATION DEPENDS_ON -> [EXT:Library:sqlparse]
# @INVARIANT Only exact schema.table matches (case-insensitive); unqualified references are NOT matched.
# @INVARIANT Returns a set[str] of fully-qualified table names (lowercased for case-insensitive matching).
> from collections.abc import Iterable
> import re
> import sqlparse
> from sqlparse.sql import Token, TokenList
> from sqlparse.tokens import Literal
# ── Phase 1: Jinja span detection ───────────────────────────
# Minimal Jinja token detection: {% ... %}, {{ ... }}, {# ... #}
> _JINJA_BLOCK_RE = re.compile(r"{%-?\s*.*?\s*-?%}", re.DOTALL)
> _JINJA_EXPR_RE = re.compile(r"{{-?\s*.*?\s*-?}}", re.DOTALL)
> _JINJA_COMMENT_RE = re.compile(r"{#.*?#}", re.DOTALL)
# ── Phase 3: schema.table regex (case-insensitive) ───────────
> _SCHEMA_TABLE_RE = re.compile(
> r"""
> (?<!['"`\w]) # not preceded by string delimiter or word char
> \b # word boundary
> (?: # begin: non-capturing group for full match
> (?:"([a-zA-Z_][\w]*)")? # optional quoted schema
> (?:([a-zA-Z_][\w]*)) # schema (unquoted)
> \.
> (?:([a-zA-Z_][\w]*)) # table
> )
> \b # word boundary
> (?!['"`]) # not followed by a string delimiter
> """,
> re.VERBOSE | re.IGNORECASE,
> )
# #region detect_jinja_spans [C:2] [TYPE Function]
# @ingroup Services
# @BRIEF Phase 1: Split raw SQL text into Jinja spans and SQL spans.
# @PRE raw_sql is a string (possibly empty).
# @POST Returns a list of (span_type: str, text: str) tuples.
# span_type is "jinja" or "sql".
> def detect_jinja_spans(raw_sql: str) -> list[tuple[str, str]]:
> """Split raw SQL+Jinja text into alternating Jinja and SQL spans.
> Phase 1 detects Jinja blocks ({%%}, {{}}, {##}) and returns the
> remaining text as SQL spans.
> """
> if not raw_sql or not raw_sql.strip():
! return [("sql", raw_sql or "")]
> spans: list[tuple[str, str]] = []
# Collect all Jinja match positions
> jinja_matches: list[tuple[int, int]] = []
> for pattern in (_JINJA_BLOCK_RE, _JINJA_EXPR_RE, _JINJA_COMMENT_RE):
> for m in pattern.finditer(raw_sql):
> jinja_matches.append((m.start(), m.end()))
# Merge overlapping Jinja spans
> if jinja_matches:
> jinja_matches.sort()
> merged: list[tuple[int, int]] = [jinja_matches[0]]
> for start, end in jinja_matches[1:]:
! if start <= merged[-1][1]:
! merged[-1] = (merged[-1][0], max(merged[-1][1], end))
! else:
! merged.append((start, end))
# Build alternating sql/jinja spans
> cursor = 0
> for start, end in merged:
> if cursor < start:
> sql_chunk = raw_sql[cursor:start].strip()
> if sql_chunk:
> spans.append(("sql", sql_chunk))
> jinja_chunk = raw_sql[start:end].strip()
> if jinja_chunk:
> spans.append(("jinja", jinja_chunk))
> cursor = end
> if cursor < len(raw_sql):
> remaining = raw_sql[cursor:].strip()
> if remaining:
> spans.append(("sql", remaining))
> else:
> spans.append(("sql", raw_sql.strip()))
> return spans
# #endregion detect_jinja_spans
# #region extract_tables_from_jinja [C:2] [TYPE Function]
# @ingroup Services
# @BRIEF Phase 2: Extract "schema.table" references from Jinja string values.
# Looks for patterns like "raw.sales" inside Jinja text.
# @PRE jinja_text is a string from a Jinja span.
# @POST Returns a set of lowercased schema.table strings.
> def extract_tables_from_jinja(jinja_text: str) -> set[str]:
> """Extract ``schema.table`` references from within Jinja template blocks.
> Looks for double-quoted or single-quoted string values that match
> the ``schema.table`` pattern, e.g. ``"raw.sales"`` or ``'raw.inventory'``.
> """
> tables: set[str] = set()
# Match string values: "schema.table" or 'schema.table'
> string_pattern = re.compile(
> r"""["']([a-zA-Z_][\w]*\.[a-zA-Z_][\w]*)["']"""
> )
> for m in string_pattern.finditer(jinja_text):
> tables.add(m.group(1).lower())
> return tables
# #endregion extract_tables_from_jinja
# #region is_string_literal [C:1] [TYPE Function]
# @BRIEF Check if a sqlparse Token is a string literal (not a schema.table identifier).
# @PRE token is a sqlparse Token.
# @POST Returns True if the token is a string/Single/Literal.String.
> def is_string_literal(token: Token) -> bool:
> """Return True if token is a string literal type in sqlparse."""
> return token.ttype is Literal.String.Single or token.ttype is Literal.String
# #endregion is_string_literal
# #region extract_tables_from_sql_span [C:2] [TYPE Function]
# @ingroup Services
# @BRIEF Phase 3: Extract schema.table references from SQL text using regex + sqlparse filtering.
# Uses regex to find all schema.table candidates, then sqlparse to filter out
# false positives inside string literals.
# @PRE sql_text is a string from a SQL span (non-Jinja).
# @POST Returns a set of lowercased schema.table strings.
> def extract_tables_from_sql_span(sql_text: str) -> set[str]:
> """Extract ``schema.table`` references from plain SQL text.
> Uses regex to find all ``schema.table`` candidates, then sqlparse
> to reject matches that fall inside string literals.
> """
> tables: set[str] = set()
> raw_matches = _SCHEMA_TABLE_RE.findall(sql_text)
> if not raw_matches:
> return tables
# Use sqlparse to identify string literal positions
> parsed = sqlparse.parse(sql_text)
> string_literal_ranges: list[tuple[int, int]] = []
> def walk_tokens(tokens: Iterable[Token], base_offset: int = 0) -> None:
> offset = base_offset
> for token in tokens:
> if isinstance(token, TokenList):
! walk_tokens(token.flatten(), offset)
> else:
> ttype = token.ttype
> val = token.value
> if is_string_literal(token):
> string_literal_ranges.append(
> (offset, offset + len(val))
> )
> offset += len(val)
> for stmt in parsed:
> if stmt is None:
! continue
> walk_tokens(stmt.flatten(), base_offset=0)
> def is_in_string(pos: int) -> bool:
> for s_start, s_end in string_literal_ranges:
> if s_start <= pos <= s_end:
! return True
> return False
# Re-scan with full match positions to filter
> for m in _SCHEMA_TABLE_RE.finditer(sql_text):
> if not is_in_string(m.start()):
# Extract schema and table from match groups
> schema = m.group(2) or m.group(1) or ""
> table = m.group(3) or ""
# Filter out column aliases (single-char schemas like "a.id", "o.id")
> if len(schema) < 2 and schema.isalpha() and schema.islower():
> continue
> if schema and table:
> tables.add(f"{schema.lower()}.{table.lower()}")
> return tables
# #endregion extract_tables_from_sql_span
# #region extract_tables_from_sql [C:2] [TYPE Function]
# @ingroup Services
# @BRIEF Three-phase entry point: extract all schema.table references from raw SQL+Jinja text.
# @PRE raw_sql is a string (possibly empty).
# @POST Returns a set of lowercased fully-qualified table names (schema.table format).
# @RELATION CALLS -> [detect_jinja_spans]
# @RELATION CALLS -> [extract_tables_from_jinja]
# @RELATION CALLS -> [extract_tables_from_sql_span]
> def extract_tables_from_sql(raw_sql: str) -> set[str]:
> """Extract all ``schema.table`` references from raw SQL + Jinja text.
> Three-phase approach per FR-003:
> 1. Detect Jinja spans vs SQL spans
> 2. In Jinja spans, extract ``schema.table`` from quoted string values
> 3. In SQL spans, regex + sqlparse filter to reject string literal false positives
> Returns a set of lowercased ``schema.table`` strings for case-insensitive matching.
> Example:
> >>> extract_tables_from_sql("SELECT * FROM raw.sales JOIN raw.inventory ON ...")
> {'raw.sales', 'raw.inventory'}
> """
> if not raw_sql or not raw_sql.strip():
> return set()
> all_tables: set[str] = set()
> spans = detect_jinja_spans(raw_sql)
> for span_type, text in spans:
> if span_type == "jinja":
> all_tables.update(extract_tables_from_jinja(text))
> else:
> all_tables.update(extract_tables_from_sql_span(text))
> return all_tables
# #endregion extract_tables_from_sql
# #endregion SqlTableExtractorModule

144
backend/src/services/superset_lookup_service.py,cover Normal file
View File

@@ -0,0 +1,144 @@
# #region superset_lookup_service [C:4] [TYPE Module] [SEMANTICS superset,account,lookup,environment]
# @defgroup Services Module group.
# @BRIEF Environment-scoped Superset account lookup with degradation fallback for network failures.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SupersetAccountLookupAdapter]
# @RELATION DEPENDS_ON -> [ProfileUtils]
# @RATIONALE Extracted from ProfileService to satisfy INV_7. Superset account lookup has distinct
# I/O patterns (external HTTP calls, environment resolution) that isolate cleanly from
# preference persistence and security badge logic.
# @REJECTED Keeping lookup inside ProfileService was rejected — it adds network I/O and
# environment coupling to preference operations that only need DB access, and inflates
# the class past 150 lines.
# @PRE current_user is authenticated; environment_id must be resolvable.
# @POST Returns success or degraded SupersetAccountLookupResponse with stable shape.
# @SIDE_EFFECT Performs external Superset HTTP calls via SupersetClient + adapter.
# @RAISES EnvironmentNotFoundError when environment_id is unknown.
> from typing import Any
> from ..core.logger import belief_scope, logger
> from ..core.async_superset_client import AsyncSupersetClient
> from ..core.superset_profile_lookup import SupersetAccountLookupAdapter
> from ..schemas.profile import (
> SupersetAccountCandidate,
> SupersetAccountLookupRequest,
> SupersetAccountLookupResponse,
> )
> from .profile_utils import EnvironmentNotFoundError
# #region SupersetLookupService [C:4] [TYPE Class] [SEMANTICS superset,lookup,environment,degradation]
# @defgroup Services Module group.
# @BRIEF Resolves environments and queries Superset users in selected environment,
# returning canonical account candidates with degradation fallback.
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SupersetAccountLookupAdapter]
# @RELATION CALLS -> [SupersetClient]
# @PRE config_manager supports get_environments().
# @POST lookup_superset_accounts returns success payload or degraded payload with warning.
> class SupersetLookupService:
> """Environment-scoped Superset account lookup with degradation fallback."""
# #region __init__ [TYPE Function]
# @BRIEF Initialize with config manager for environment resolution.
> def __init__(self, config_manager: Any):
> self.config_manager = config_manager
# #endregion __init__
# #region lookup_superset_accounts [TYPE Function]
# @ingroup Services
# @BRIEF Query Superset users in selected environment and project canonical account candidates.
# @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API.
# @PRE current_user is authenticated and environment_id exists.
# @POST Returns success payload or degraded payload with warning while preserving manual fallback.
> async def lookup_superset_accounts(
> self,
> current_user: Any,
> request: SupersetAccountLookupRequest,
> ) -> SupersetAccountLookupResponse:
> with belief_scope(
> "SupersetLookupService.lookup_superset_accounts",
> f"user_id={getattr(current_user, 'id', '?')}, environment_id={request.environment_id}",
> ):
> environment = self._resolve_environment(request.environment_id)
> if environment is None:
> logger.explore("Lookup aborted: environment not found")
> raise EnvironmentNotFoundError(
> f"Environment '{request.environment_id}' not found"
> )
> sort_column = str(request.sort_column or "username").strip().lower()
> sort_order = str(request.sort_order or "desc").strip().lower()
> allowed_columns = {"username", "first_name", "last_name", "email"}
> if sort_column not in allowed_columns:
> sort_column = "username"
> if sort_order not in {"asc", "desc"}:
> sort_order = "desc"
> logger.reflect(
> "Normalized lookup request "
> f"(env={request.environment_id}, sort_column={sort_column}, sort_order={sort_order}, "
> f"page_index={request.page_index}, page_size={request.page_size}, "
> f"search={(request.search or '').strip()!r})"
> )
> try:
> logger.reason("Performing Superset account lookup")
> superset_client = AsyncSupersetClient(environment)
> adapter = SupersetAccountLookupAdapter(
> network_client=superset_client.client,
> environment_id=request.environment_id,
> )
> lookup_result = await adapter.get_users_page(
> search=request.search,
> page_index=request.page_index,
> page_size=request.page_size,
> sort_column=sort_column,
> sort_order=sort_order,
> )
> items = [
> SupersetAccountCandidate.model_validate(item)
> for item in lookup_result.get("items", [])
> ]
> return SupersetAccountLookupResponse(
> status="success",
> environment_id=request.environment_id,
> page_index=request.page_index,
> page_size=request.page_size,
> total=max(int(lookup_result.get("total", len(items))), 0),
> warning=None,
> items=items,
> )
> except Exception as exc:
> logger.explore(
> f"Lookup degraded due to upstream error: {exc}"
> )
> return SupersetAccountLookupResponse(
> status="degraded",
> environment_id=request.environment_id,
> page_index=request.page_index,
> page_size=request.page_size,
> total=0,
> warning=(
> "Cannot load Superset accounts for this environment right now. "
> "You can enter username manually."
> ),
> items=[],
> )
# #endregion lookup_superset_accounts
# #region _resolve_environment [C:2] [TYPE Function]
# @BRIEF Resolve environment model from configured environments by id.
# @PRE environment_id is provided.
# @POST Returns environment object when found else None.
> def _resolve_environment(self, environment_id: str):
> environments = self.config_manager.get_environments()
> for env in environments:
> if str(getattr(env, "id", "")) == str(environment_id):
> return env
> return None
# #endregion _resolve_environment
# #endregion SupersetLookupService
# #endregion superset_lookup_service